Fix History tab £0 recovered: work around T212 pagination dropping orders

README.md

@@ -19,9 +19,11 @@ A web dashboard for viewing your Trading212 portfolio positions at a glance.
 - Buy history tooltips: hover over stock name to see purchase dates and quantities
 - Historical FX rate conversion for foreign-currency orders
 - Incremental history caching: orders and dividends cached to disk
-  - Local: `~/.cache/t2/` — systemd fallback: `/var/cache/t2/` (owner-only permissions)
+  - Location: `$CACHE_DIRECTORY` (set by systemd `CacheDirectory=t2`), then `~/.cache/t2/`, then `/var/cache/t2/` (owner-only permissions)
   - Cold start: fetches all pages from API, saves to disk
   - Warm start: fetches only new data by finding overlap with cache (typically 1 API call)
+  - Optional `backup_dir`: mirror cache to a Dropbox/OneDrive-synced folder for off-machine durability and rehydration after a fresh install
+- Per-ticker reconciliation: works around a Trading212 API bug where the unfiltered paginated `/equity/history/orders` endpoint silently drops orders in certain time windows (the same orders are returned when queried with `?ticker=X`). After each refresh, t2 detects closed positions with suspiciously zero recovered amounts and fetches them per-ticker to fill in the gaps.
 - Sortable columns (click headers to toggle ascending/descending)
 - Default sort by market value descending
 - Auto-refresh every 15 minutes + manual refresh button per row
@@ -104,6 +106,22 @@ The config file is searched in order:
 | `refresh_interval` | `15m` | How often to refresh positions |
 | `listen` | `:8080` | HTTP server listen address |
 | `finnhub_api_key` | (optional) | [Finnhub](https://finnhub.io) API key for US stock fundamentals (free tier) |
+| `backup_dir` | (optional) | Mirror cache to a second directory (e.g. a Dropbox/OneDrive-synced folder) for off-machine durability |
+
+### Running as a systemd service
+
+If you install t2 as a system service running under a dedicated `User=t2`, add the following to your unit file so the cache lives in a directory the service user can actually write to:
+
+```ini
+[Service]
+User=t2
+Group=t2
+CacheDirectory=t2
+StateDirectory=t2
+ExecStart=/usr/bin/t2
+```
+
+`CacheDirectory=t2` makes systemd create `/var/cache/t2/` owned by user `t2` on each start and exposes the path via `$CACHE_DIRECTORY`, which t2 prefers over the user's home (`/nonexistent` for system users). Without this, t2 silently falls back to writing the cache to the working directory (`/`) where it can't be persisted, causing every restart to re-fetch ~1,100 orders from Trading212 (a ~4-minute warmup).
 
 ## Architecture
 

cmd/t2/main.go

@@ -31,7 +31,7 @@ func main() {
 	}
 
 	log.Println("loading instrument and exchange metadata...")
-	svc, err := portfolio.NewService(client, fundsSvc)
+	svc, err := portfolio.NewService(client, fundsSvc, cfg.BackupDir)
 	if err != nil {
 		log.Fatalf("portfolio service: %v", err)
 	}

config.example.yaml

@@ -15,3 +15,13 @@ base_url: "https://live.trading212.com/api/v0"
 # Optional: Finnhub API key for US stock fundamentals (free at https://finnhub.io)
 # Without this, all fundamentals are fetched from Yahoo Finance
 # finnhub_api_key: "your-finnhub-key-here"
+
+# Optional: mirror the orders.json / dividends.json cache files to a second
+# location after every successful fetch. Point this at a Dropbox / OneDrive /
+# rclone-synced directory to keep an off-machine archive of your trading
+# history — useful because Trading212 is known to drop orders from their
+# paginated history endpoint and may eventually purge older records entirely.
+# At startup, if the primary cache is missing but the backup exists, t2
+# rehydrates from it (so a fresh DietPi install picks up where the old one
+# left off). Leave commented out to disable.
+# backup_dir: "/home/dietpi/Dropbox/t2"

internal/config/config.go

@@ -17,6 +17,7 @@ type Config struct {
 	Listen          string        `yaml:"listen"`
 	BaseURL         string        `yaml:"base_url"`
 	FinnhubAPIKey   string        `yaml:"finnhub_api_key"`
+	BackupDir       string        `yaml:"backup_dir"`
 }
 
 // rawConfig is used for YAML unmarshaling since time.Duration
@@ -28,6 +29,7 @@ type rawConfig struct {
 	Listen          string `yaml:"listen"`
 	BaseURL         string `yaml:"base_url"`
 	FinnhubAPIKey   string `yaml:"finnhub_api_key"`
+	BackupDir       string `yaml:"backup_dir"`
 }
 
 // Load reads the configuration file and returns a validated Config.
@@ -54,6 +56,7 @@ func Load() (*Config, error) {
 		Listen:        raw.Listen,
 		BaseURL:       raw.BaseURL,
 		FinnhubAPIKey: raw.FinnhubAPIKey,
+		BackupDir:     raw.BackupDir,
 	}
 
 	// Parse refresh interval with default.

internal/portfolio/history_cache.go

@@ -39,19 +39,31 @@
 }
 
 // cacheDir returns the t2 cache directory path.
-// Prefers ~/.cache/t2 for local users, falls back to /var/cache/t2 for system services.
+// Order: $CACHE_DIRECTORY (set by systemd CacheDirectory=), $HOME/.cache/t2,
+// then /var/cache/t2 as a last resort.
 func cacheDir() string {
-	if home, err := os.UserHomeDir(); err == nil {
+	// systemd sets CACHE_DIRECTORY when the unit has CacheDirectory=t2.
+	// This is the right answer for service installations.
+	if dir := os.Getenv("CACHE_DIRECTORY"); dir != "" {
+		if err := os.MkdirAll(dir, 0700); err == nil {
+			return dir
+		}
+		log.Printf("history-cache: CACHE_DIRECTORY=%q is not writable", dir)
+	}
+	// Honor $HOME but skip the well-known /nonexistent placeholder used for
+	// system users — MkdirAll would silently succeed if a stale directory
+	// existed and silently fail otherwise.
+	if home, err := os.UserHomeDir(); err == nil && home != "" && home != "/nonexistent" {
 		dir := filepath.Join(home, ".cache", "t2")
 		if err := os.MkdirAll(dir, 0700); err == nil {
 			return dir
 		}
 	}
-	// Fallback for systemd services where $HOME is /nonexistent.
 	const fallback = "/var/cache/t2"
 	if err := os.MkdirAll(fallback, 0700); err == nil {
 		return fallback
 	}
+	log.Printf("history-cache: WARNING — no writable cache directory found; cache disabled")
 	return ""
 }
 
@@ -71,6 +83,27 @@
 	return cache.Items
 }
 
+// loadOrdersCacheWithFallback reads the primary cache and falls back to the
+// backup_dir copy if the primary is missing. Used at startup so a fresh
+// install (or wiped cache directory) can rehydrate from the user's cloud
+// backup without re-fetching everything from Trading212.
+func loadOrdersCacheWithFallback(path, backupDir string) []trading212.OrderHistoryItem {
+	if items := loadOrdersCache(path); items != nil {
+		return items
+	}
+	if backupDir == "" {
+		return nil
+	}
+	bp := filepath.Join(backupDir, "orders.json")
+	items := loadOrdersCache(bp)
+	if items != nil {
+		log.Printf("history-cache: rehydrated %d orders from backup_dir (%s)", len(items), bp)
+		// Seed the primary cache so subsequent loads don't re-touch the backup.
+		saveOrdersCache(path, items)
+	}
+	return items
+}
+
 // saveOrdersCache writes the orders cache to disk.
 func saveOrdersCache(path string, items []trading212.OrderHistoryItem) {
 	if path == "" {
@@ -109,6 +142,23 @@
 	return cache.Items
 }
 
+// loadDividendsCacheWithFallback mirrors loadOrdersCacheWithFallback.
+func loadDividendsCacheWithFallback(path, backupDir string) []trading212.DividendHistoryItem {
+	if items := loadDividendsCache(path); items != nil {
+		return items
+	}
+	if backupDir == "" {
+		return nil
+	}
+	bp := filepath.Join(backupDir, "dividends.json")
+	items := loadDividendsCache(bp)
+	if items != nil {
+		log.Printf("history-cache: rehydrated %d dividends from backup_dir (%s)", len(items), bp)
+		saveDividendsCache(path, items)
+	}
+	return items
+}
+
 // saveDividendsCache writes the dividends cache to disk.
 func saveDividendsCache(path string, items []trading212.DividendHistoryItem) {
 	if path == "" {
@@ -131,10 +181,30 @@
 	log.Printf("history-cache: saved %d dividends to disk", len(items))
 }
 
+// mirrorOrders writes the same orders cache to a backup location (e.g. a
+// Dropbox- or rclone-synced directory). Errors are logged but not returned —
+// the primary cache write is what matters for runtime correctness.
+func mirrorOrders(backupDir string, items []trading212.OrderHistoryItem) {
+	if backupDir == "" {
+		return
+	}
+	saveOrdersCache(filepath.Join(backupDir, "orders.json"), items)
+}
+
+// mirrorDividends mirrors the dividends cache to a backup location.
+func mirrorDividends(backupDir string, items []trading212.DividendHistoryItem) {
+	if backupDir == "" {
+		return
+	}
+	saveDividendsCache(filepath.Join(backupDir, "dividends.json"), items)
+}
+
 // fetchOrdersIncremental fetches order history incrementally, using the cache
 // to avoid re-fetching pages that overlap with already-known data.
-func fetchOrdersIncremental(client *trading212.Client, cachePath string) ([]trading212.OrderHistoryItem, error) {
-	cached := loadOrdersCache(cachePath)
+// If backupDir is non-empty, the cache is mirrored there after each successful
+// save and used as a fallback rehydration source if the primary cache is missing.
+func fetchOrdersIncremental(client *trading212.Client, cachePath, backupDir string) ([]trading212.OrderHistoryItem, error) {
+	cached := loadOrdersCacheWithFallback(cachePath, backupDir)
 
 	// Build set of known keys from cache.
 	known := make(map[string]bool, len(cached))
@@ -180,14 +250,17 @@
 	// Prepend new items (newest-first) to cached items.
 	merged := append(newItems, cached...)
 	saveOrdersCache(cachePath, merged)
+	mirrorOrders(backupDir, merged)
 	log.Printf("history-cache: %d new orders fetched, total %d", len(newItems), len(merged))
 	return merged, nil
 }
 
 // fetchDividendsIncremental fetches dividend history incrementally, using the cache
 // to avoid re-fetching pages that overlap with already-known data.
-func fetchDividendsIncremental(client *trading212.Client, cachePath string) ([]trading212.DividendHistoryItem, error) {
-	cached := loadDividendsCache(cachePath)
+// If backupDir is non-empty, the cache is mirrored there after each successful
+// save and used as a fallback rehydration source if the primary cache is missing.
+func fetchDividendsIncremental(client *trading212.Client, cachePath, backupDir string) ([]trading212.DividendHistoryItem, error) {
+	cached := loadDividendsCacheWithFallback(cachePath, backupDir)
 
 	known := make(map[string]bool, len(cached))
 	for _, item := range cached {
@@ -231,6 +304,7 @@
 
 	merged := append(newItems, cached...)
 	saveDividendsCache(cachePath, merged)
+	mirrorDividends(backupDir, merged)
 	log.Printf("history-cache: %d new dividends fetched, total %d", len(newItems), len(merged))
 	return merged, nil
 }