p2pool-starter-stack · VijitSingh97 · Aug 14, 2026 · Aug 14, 2026 · Aug 14, 2026 · Aug 14, 2026
@@ -235,6 +235,12 @@ jobs:
         run: make lint-sh
       - name: Run pithead test suite
         run: bash tests/stack/run.sh
+      - name: Test-inventory drift check
+        # The inventory generator is grep-based; it exits non-zero when any suite it
+        # enumerates counts zero — i.e. a suite moved or changed shape and the inventory
+        # would silently under-count (#981). Output discarded: the generated file is
+        # git-ignored and read on demand via `make test-inventory`.
+        run: bash tests/inventory.sh > /dev/null
       - name: Run integration harness self-test
         # Pure-logic checks for the tests/integration/ harness (config rendering, matrix
         # coverage, redaction). The LIVE matrix (tests/integration/run.sh) needs a real test

@@ -50,7 +50,11 @@ runs `ruff` (plus a few hygiene hooks) on your changed files. If you change depe
      (`lychee`) runs on a weekly schedule, not per-PR.
    - **test-dashboard** — the dashboard `pytest` suite (must stay ≥ the **80% total coverage gate**).
      CI also runs **`make test-patch-coverage`** (`diff-cover`): new/changed lines must be **≥ 90%**
-     covered vs `origin/develop`, the ratchet that stops coverage rotting at the margin.
+     covered vs `origin/develop`, the ratchet that stops coverage rotting at the margin. The gate
+     says so explicitly when a diff has nothing it measures (shell/docs-only PRs pass loudly), and
+     fails if a changed dashboard Python file is missing from `coverage.xml` entirely — the
+     silent no-op it used to be. Run it right after `make test-dashboard`, so `coverage.xml`
+     is fresh.
    - **test-frontend** — the frontend logic tests (`node --test`); uses the same Node that the
      lint surfaces already require.
    - **test-stack** — the `pithead` shell test suite.

@@ -10,9 +10,8 @@ test-dashboard: ## Dashboard unit/component tests with coverage gate (deps from
 test-frontend: ## Frontend logic tests with Node's built-in runner (#632; same invocation as CI)
 	node --test build/dashboard/tests/frontend/*.test.mjs
 
-test-patch-coverage: ## diff-cover (#286): new/changed lines must be >=90% covered (run after test-dashboard)
-	cd build/dashboard && uv run --locked --extra test \
-		diff-cover coverage.xml --compare-branch=origin/$${GITHUB_BASE_REF:-develop} --fail-under=90
+test-patch-coverage: ## diff-cover (#286) minus its vacuous pass (#1000): >=90% on changed lines (run after test-dashboard)
+	bash scripts/patch-coverage.sh
 
 test-stack: ## pithead shell test suite
 	bash tests/stack/run.sh

@@ -50,8 +50,10 @@ RUN python -m pytest --cov=mining_dashboard --cov-report=term-missing --cov-fail
 # ==========================================================================
 FROM base AS production
 COPY --from=build /app/.venv /app/.venv
-COPY entrypoint.sh .
-RUN chmod +x entrypoint.sh
+# entrypoint runs the app; healthcheck.sh (#904) HEADs /api/state with the venv's own python3
+# (the slim image ships no curl/wget).
+COPY entrypoint.sh healthcheck.sh ./
+RUN chmod +x entrypoint.sh healthcheck.sh
 
 # Stack version, baked at build so the running container is self-describing (Issue #58).
 # PITHEAD_VERSION comes from the top-level VERSION file (the source of truth); the git args

@@ -29,10 +29,13 @@ Testing: the Python API, where the logic and formatting live, is unit-tested. Th
 tests run under Node's built-in runner (`node --test build/dashboard/tests/frontend/`) — no
 `package.json`/`node_modules`/build step, so the repo stays Node-free. They cover the pure logic
 (`logic.test.mjs`: worker sort, tooltip formatting, hero-KPI selection), the chart helpers
-(`chart.test.mjs`: `withAlpha`, `padYAxis`), the topology geometry (`topology.test.mjs`), and
+(`chart.test.mjs`: `withAlpha`, `padYAxis`), the topology geometry (`topology.test.mjs`),
 component rendering (`components.test.mjs`: every card driven through `App` against a real
-`build_state()` fixture, via a DOM-free vnode walker). The DOM-bound wiring (Chart.js canvas,
-the SVG topology component) needs a browser and is left to a manual smoke test.
+`build_state()` fixture, via a DOM-free vnode walker), and the entry point
+(`dashboard.test.mjs`: `initDashboard()` takes its browser seams — DOM, storage, fetch, history,
+timer, render — as parameters with real defaults, so the poll loop, its hang-abort, and the
+preference wiring run against fakes). The DOM-bound wiring (Chart.js canvas, the SVG topology
+component) needs a browser and is left to a manual smoke test.
 
 ## Layout
 

@@ -0,0 +1,10 @@
+#!/bin/bash
+# Dashboard HTTP liveness (#904).
+#
+# HEAD /api/state against the app's fixed loopback bind (main.py: 127.0.0.1:8000). aiohttp
+# serves HEAD on every GET route, so a 200 proves the web server is listening AND build_state
+# assembles — the exact "container Up while Caddy serves 502s" state the v1.8.1 one-click
+# incident exposed (#622). The slim image ships no curl/wget; python3 (the app's own venv,
+# already on PATH) probes with stdlib urllib, which raises — exiting non-zero — on any
+# connect failure or non-2xx status.
+exec python3 -c 'import urllib.request as u; u.urlopen(u.Request("http://127.0.0.1:8000/api/state", method="HEAD"), timeout=5)'
@@ -78,6 +78,12 @@ def get_sync_status(self):
         `db_size` is monerod's on-disk database size in bytes (from get_info, available even
         under restricted RPC). The UI shows it next to the configured pruned/full mode so a
         config/DB mismatch is visible at a glance (Issue #32).
+
+        `synchronized` is monerod's raw network-sync verdict, passed through for the peer-loss
+        detector (#972): after a tor restart a stranded node can read as "synced" here (stale
+        target_height 0) while `synchronized` is false. Only this RPC path sets the key — the
+        log-scrape fallback and remote nodes have no verdict, and the detector treats absence
+        as no verdict.
         """
         info = self.get_info()
         if info is None:
@@ -86,12 +92,13 @@ def get_sync_status(self):
         height = int(info.get("height", 0) or 0)
         target = int(info.get("target_height", 0) or 0)
         db_size = int(info.get("database_size", 0) or 0)
+        synchronized = bool(info.get("synchronized", False))
 
         # `synchronized` is monerod's authoritative "caught up" flag; once synced it also
         # reports target_height: 0. Trust it over the height comparison (mirrors how the
         # Tari client trusts initial_sync_achieved).
-        if info.get("synchronized") or target == 0 or height >= target:
-            return {"is_syncing": False, "db_size": db_size}
+        if synchronized or target == 0 or height >= target:
+            return {"is_syncing": False, "db_size": db_size, "synchronized": synchronized}
 
         percent = int((height / target) * 100)
         return {
@@ -100,4 +107,5 @@ def get_sync_status(self):
             "target": target,
             "percent": percent,
             "db_size": db_size,
+            "synchronized": synchronized,
         }
@@ -79,16 +79,20 @@ def parse_rigforge(payload):
     }
 
 
-# Terminal control-apply outcomes (pithead control_worker_apply / rigforge#236). "accepted" and
-# "running" are non-terminal — never reconciled from a read poll, only ever written by the host
-# runner itself while a change is still in flight.
-_CONTROL_TERMINAL = ("applied", "rejected", "rolled_back")
+# Terminal control outcomes the rig may mirror: applied/rejected/rolled_back/failed from a
+# control-apply (pithead control_worker_apply / rigforge#236), plus noop/throttled from a
+# control-upgrade (rigforge#320, v1.12.0). "started" (rigforge#320's in-flight upgrade marker) and
+# "accepted"/"running" (this dashboard's own still-polling placeholders) are non-terminal — never
+# reconciled from a read poll, only ever written while a change is still in flight. Mirrors
+# pithead's own control_worker_apply/control_worker_upgrade poll cases (#1001) so the mirror-side
+# and poll-side vocabularies can't drift apart again.
+_CONTROL_TERMINAL = ("applied", "rejected", "rolled_back", "failed", "noop", "throttled")
 
 
 def parse_worker_control_status(payload):
-    """The rig's last control-apply outcome, mirrored read-only into the SAME enriched feed body
-    under ``rigforge.control`` (#579) — no new port, no token, it rides the poll that already
-    fetches the ``rigforge`` block for :func:`parse_rigforge`.
+    """The rig's last control-apply/control-upgrade outcome, mirrored read-only into the SAME
+    enriched feed body under ``rigforge.control`` (#579, rigforge#346) — no new port, no token, it
+    rides the poll that already fetches the ``rigforge`` block for :func:`parse_rigforge`.
 
     The host runner's synchronous ``/status`` poll after a worker-apply is capped at 20s
     (rigforge#236's auto-rollback can take minutes); a change still mid-flight past that deadline
@@ -97,10 +101,12 @@ def parse_worker_control_status(payload):
     mirrors its own last outcome into the already-open, unauthenticated read feed so the next
     routine poll can catch up.
 
-    Returns ``{"change_id", "status", "reason"}`` only for a TERMINAL outcome
-    (``applied``/``rejected``/``rolled_back``); ``None`` for a still-in-flight change, a malformed
-    block, or a rig that doesn't mirror this yet (older RigForge, plain xmrig) — so a #185 history
-    row is never force-terminaled on bad or absent data.
+    Returns ``{"change_id", "status", "reason"}`` only for a TERMINAL outcome: applied / rejected /
+    rolled_back / failed (rigforge#236), or noop (already on the target)/throttled (the rig's own
+    anti-beacon window — retry-later, not a fault) from rigforge#320. Returns ``None`` for a
+    still-in-flight change (``started``/``accepted``/``running``), a malformed block, or a rig that
+    doesn't mirror this yet (older RigForge, plain xmrig) — so a #185 history row is never
+    force-terminaled on bad or absent data.
     """
     rf = payload.get("rigforge") if isinstance(payload, dict) else None
     ctrl = rf.get("control") if isinstance(rf, dict) else None

@@ -412,6 +412,11 @@ def _nonneg(v):
 # How long a preview/commit POST waits for the host-side runner's result before returning 202 and
 # leaving the client to poll /api/control/result. The systemd path unit fires within seconds.
 CONTROL_WAIT_S = float(os.environ.get("CONTROL_WAIT_S", 30))
+# A worker-upgrade POST never waits inline (a rig rebuild can run minutes) — it returns 202 at
+# once and a background task records the terminal outcome once the host runner writes it. This
+# bounds that background wait; matches the client's own polling budget (workerview.mjs
+# UPGRADE_POLL_MAX = 150 * 2s = 300s), well past the host's own ~90s dial+poll cap.
+CONTROL_WORKER_UPGRADE_WAIT_S = float(os.environ.get("CONTROL_WORKER_UPGRADE_WAIT_S", 300))
 GITHUB_RELEASES_API = os.environ.get(
     "GITHUB_RELEASES_API",
     "https://api.github.com/repos/p2pool-starter-stack/pithead/releases/latest",
@@ -503,6 +508,13 @@ def _nonneg(v):
 NODE_DOWN_AFTER_SEC = int(os.environ.get("NODE_DOWN_AFTER_SEC", 90))
 NODE_RECOVERY_AFTER_SEC = int(os.environ.get("NODE_RECOVERY_AFTER_SEC", 60))
 
+# Peer-loss staleness (#972): a reachable monerod reporting `synchronized: false` must persist
+# this long before the out-of-sync alert fires. 10 minutes rides out normal tip-lag blips AND
+# the coupled monerod restart after a tor recreate (compose depends_on restart / tor_heal), so
+# only a node that genuinely failed to re-peer alarms. Env-only (tests/mini-stack), not a
+# config.json knob — nobody should have to tune a detector.
+NODE_STALE_AFTER_SEC = int(os.environ.get("NODE_STALE_AFTER_SEC", 600))
+
 # --- Healthchecks.io dead-man's switch (Issue #79) ---
 # Optional external liveness monitor. Set a ping URL and the dashboard loop pings it every cycle;
 # if the whole host dies (power loss, kernel panic, NIC death) the dashboard dies with it, the pings

@@ -61,6 +61,10 @@ class AlertService:
       flag per node (#31). Tari is only alerted when it's treated as required; a non-blocking
       Tari going down isn't operator-critical (we keep mining Monero), matching the
       worker-rejection rule.
+    - **node out of sync / back in sync** — the debounced peer-loss strand (#972): monerod
+      reachable and healthy-looking but reporting ``synchronized: false`` past the stale
+      threshold (a tor restart kills its SOCKS peers and it doesn't re-dial). Rides the
+      ``node_down``/``node_recovered`` toggles — same conversation, different failure mode.
     - **sync finished** — the sync gate's ``miner_released`` latch flipping open once (#35).
     - **worker offline / back online / joined / left** — a debounced :class:`WorkerPresenceMonitor`
       over the live worker rows (offline keys off the same DOWN status the dashboard shows; joined /
@@ -184,6 +188,7 @@ def __init__(
         self.host_label = "" if host_label in (None, "", "Unknown Host") else host_label
         # None = "not yet observed": the first cycle seeds the baseline without emitting.
         self._prev_monero_down = None
+        self._prev_monero_stale = None
         self._prev_tari_down = None
         self._prev_released = None
         self._prev_disk_level = None
@@ -232,6 +237,7 @@ def evaluate(
         self,
         *,
         monero_down,
+        monero_stale=False,
         tari_down,
         tari_required,
         miner_released,
@@ -273,6 +279,7 @@ def evaluate(
 
         # --- Node down / recovered (consume NodeHealthMonitor edges) ---
         alerts += self._node_edges("Monero", monero_down, "_prev_monero_down")
+        alerts += self._stale_edges(monero_stale)
         if tari_required:
             alerts += self._node_edges("Tari", tari_down, "_prev_tari_down")
         else:
@@ -380,6 +387,35 @@ def _node_edges(self, label, down, attr):
             )
         ]
 
+    def _stale_edges(self, stale):
+        """Monero node reachable but OUT of sync (#972): the debounced ``synchronized: false``
+        strand a tor restart leaves behind. Distinct from node-down — the node answers its RPC
+        and every container reads healthy while mining sits on a stale tip. Rides the
+        node_down/node_recovered toggles: same conversation, different failure mode."""
+        prev = self._prev_monero_stale
+        self._prev_monero_stale = stale
+        if prev is None or stale == prev:
+            return []
+        if stale:
+            self._record_incident(self.EVT_NODE_DOWN)
+            return [
+                (
+                    self.EVT_NODE_DOWN,
+                    self._fmt(
+                        "\U0001f534 ⛓️ Monero node is OUT OF SYNC — reachable but reporting "
+                        "not-synchronized (peers usually die like this after a Tor restart). "
+                        "Mining sits on a stale tip until it re-peers: run "
+                        "'./pithead restart monerod'."
+                    ),
+                )
+            ]
+        return [
+            (
+                self.EVT_NODE_RECOVERED,
+                self._fmt("\U0001f7e2 ⛓️ Monero node is back in sync with the network."),
+            )
+        ]
+
     def _disk_edges(self, disk_percent):
         """Alert on the data disk crossing the dashboard's own warn/critical thresholds (#138)."""
         level = (

@@ -52,6 +52,7 @@
     HOST_IP,
     MONERO_CLEARNET_SYNC,
     MONERO_WALLET_ADDRESS,
+    NODE_STALE_AFTER_SEC,
     PAYOUT_CONFIRM_ENABLED,
     REJECT_WORKERS_CONTAINER,
     SYNC_GATE_CONTAINERS,
@@ -636,6 +637,11 @@ def __init__(self, state_manager, proxy_client, xvb_client):
         self.docker_control = DockerControl()
         self.monero_health = NodeHealthMonitor()
         self.tari_health = NodeHealthMonitor()
+        # Peer-loss staleness (#972): the same debounce machine, fed monerod's own
+        # `synchronized` flag instead of reachability. "Ever synchronized" plays the ever-up
+        # guard, so a node mid-initial-sync (synchronized false for days) never alarms; only a
+        # node that WAS in sync and stayed out for NODE_STALE_AFTER_SEC trips `down` (= stale).
+        self.monero_sync_stale = NodeHealthMonitor(down_after=NODE_STALE_AFTER_SEC)
 
         # Healthchecks.io dead-man's switch (Issue #79). Disabled by default — when off this is
         # a no-op. When on, each cycle pings a unique URL; the alert fires externally on the
@@ -1512,6 +1518,17 @@ async def run(self):
                     monero_sync["down"] = monero_down
                     tari_sync["down"] = tari_down
 
+                    # 3b. Peer-loss staleness (#972): monerod can survive a tor restart with
+                    # every SOCKS peer dead — reachable, healthcheck green, height creeping,
+                    # but `synchronized: false` for hours. The RPC path is the only one that
+                    # carries the flag; absence (log-scrape fallback, remote node) is no
+                    # verdict, so the monitor isn't fed and its streaks stay put.
+                    monero_reports_synced = monero_sync.get("synchronized")
+                    if monero_reports_synced is not None:
+                        self.monero_sync_stale.update(monero_reports_synced)
+                    monero_stale = self.monero_sync_stale.down
+                    monero_sync["stale"] = monero_stale
+
                     # 4. Sync gate (Issue #35): hold p2pool + xmrig-proxy until the required
                     # chain(s) first sync, then release. monerod must be synced; Tari must be
                     # synced too unless it's non-blocking. #31's runtime failover only applies
@@ -1567,6 +1584,9 @@ async def run(self):
                     )
                     await self.alert_service.process(
                         monero_down=monero_down,
+                        # Debounced "reachable but out of sync" (#972) — the 0-peer strand
+                        # after a tor restart that node-down can't see.
+                        monero_stale=monero_stale,
                         tari_down=tari_down,
                         tari_required=TARI_REQUIRED,
                         miner_released=self.miner_released,