From 37f609b736847a26ff461b838eb4b63c7cd35c7f Mon Sep 17 00:00:00 2001
From: Azorlogh <bott.alix@gmail.com>
Date: Fri, 10 Jul 2026 07:04:12 +0000
Subject: [PATCH 01/17] feat(antithesis): exercise RebuildDelta via a
 model-test restore cycle

The model driver periodically drains its workers, hands the quiesced
ledger to an external orchestrator that backs it up and restores the
backup into a fresh store (running the RebuildDelta replay), then
relaunches the node on it. The driver keeps running against the restored
node, so its ordinary read/commit checks validate the rebuilt state with
no separate comparison.

The trigger is environment-specific behind RestoreTrigger: locally a
request/response file rendezvous consumed by run_model_test.sh (bash
polls files far more robustly than sockets); the operator-driven k8s path
is left for the Antithesis environment.

run_model_test.sh --restore (single-node) stands up MinIO, takes a
startup full backup, and per cycle runs an incremental backup + store
bootstrap into a fresh dir + relaunch on the restored store with a fresh
WAL (the RESTORED marker self-bootstraps it). Each cycle asserts the
manifest carried exports, since RebuildDelta only runs on the delta
beyond the checkpoint.

pauseAndDrain flushes the re-order buffer once nothing is outstanding:
handleObservation's failure/transient paths drop the inflight ticket
without calling tryDrain, so with no further op to re-trigger it a
committed success could otherwise sit un-drained forever.
---
 tests/antithesis/run_model_test.sh            | 133 ++++++++++++-
 .../model/singleton_driver_model/checker.go   |   5 +
 .../model/singleton_driver_model/config.go    |  15 ++
 .../cmds/model/singleton_driver_model/main.go |  26 ++-
 .../model/singleton_driver_model/restore.go   | 178 ++++++++++++++++++
 5 files changed, 352 insertions(+), 5 deletions(-)
 create mode 100644 tests/antithesis/workload/bin/cmds/model/singleton_driver_model/restore.go

diff --git a/tests/antithesis/run_model_test.sh b/tests/antithesis/run_model_test.sh
index f9ad6f674f..96f477704a 100755
--- a/tests/antithesis/run_model_test.sh
+++ b/tests/antithesis/run_model_test.sh
@@ -43,17 +43,30 @@ set -uo pipefail
 
 # --- Arguments ------------------------------------------------------------
 NODES="${NODES:-1}"
+RESTORE="${RESTORE:-0}"
 DURATION=""
 while [ $# -gt 0 ]; do
 	case "$1" in
 		--cluster)  NODES=3; shift ;;
 		--nodes)    NODES="$2"; shift 2 ;;
 		--nodes=*)  NODES="${1#*=}"; shift ;;
+		--restore)  RESTORE=1; shift ;;
 		-h|--help)  sed -n '2,40p' "$0"; exit 0 ;;
 		*)          DURATION="$1"; shift ;;
 	esac
 done
 
+case "$RESTORE" in ''|0|off|false|no) RESTORE=0 ;; *) RESTORE=1 ;; esac
+# The restore cycle periodically backs the node up, restores the backup into a
+# fresh store (exercising RebuildDelta), and relaunches the node on it while the
+# driver keeps running -- its ordinary checks validate the rebuilt state. Only
+# single-node is wired: a whole-store swap keeps the served state == RebuildDelta's
+# output (a cluster rejoin would snapshot-install over it).
+if [ "$RESTORE" = 1 ] && [ "$NODES" -ne 1 ]; then
+	echo "ERROR: --restore is only supported with a single node (got --nodes $NODES)" >&2
+	exit 2
+fi
+
 case "$NODES" in
 	''|*[!0-9]*) echo "ERROR: --nodes must be a positive integer (got '$NODES')" >&2; exit 2 ;;
 esac
@@ -98,6 +111,7 @@ REPO="${REPO:-$(cd "$SCRIPT_DIR/../.." && pwd)}"
 GRPC_BASE="${GRPC_BASE:-$(( 20000 + RANDOM % 10000 ))}"
 RAFT_BASE=$(( GRPC_BASE + 100 ))
 HTTP_BASE=$(( GRPC_BASE + 200 ))
+MINIO_PORT=$(( GRPC_BASE + 300 ))
 
 WORKDIR="$(mktemp -d /tmp/model-test.XXXXXX)"
 DRIVER_LOG="$WORKDIR/driver.log"
@@ -109,6 +123,20 @@ DRIVER_PID=""
 RECOVERY_FAILED=0
 DRIVER_EXITED_EARLY=0
 
+# --- Restore cycle (single-node, --restore) -------------------------------
+RESTORE_INTERVAL="${RESTORE_INTERVAL:-45}"
+RESTORE_REQ="$WORKDIR/restore.req"
+RESTORE_RESP="$WORKDIR/restore.resp"
+RESTORED_DATA="$WORKDIR/restored-data"
+MINIO_CONTAINER="model-minio-$$"
+S3_BUCKET="backups"
+S3_FLAGS=(
+	--driver s3 --s3-bucket "$S3_BUCKET"
+	--s3-endpoint "http://127.0.0.1:$MINIO_PORT" --s3-region us-east-1
+	--s3-access-key-id minioadmin --s3-secret-access-key minioadmin
+	--bucket-id "$CLUSTER_ID"
+)
+
 GRPC_PORTS=() RAFT_PORTS=() HTTP_PORTS=() NODE_DIRS=() SERVER_LOGS=() SERVER_PIDS=()
 for i in $(seq 0 $(( NODES - 1 ))); do
 	GRPC_PORTS+=( $(( GRPC_BASE + i )) )
@@ -129,11 +157,79 @@ cleanup() {
 	sleep 1
 	[ -n "$DRIVER_PID" ] && kill -9 "$DRIVER_PID" 2>/dev/null
 	for pid in "${SERVER_PIDS[@]}"; do [ -n "$pid" ] && kill -9 "$pid" 2>/dev/null; done
+	[ "$RESTORE" = 1 ] && stop_minio
 	wait 2>/dev/null
 	if [ -z "${KEEP_WORKDIR:-}" ]; then rm -rf "$WORKDIR"; else log "work dir kept at $WORKDIR"; fi
 }
 trap cleanup EXIT INT TERM
 
+# --- MinIO (backup storage for --restore) ---------------------------------
+start_minio() {
+	docker rm -f "$MINIO_CONTAINER" >/dev/null 2>&1
+	docker run -d --rm --name "$MINIO_CONTAINER" -p "127.0.0.1:$MINIO_PORT:9000" \
+		-e MINIO_ROOT_USER=minioadmin -e MINIO_ROOT_PASSWORD=minioadmin \
+		minio/minio:RELEASE.2024-06-13T22-53-53Z server /data >/dev/null 2>&1 || return 1
+	# Retry until MinIO is reachable, then create the bucket (idempotent). The mc
+	# image's entrypoint is mc itself, so override it to run a shell.
+	for _ in $(seq 1 30); do
+		if docker run --rm --network host --entrypoint sh minio/mc:RELEASE.2024-06-12T14-34-03Z -c \
+			"mc alias set m http://127.0.0.1:$MINIO_PORT minioadmin minioadmin && mc mb --ignore-existing m/$S3_BUCKET" >/dev/null 2>&1; then
+			return 0
+		fi
+		sleep 1
+	done
+	return 1
+}
+stop_minio() { docker rm -f "$MINIO_CONTAINER" >/dev/null 2>&1; }
+
+# --- Restore orchestrator (single-node) -----------------------------------
+# One cycle: incremental-backup the (quiesced) node, kill it, bootstrap the
+# backup into a fresh store (RebuildDelta), then relaunch on it. The driver is
+# paused across this and resumes against the restored node.
+do_one_restore() {
+	local srv="127.0.0.1:${GRPC_PORTS[0]}" out
+	out="$("$LEDGERCTL_BIN" store incremental-backup --insecure --server "$srv" \
+		"${S3_FLAGS[@]}" --json --timeout 120s 2>&1)" \
+		|| { log "restore: incremental-backup failed: $(printf '%s' "$out" | tail -1)"; return 1; }
+
+	# RebuildDelta only runs when the manifest carries exports; a cycle that
+	# produced none would silently test nothing, so skip it loudly.
+	if command -v jq >/dev/null 2>&1; then
+		local segs logs
+		segs="$(printf '%s' "$out" | jq -r '.segmentsUploaded // 0' 2>/dev/null)"
+		logs="$(printf '%s' "$out" | jq -r '.logEntriesExported // 0' 2>/dev/null)"
+		if [ "${segs:-0}" = 0 ] && [ "${logs:-0}" = 0 ]; then
+			log "restore: incremental produced no exports (RebuildDelta would be a no-op); skipping"
+			return 1
+		fi
+	fi
+
+	kill -9 "${SERVER_PIDS[0]}" 2>/dev/null; wait "${SERVER_PIDS[0]}" 2>/dev/null; SERVER_PIDS[0]=""
+	rm -rf "$RESTORED_DATA"
+	if ! "$LEDGERCTL_BIN" store bootstrap --data-dir "$RESTORED_DATA" "${S3_FLAGS[@]}" -y >>"$WORKDIR/restore.log" 2>&1; then
+		log "restore: bootstrap failed; relaunching on original data"
+		start_node 0 rejoin; wait_leader 0
+		return 1
+	fi
+
+	# Swap the restored store in with a fresh WAL: on boot the RESTORED marker
+	# makes the node self-bootstrap as a single voter from the rebuilt data.
+	rm -rf "${NODE_DIRS[0]}/data" "${NODE_DIRS[0]}/wal"
+	mv "$RESTORED_DATA" "${NODE_DIRS[0]}/data"
+	start_node 0 bootstrap
+	wait_leader 0 || return 1
+	log "restore: node back up on restored (RebuildDelta) store"
+}
+
+# service_restore_request handles one pending request, if any. Runs in the main
+# shell (not a subshell) so start_node's SERVER_PIDS update is visible to cleanup.
+service_restore_request() {
+	[ "$RESTORE" = 1 ] && [ -f "$RESTORE_REQ" ] || return 0
+	rm -f "$RESTORE_REQ"
+	log "restore: cycle requested"
+	if do_one_restore; then printf 'ok\n' >"$RESTORE_RESP"; else printf 'err\n' >"$RESTORE_RESP"; fi
+}
+
 if [ ! -d "$REPO" ]; then
 	echo "ERROR: REPO not found at $REPO (set REPO=...)" >&2
 	exit 2
@@ -237,9 +333,13 @@ check_fail_fast() {
 # ---------------------------------------------------------------------------
 # Build (server + driver; ledgerctl only when a cluster needs health checks).
 # ---------------------------------------------------------------------------
-build_cmd="go build -o '$SERVER_BIN' . && "
-if [ "$NODES" -gt 1 ]; then
-	build_cmd="${build_cmd}go build -o '$LEDGERCTL_BIN' ./cmd/ledgerctl && "
+# --restore drives S3 backups (to MinIO), so the server and ledgerctl need the
+# s3 build tag; the light default build stubs S3 out.
+server_tags=""
+[ "$RESTORE" = 1 ] && server_tags="-tags s3"
+build_cmd="go build $server_tags -o '$SERVER_BIN' . && "
+if [ "$NODES" -gt 1 ] || [ "$RESTORE" = 1 ]; then
+	build_cmd="${build_cmd}go build $server_tags -o '$LEDGERCTL_BIN' ./cmd/ledgerctl && "
 fi
 build_cmd="${build_cmd}cd tests/antithesis/workload && go build -o '$DRIVER_BIN' ./bin/cmds/model/singleton_driver_model"
 
@@ -285,6 +385,24 @@ else
 	log "server ready"
 fi
 
+# ---------------------------------------------------------------------------
+# Restore setup (--restore): MinIO + one full backup as the checkpoint every
+# per-cycle incremental exports beyond (so bootstrap replays through RebuildDelta).
+# ---------------------------------------------------------------------------
+if [ "$RESTORE" = 1 ]; then
+	log "starting MinIO (:$MINIO_PORT) for backup storage..."
+	if ! start_minio; then echo "ERROR: MinIO did not start" >&2; exit 2; fi
+	log "taking initial full backup..."
+	if ! "$LEDGERCTL_BIN" store backup --insecure --server "127.0.0.1:${GRPC_PORTS[0]}" \
+		"${S3_FLAGS[@]}" --timeout 120s >>"$WORKDIR/restore.log" 2>&1; then
+		echo "ERROR: initial backup failed (see $WORKDIR/restore.log)" >&2
+		[ -n "${KEEP_WORKDIR:-}" ] || cat "$WORKDIR/restore.log" >&2
+		exit 2
+	fi
+	command -v jq >/dev/null 2>&1 || log "WARNING: jq not found; cannot verify each cycle produced exports"
+	log "restore cycle armed (interval ~${RESTORE_INTERVAL}s)"
+fi
+
 # ---------------------------------------------------------------------------
 # Run the driver against all node(s).
 # ---------------------------------------------------------------------------
@@ -295,12 +413,19 @@ log "running driver for ${DURATION}s against $ADDR_LIST ..."
 # MODEL_MAX_SECONDS makes the driver self-terminate even if this script never
 # gets to signal it (defence-in-depth against orphaned drivers). A small buffer
 # over DURATION lets the script-driven stop win in the normal case.
+RESTORE_REQ_ENV="" RESTORE_RESP_ENV="" RESTORE_INTERVAL_ENV=""
+if [ "$RESTORE" = 1 ]; then
+	RESTORE_REQ_ENV="$RESTORE_REQ"; RESTORE_RESP_ENV="$RESTORE_RESP"; RESTORE_INTERVAL_ENV="$RESTORE_INTERVAL"
+fi
 LEDGER_GRPC_ADDR="$ADDR_LIST" \
 ANTITHESIS_SDK_LOCAL_OUTPUT="$ASSERTIONS" \
 MODEL_DEBUG="${MODEL_DEBUG:-}" \
 MODEL_LEDGERS="${MODEL_LEDGERS:-}" \
 MODEL_WORKERS="${MODEL_WORKERS:-}" \
 MODEL_MAX_SECONDS="$(( DURATION + 15 ))" \
+MODEL_RESTORE_REQ="$RESTORE_REQ_ENV" \
+MODEL_RESTORE_RESP="$RESTORE_RESP_ENV" \
+MODEL_RESTORE_INTERVAL="$RESTORE_INTERVAL_ENV" \
 	"$DRIVER_BIN" > "$DRIVER_LOG" 2>&1 &
 DRIVER_PID=$!
 
@@ -341,6 +466,8 @@ while [ "$(date +%s)" -lt "$deadline" ]; do
 		next_restart=$(( $(date +%s) + RESTART_INTERVAL ))
 		continue
 	fi
+
+	service_restore_request
 	sleep 1
 done
 
diff --git a/tests/antithesis/workload/bin/cmds/model/singleton_driver_model/checker.go b/tests/antithesis/workload/bin/cmds/model/singleton_driver_model/checker.go
index 2e0b19eba6..4227daa05b 100644
--- a/tests/antithesis/workload/bin/cmds/model/singleton_driver_model/checker.go
+++ b/tests/antithesis/workload/bin/cmds/model/singleton_driver_model/checker.go
@@ -56,6 +56,11 @@ type Checker struct {
 	// receipt-carried revert path. Populated at commit (captureReceipts) and read
 	// during generation, both under mu.
 	receiptByRef map[string]string
+
+	// paused gates worker dispatch during a restore cycle; resumeCh is closed on
+	// resume so parked workers wake. Both guarded by mu (see restore.go).
+	paused   bool
+	resumeCh chan struct{}
 }
 
 // One worker → processor message. observeTicket is the ticket high-water mark
diff --git a/tests/antithesis/workload/bin/cmds/model/singleton_driver_model/config.go b/tests/antithesis/workload/bin/cmds/model/singleton_driver_model/config.go
index 79cdb3f69f..dea3c66c1e 100644
--- a/tests/antithesis/workload/bin/cmds/model/singleton_driver_model/config.go
+++ b/tests/antithesis/workload/bin/cmds/model/singleton_driver_model/config.go
@@ -125,3 +125,18 @@ const workerLoopPause = 100 * time.Millisecond
 
 // Worker → processor channel cap, well above steady-state inflight.
 const incomingBuffer = 256
+
+// --- Restore cycle ------------------------------------------------------
+
+// Poll cadence while waiting for the driver to fully drain before a backup.
+const quiescePoll = 25 * time.Millisecond
+
+// Poll cadence and hard cap while waiting for the external orchestrator to finish
+// a restore. The cap is a lease: a dead orchestrator cannot park workers forever.
+const (
+	restorePoll    = 200 * time.Millisecond
+	restoreTimeout = 3 * time.Minute
+)
+
+// Base restore-cycle interval (seconds) when MODEL_RESTORE_INTERVAL is unset.
+const defaultRestoreIntervalSecs = 90
diff --git a/tests/antithesis/workload/bin/cmds/model/singleton_driver_model/main.go b/tests/antithesis/workload/bin/cmds/model/singleton_driver_model/main.go
index 4225dc0289..9550eca3dd 100644
--- a/tests/antithesis/workload/bin/cmds/model/singleton_driver_model/main.go
+++ b/tests/antithesis/workload/bin/cmds/model/singleton_driver_model/main.go
@@ -132,9 +132,20 @@ func main() {
 		}()
 	}
 
-	// Workers stop on ctx.Done. Close the processor's channel so it can
-	// drain and exit.
+	var restore sync.WaitGroup
+	if trigger := selectRestoreTrigger(); trigger != nil {
+		restore.Add(1)
+		go func() {
+			defer restore.Done()
+			runRestoreCycle(ctx, checker, trigger, restoreInterval())
+		}()
+		log.Printf("restore cycle enabled (interval ~%s)", restoreInterval())
+	}
+
+	// Workers stop on ctx.Done. Wait for the restore cycle too before closing the
+	// processor's channel, so no cycle touches the checker during teardown.
 	workers.Wait()
+	restore.Wait()
 	close(checker.incoming)
 	processors.Wait()
 }
@@ -153,6 +164,11 @@ func runWorker(
 		default:
 		}
 
+		// Park here while a restore cycle has quiesced the driver.
+		if !c.awaitResume(ctx) {
+			return
+		}
+
 		// 1-in-5: a read this iteration, split across the whole-ledger read
 		// (chart + ledger metadata), a single-account read, a transaction read
 		// (id + postings + reverted + metadata), and a metadata-schema read
@@ -174,6 +190,12 @@ func runWorker(
 		}
 
 		c.mu.Lock()
+		// A pause committed after the gate opened: back out without dispatching,
+		// so no bulk commits between the drain and the backup.
+		if c.paused {
+			c.mu.Unlock()
+			continue
+		}
 		bulk := generateBulk(c.modelState, c.ledgerNames, c.receiptByRef)
 		if len(bulk.Requests) == 0 {
 			c.mu.Unlock()
diff --git a/tests/antithesis/workload/bin/cmds/model/singleton_driver_model/restore.go b/tests/antithesis/workload/bin/cmds/model/singleton_driver_model/restore.go
new file mode 100644
index 0000000000..077ac97dd8
--- /dev/null
+++ b/tests/antithesis/workload/bin/cmds/model/singleton_driver_model/restore.go
@@ -0,0 +1,178 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/formancehq/ledger/v3/tests/antithesis/workload/internal"
+)
+
+// The restore cycle periodically hands the live ledger to an external
+// orchestrator that backs it up, restores the backup into a fresh store (running
+// the RebuildDelta replay), and brings a node back up on it. The driver keeps
+// running against the restored node afterward, so its ordinary read/commit
+// checks validate the rebuilt state — no separate comparison. The driver owns
+// only the timing and the quiescence; the environment-specific work lives behind
+// RestoreTrigger (a file rendezvous locally, the k8s client on Antithesis).
+
+// awaitResume blocks while a restore cycle has paused dispatch, returning false
+// if ctx ends. Safe to call without holding mu.
+func (c *Checker) awaitResume(ctx context.Context) bool {
+	for {
+		c.mu.Lock()
+		if !c.paused {
+			c.mu.Unlock()
+			return true
+		}
+		ch := c.resumeCh
+		c.mu.Unlock()
+
+		select {
+		case <-ch:
+		case <-ctx.Done():
+			return false
+		}
+	}
+}
+
+// pauseAndDrain stops new dispatch, then waits until every dispatched operation
+// (in-flight bulk, outstanding read, buffered success) has drained — at which
+// point modelState is the exact committed state the backup will capture. Returns
+// false if ctx ends first. Pair with resume.
+func (c *Checker) pauseAndDrain(ctx context.Context) bool {
+	c.mu.Lock()
+	if !c.paused {
+		c.paused = true
+		c.resumeCh = make(chan struct{})
+	}
+	c.mu.Unlock()
+
+	for {
+		c.mu.Lock()
+		_, empty := c.earliestOutstanding()
+		if empty {
+			// Nothing is in flight; flush the re-order buffer. handleObservation's
+			// failure/transient paths drop the ticket without calling tryDrain, so
+			// with no further op to re-trigger it a committed success can sit here.
+			c.tryDrain()
+		}
+		idle := empty && len(c.pending) == 0
+		c.mu.Unlock()
+		if idle {
+			return true
+		}
+
+		select {
+		case <-ctx.Done():
+			return false
+		case <-time.After(quiescePoll):
+		}
+	}
+}
+
+// resume releases workers parked in awaitResume.
+func (c *Checker) resume() {
+	c.mu.Lock()
+	if c.paused {
+		c.paused = false
+		close(c.resumeCh)
+	}
+	c.mu.Unlock()
+}
+
+// RestoreTrigger fires one backup+restore cycle and returns once the ledger is
+// serving again from restored state. Fire runs while the driver is quiesced.
+type RestoreTrigger interface {
+	Fire(ctx context.Context) error
+}
+
+// runRestoreCycle quiesces the driver, fires a restore, and resumes, on a jittered
+// interval. A cycle that errors (e.g. an injected fault interrupted the backup) is
+// logged and skipped: only wrong restored state — caught by the normal checks once
+// the driver resumes against the restored node — is a finding. resume always runs
+// so a stuck trigger cannot leave workers parked.
+func runRestoreCycle(ctx context.Context, c *Checker, trigger RestoreTrigger, interval time.Duration) {
+	for {
+		jitter := time.Duration(internal.Rand().Int63n(int64(interval/2) + 1))
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.After(interval + jitter):
+		}
+
+		log.Printf("restore cycle: quiescing")
+		if !c.pauseAndDrain(ctx) {
+			c.resume()
+			return
+		}
+
+		log.Printf("restore cycle: drained, firing restore")
+		err := func() error {
+			defer c.resume()
+			return trigger.Fire(ctx)
+		}()
+		if err != nil {
+			log.Printf("restore cycle: %v (continuing)", err)
+		} else {
+			log.Printf("restore cycle: complete, resumed")
+		}
+	}
+}
+
+// fileTrigger signals the local orchestrator (run_model_test.sh) through a
+// request/response file rendezvous: it writes a request file, then waits for the
+// orchestrator's response file. The orchestrator does the backup, restore, and
+// node relaunch. A file rendezvous — not a socket — because the local orchestrator
+// is bash, which polls files far more robustly than it speaks sockets.
+type fileTrigger struct {
+	reqPath  string
+	respPath string
+}
+
+func (t *fileTrigger) Fire(ctx context.Context) error {
+	_ = os.Remove(t.respPath)
+	if err := os.WriteFile(t.reqPath, []byte("restore\n"), 0o600); err != nil {
+		return fmt.Errorf("writing restore request: %w", err)
+	}
+
+	timeout := time.NewTimer(restoreTimeout)
+	defer timeout.Stop()
+
+	for {
+		if data, err := os.ReadFile(t.respPath); err == nil {
+			_ = os.Remove(t.respPath)
+			_ = os.Remove(t.reqPath)
+			if line := strings.TrimSpace(string(data)); line != "" && line != "ok" {
+				return fmt.Errorf("orchestrator reported: %s", line)
+			}
+			return nil
+		}
+
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-timeout.C:
+			return fmt.Errorf("restore timed out after %s", restoreTimeout)
+		case <-time.After(restorePoll):
+		}
+	}
+}
+
+// selectRestoreTrigger returns the configured trigger, or nil when the restore
+// cycle is not enabled for this run. Local runs set MODEL_RESTORE_REQ/RESP.
+func selectRestoreTrigger() RestoreTrigger {
+	if req, resp := os.Getenv("MODEL_RESTORE_REQ"), os.Getenv("MODEL_RESTORE_RESP"); req != "" && resp != "" {
+		return &fileTrigger{reqPath: req, respPath: resp}
+	}
+
+	return nil
+}
+
+// restoreInterval is the base cycle interval, from MODEL_RESTORE_INTERVAL seconds.
+func restoreInterval() time.Duration {
+	return time.Duration(envInt("MODEL_RESTORE_INTERVAL", defaultRestoreIntervalSecs)) * time.Second
+}

From 8f1ba6bddc8f4475b268d9754f030e8c0e9c306b Mon Sep 17 00:00:00 2001
From: Azorlogh <bott.alix@gmail.com>
Date: Fri, 10 Jul 2026 12:07:53 +0000
Subject: [PATCH 02/17] test(antithesis): run_model_test honors TMPDIR + a
 disk-guard knob

The --restore cycle bootstraps a full store per iteration, so on a host whose
filesystem is already near-full the server's WAL/data disk guard blocks writes
and the run fails with ResourceExhausted noise unrelated to the test.

- WORKDIR is placed under $TMPDIR (default /tmp), so disk-heavy runs can target
  a larger volume.
- HEALTH_THRESHOLD (default 0.8, matching the server) is passed to
  --health-wal-threshold / --health-data-threshold, so a run on a >80%-full host
  can raise the guard.

Both default to the prior behaviour.
---
 tests/antithesis/run_model_test.sh | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/tests/antithesis/run_model_test.sh b/tests/antithesis/run_model_test.sh
index 96f477704a..eb3873bcc9 100755
--- a/tests/antithesis/run_model_test.sh
+++ b/tests/antithesis/run_model_test.sh
@@ -88,6 +88,9 @@ RECOVER_TIMEOUT="${RECOVER_TIMEOUT:-90}"
 DEAD_TIME="${DEAD_TIME:-30}"
 COMPACTION_MARGIN="${COMPACTION_MARGIN:-200}"
 MAINTENANCE_INTERVAL="${MAINTENANCE_INTERVAL:-10s}"
+# WAL/data disk-usage guard (fraction). Default matches the server; raise it when
+# the host filesystem is already above 80% used (the guard blocks writes then).
+HEALTH_THRESHOLD="${HEALTH_THRESHOLD:-0.8}"
 CLUSTER_ID="model-test-cluster"
 # HMAC key for signed transaction receipts, so the driver can exercise the
 # receipt-carried revert path (admission verifies the receipt and reverses its
@@ -113,7 +116,7 @@ RAFT_BASE=$(( GRPC_BASE + 100 ))
 HTTP_BASE=$(( GRPC_BASE + 200 ))
 MINIO_PORT=$(( GRPC_BASE + 300 ))
 
-WORKDIR="$(mktemp -d /tmp/model-test.XXXXXX)"
+WORKDIR="$(mktemp -d "${TMPDIR:-/tmp}/model-test.XXXXXX")"
 DRIVER_LOG="$WORKDIR/driver.log"
 ASSERTIONS="$WORKDIR/assertions.json"
 SERVER_BIN="$WORKDIR/ledger-server"
@@ -259,6 +262,8 @@ start_node() {
 		--data-dir "${NODE_DIRS[$i]}/data" \
 		--raft-compaction-margin "$COMPACTION_MARGIN" \
 		--maintenance-interval "$MAINTENANCE_INTERVAL" \
+		--health-wal-threshold "$HEALTH_THRESHOLD" \
+		--health-data-threshold "$HEALTH_THRESHOLD" \
 		${flags[@]+"${flags[@]}"} \
 		>> "${SERVER_LOGS[$i]}" 2>&1 &
 	SERVER_PIDS[$i]=$!

From 84c83f93174a9039b552e49200d57e5b679618a1 Mon Sep 17 00:00:00 2001
From: Azorlogh <bott.alix@gmail.com>
Date: Wed, 15 Jul 2026 12:50:25 +0000
Subject: [PATCH 03/17] fix(antithesis): guarantee the restore run exercises a
 cycle
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Review feedback on #1593 (NumaryBot):

1. With --restore and no explicit duration, the single-node default
   (30s) ended the run before the first jittered cycle (fires within
   [interval, 1.5*interval], 45-67s by default) — a vacuous PASS. The
   restore default is now 2*interval+30, which covers the first fire
   plus the cycle for any interval, and the report FAILS outright when
   --restore completed zero cycles (fail-fast preempting the first
   cycle with a real finding stays a legitimate early stop). The cycle
   count is printed like the cluster restart count.

2. wait_leader grepped the whole server log, which is appended across
   boots, so the initial boot's leadership line satisfied the
   post-restore wait instantly — the driver could resume against a
   node that never came up. Relaunching callers now pass the log's
   byte offset and only lines after it count.

Also documents --restore and RESTORE_INTERVAL in the script header.

Validated: default-duration run armed at 120s, completed a cycle, and
reported it; a 20s run fails with NO RESTORE CYCLES.
---
 tests/antithesis/run_model_test.sh | 62 ++++++++++++++++++++++++------
 1 file changed, 51 insertions(+), 11 deletions(-)

diff --git a/tests/antithesis/run_model_test.sh b/tests/antithesis/run_model_test.sh
index eb3873bcc9..7dc6d17a85 100755
--- a/tests/antithesis/run_model_test.sh
+++ b/tests/antithesis/run_model_test.sh
@@ -21,7 +21,7 @@
 # the cluster failing to recover to N voters after a restart.
 #
 # Usage:
-#   ./run_model_test.sh [--nodes N | --cluster] [DURATION_SECONDS]
+#   ./run_model_test.sh [--nodes N | --cluster] [--restore] [DURATION_SECONDS]
 #   NODES=3 ./run_model_test.sh 300
 #
 # Environment:
@@ -30,6 +30,7 @@
 #   GRPC_BASE         base gRPC port; raft = +100, http = +200 per band (default: random)
 #   RESTART_INTERVAL  seconds to soak between restarts, N>1 only; 0 disables restarts (default: 15)
 #   RECOVER_TIMEOUT   seconds to wait for N-voter recovery after a restart, N>1 only (default: 90)
+#   RESTORE_INTERVAL  seconds between backup/restore cycles, --restore only (default: 45)
 #   DEAD_TIME             seconds a killed node stays down, N>1 only (default: 30)
 #   COMPACTION_MARGIN     raft log entries between snapshots; low forces snapshot recovery (default: 200)
 #   MAINTENANCE_INTERVAL  background WAL snapshot + checkpoint cadence (default: 10s)
@@ -51,7 +52,7 @@ while [ $# -gt 0 ]; do
 		--nodes)    NODES="$2"; shift 2 ;;
 		--nodes=*)  NODES="${1#*=}"; shift ;;
 		--restore)  RESTORE=1; shift ;;
-		-h|--help)  sed -n '2,40p' "$0"; exit 0 ;;
+		-h|--help)  sed -n '2,42p' "$0"; exit 0 ;;
 		*)          DURATION="$1"; shift ;;
 	esac
 done
@@ -72,9 +73,22 @@ case "$NODES" in
 esac
 [ "$NODES" -ge 1 ] || { echo "ERROR: --nodes must be >= 1" >&2; exit 2; }
 
-# Single-node iterates fast; a cluster needs longer to see restart cycles.
+# The driver fires the first restore within [interval, 1.5*interval] (jittered,
+# see runRestoreCycle), so a restore run's duration must cover at least one full
+# cycle or the run passes without exercising the restore path at all.
+RESTORE_INTERVAL="${RESTORE_INTERVAL:-45}"
+
+# Single-node iterates fast; a cluster needs longer to see restart cycles; a
+# restore run needs at least one backup/restore cycle (2*interval covers the
+# jittered first fire plus the cycle itself for any interval).
 if [ -z "$DURATION" ]; then
-	if [ "$NODES" -gt 1 ]; then DURATION=120; else DURATION=30; fi
+	if [ "$RESTORE" = 1 ]; then
+		DURATION=$(( RESTORE_INTERVAL * 2 + 30 ))
+	elif [ "$NODES" -gt 1 ]; then
+		DURATION=120
+	else
+		DURATION=30
+	fi
 fi
 
 RESTART_INTERVAL="${RESTART_INTERVAL:-15}"
@@ -125,9 +139,9 @@ LEDGERCTL_BIN="$WORKDIR/ledgerctl"
 DRIVER_PID=""
 RECOVERY_FAILED=0
 DRIVER_EXITED_EARLY=0
+RESTORE_CYCLES=0
 
 # --- Restore cycle (single-node, --restore) -------------------------------
-RESTORE_INTERVAL="${RESTORE_INTERVAL:-45}"
 RESTORE_REQ="$WORKDIR/restore.req"
 RESTORE_RESP="$WORKDIR/restore.resp"
 RESTORED_DATA="$WORKDIR/restored-data"
@@ -208,10 +222,16 @@ do_one_restore() {
 	fi
 
 	kill -9 "${SERVER_PIDS[0]}" 2>/dev/null; wait "${SERVER_PIDS[0]}" 2>/dev/null; SERVER_PIDS[0]=""
+
+	# The relaunched node appends to the same log file; only leadership lines
+	# past this offset count (see wait_leader).
+	local log_offset
+	log_offset=$(wc -c < "${SERVER_LOGS[0]}" 2>/dev/null || echo 0)
+
 	rm -rf "$RESTORED_DATA"
 	if ! "$LEDGERCTL_BIN" store bootstrap --data-dir "$RESTORED_DATA" "${S3_FLAGS[@]}" -y >>"$WORKDIR/restore.log" 2>&1; then
 		log "restore: bootstrap failed; relaunching on original data"
-		start_node 0 rejoin; wait_leader 0
+		start_node 0 rejoin; wait_leader 0 "$log_offset"
 		return 1
 	fi
 
@@ -220,7 +240,7 @@ do_one_restore() {
 	rm -rf "${NODE_DIRS[0]}/data" "${NODE_DIRS[0]}/wal"
 	mv "$RESTORED_DATA" "${NODE_DIRS[0]}/data"
 	start_node 0 bootstrap
-	wait_leader 0 || return 1
+	wait_leader 0 "$log_offset" || return 1
 	log "restore: node back up on restored (RebuildDelta) store"
 }
 
@@ -230,7 +250,12 @@ service_restore_request() {
 	[ "$RESTORE" = 1 ] && [ -f "$RESTORE_REQ" ] || return 0
 	rm -f "$RESTORE_REQ"
 	log "restore: cycle requested"
-	if do_one_restore; then printf 'ok\n' >"$RESTORE_RESP"; else printf 'err\n' >"$RESTORE_RESP"; fi
+	if do_one_restore; then
+		RESTORE_CYCLES=$(( RESTORE_CYCLES + 1 ))
+		printf 'ok\n' >"$RESTORE_RESP"
+	else
+		printf 'err\n' >"$RESTORE_RESP"
+	fi
 }
 
 if [ ! -d "$REPO" ]; then
@@ -269,11 +294,14 @@ start_node() {
 	SERVER_PIDS[$i]=$!
 }
 
-# Wait (bounded) for a node to log leadership; fail if it dies first.
+# Wait (bounded) for a node to log leadership; fail if it dies first. A caller
+# relaunching a node passes the log's current byte size: server logs are
+# appended across boots, so without the offset a leadership line from an
+# earlier boot satisfies the check instantly.
 wait_leader() {
-	local i="$1"
+	local i="$1" offset="${2:-0}"
 	for _ in $(seq 1 60); do
-		if grep -qE "Became leader|became leader at term" "${SERVER_LOGS[$i]}" 2>/dev/null; then return 0; fi
+		if tail -c +"$(( offset + 1 ))" "${SERVER_LOGS[$i]}" 2>/dev/null | grep -qE "Became leader|became leader at term"; then return 0; fi
 		if ! kill -0 "${SERVER_PIDS[$i]}" 2>/dev/null; then
 			echo "ERROR: node $(( i + 1 )) exited during startup:" >&2
 			tail -20 "${SERVER_LOGS[$i]}" >&2
@@ -490,6 +518,7 @@ echo
 echo "================= model test report ================="
 log "topology: $NODES node(s)"
 [ "$NODES" -gt 1 ] && log "restart cycles completed: $cycle"
+[ "$RESTORE" = 1 ] && log "restore cycles completed: $RESTORE_CYCLES"
 
 findings=0
 
@@ -562,6 +591,17 @@ if [ "$DRIVER_EXITED_EARLY" -ne 0 ] && [ "$findings" -eq 0 ]; then
 	findings=$((findings + 1))
 fi
 
+# 6. --restore ran zero cycles: the restore path was never exercised, so a
+# green run says nothing about RebuildDelta -- that vacuous pass is itself a
+# failure. Skipped when a finding already stopped the run early (fail-fast
+# legitimately preempts the first cycle).
+if [ "$RESTORE" = 1 ] && [ "$RESTORE_CYCLES" -eq 0 ] && [ "$findings" -eq 0 ]; then
+	echo
+	echo "NO RESTORE CYCLES: --restore was requested but no backup/restore cycle completed"
+	echo "  (duration ${DURATION}s vs first cycle within ~$(( RESTORE_INTERVAL * 3 / 2 ))s; increase the duration)"
+	findings=$((findings + 1))
+fi
+
 echo "-----------------------------------------------------"
 if [ "$findings" -eq 0 ]; then
 	echo "RESULT: PASS (no findings)"

From 1b0bed5ab97e88ab324f234149520f942dc87140 Mon Sep 17 00:00:00 2001
From: Azorlogh <bott.alix@gmail.com>
Date: Wed, 15 Jul 2026 13:29:23 +0000
Subject: [PATCH 04/17] fix(antithesis): fail the restore run on any failed
 cycle
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Review feedback on #1593 (NumaryBot, third finding): a restore cycle
failing after a prior success (backup RPC, no exports, bootstrap, or
the post-restore leader wait) was logged and the run continued — the
report only failed on zero completed cycles, so a mid-run bootstrap
regression could still report PASS. Failed cycles are now counted,
shown next to the completed count, and any failure fails the report.

Validated by killing MinIO mid-run: cycle 3's incremental backup
failed with connection refused, the report showed "2 (1 failed)" and
RESTORE CYCLE FAILURES, and the driver resumed after the err response.
---
 tests/antithesis/run_model_test.sh | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/tests/antithesis/run_model_test.sh b/tests/antithesis/run_model_test.sh
index 7dc6d17a85..094c8cb338 100755
--- a/tests/antithesis/run_model_test.sh
+++ b/tests/antithesis/run_model_test.sh
@@ -140,6 +140,7 @@ DRIVER_PID=""
 RECOVERY_FAILED=0
 DRIVER_EXITED_EARLY=0
 RESTORE_CYCLES=0
+RESTORE_FAILED_CYCLES=0
 
 # --- Restore cycle (single-node, --restore) -------------------------------
 RESTORE_REQ="$WORKDIR/restore.req"
@@ -254,6 +255,7 @@ service_restore_request() {
 		RESTORE_CYCLES=$(( RESTORE_CYCLES + 1 ))
 		printf 'ok\n' >"$RESTORE_RESP"
 	else
+		RESTORE_FAILED_CYCLES=$(( RESTORE_FAILED_CYCLES + 1 ))
 		printf 'err\n' >"$RESTORE_RESP"
 	fi
 }
@@ -518,7 +520,7 @@ echo
 echo "================= model test report ================="
 log "topology: $NODES node(s)"
 [ "$NODES" -gt 1 ] && log "restart cycles completed: $cycle"
-[ "$RESTORE" = 1 ] && log "restore cycles completed: $RESTORE_CYCLES"
+[ "$RESTORE" = 1 ] && log "restore cycles completed: $RESTORE_CYCLES ($RESTORE_FAILED_CYCLES failed)"
 
 findings=0
 
@@ -602,6 +604,15 @@ if [ "$RESTORE" = 1 ] && [ "$RESTORE_CYCLES" -eq 0 ] && [ "$findings" -eq 0 ]; t
 	findings=$((findings + 1))
 fi
 
+# 7. A restore cycle failed mid-run (backup RPC, no exports, bootstrap, or the
+# post-restore leader wait). The driver resumes so later cycles still run, but
+# a broken restore path is a defect regardless of how many cycles succeeded.
+if [ "$RESTORE" = 1 ] && [ "$RESTORE_FAILED_CYCLES" -gt 0 ]; then
+	echo
+	echo "RESTORE CYCLE FAILURES: $RESTORE_FAILED_CYCLES cycle(s) failed (see 'restore:' lines above)"
+	findings=$((findings + 1))
+fi
+
 echo "-----------------------------------------------------"
 if [ "$findings" -eq 0 ]; then
 	echo "RESULT: PASS (no findings)"

From eeee813e238b218e90c4ec9e46af3e575ba4d853 Mon Sep 17 00:00:00 2001
From: Azorlogh <bott.alix@gmail.com>
Date: Wed, 15 Jul 2026 14:59:50 +0000
Subject: [PATCH 05/17] fix(antithesis): ship tests/oracle in the workload
 image build
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The workload module imports the oracle library from the main module
since the tests/oracle extraction, but the image build copies only
go.*, internal/, and pkg/ into the partial repo the replace directive
resolves to — so the instrumentor's go mod tidy fails to find
tests/oracle (and oracletest). Copy tests/oracle into both replacement
roots (/src for the pre-instrumentation module, / for the instrumented
copy), like internal/ and pkg/.
---
 tests/antithesis/workload/Dockerfile | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/antithesis/workload/Dockerfile b/tests/antithesis/workload/Dockerfile
index 92e3cc778f..60c2c12f1f 100644
--- a/tests/antithesis/workload/Dockerfile
+++ b/tests/antithesis/workload/Dockerfile
@@ -10,13 +10,15 @@ RUN go install github.com/antithesishq/antithesis-sdk-go/tools/antithesis-go-ins
 COPY go.* /src/
 COPY internal /src/internal
 COPY pkg /src/pkg
+COPY tests/oracle /src/tests/oracle
 COPY tests/antithesis/workload/go.* /src/tests/antithesis/workload/
 COPY tests/antithesis/workload/bin /src/tests/antithesis/workload/bin
 COPY tests/antithesis/workload/internal /src/tests/antithesis/workload/internal
 
 # The instrumentor outputs to /instrumented/customer/ and runs go mod tidy there.
 # The replace directive (../../../) resolves to / from that path, so place the root module at /.
-RUN cp /src/go.* / && cp -r /src/internal /internal && cp -r /src/pkg /pkg
+RUN cp /src/go.* / && cp -r /src/internal /internal && cp -r /src/pkg /pkg \
+    && mkdir -p /tests && cp -r /src/tests/oracle /tests/oracle
 
 RUN mkdir /instrumented
 WORKDIR /src/tests/antithesis/workload

From 9cb441051838fc0f0e1777445c45ab2b341c52ea Mon Sep 17 00:00:00 2001
From: Azorlogh <bott.alix@gmail.com>
Date: Wed, 15 Jul 2026 15:00:14 +0000
Subject: [PATCH 06/17] feat(antithesis): drive the restore cycle through the
 operator on k8s
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The model driver's restore rendezvous now works in the Antithesis k8s
template: a restore-orchestrator sidecar in the workload pod services
each request with a full disaster-recovery pass through the operator —
quiesce-point incremental backup (ledgerctl exec'd in a ledger pod,
retried across pods), Cluster CR + PVC teardown, restore-mode
re-create at one replica, restore download/finalize, then the flip
back to the full cluster (pod 0 self-bootstraps from the restored
store, the others join fresh and snapshot-install it). The sidecar
snapshots the live Cluster spec at startup and derives both mode
variants from it; on any failure it re-applies the normal spec before
answering err, so the driver never resumes against a cluster parked in
restore mode. An initial full backup at startup makes every cycle's
delta force RebuildDelta.

Driver: MODEL_RESTORE_TIMEOUT makes the per-cycle lease tunable (the
k8s cycle takes minutes under fault injection; the local default is
unchanged), and a Sometimes assertion — "restore cycle completed" —
is the report-visible proof the path ran, mirroring the local runner's
zero-cycles guard.

The workload image gains jq, a pinned kubectl, and the orchestrator
script (outside the test-command tree so the Composer never runs it);
the workload RBAC gains cluster create/delete, pods/exec, and PVC
deletion.
---
 docs/technical/contributing/testing.md        |  20 +++
 tests/antithesis/k8s/workload.yaml            |  60 ++++++-
 tests/antithesis/workload/Dockerfile          |  13 +-
 .../model/singleton_driver_model/config.go    |  14 +-
 .../model/singleton_driver_model/restore.go   |  21 ++-
 .../workload/restore-orchestrator.sh          | 167 ++++++++++++++++++
 6 files changed, 283 insertions(+), 12 deletions(-)
 create mode 100755 tests/antithesis/workload/restore-orchestrator.sh

diff --git a/docs/technical/contributing/testing.md b/docs/technical/contributing/testing.md
index 99837d02f6..cf8266c1e1 100644
--- a/docs/technical/contributing/testing.md
+++ b/docs/technical/contributing/testing.md
@@ -384,6 +384,26 @@ default. Common tunables (full list in the script header):
 | `MODEL_DUMP_BATCHES` | Log every submitted bulk (`[batch-dump]` lines) for deterministic offline replay through `tests/oracle/cmd/replay`. |
 | `RESTART_INTERVAL` / `DEAD_TIME` | Cluster restart cadence and how long a killed node stays down. |
 | `COMPACTION_MARGIN` | Raft entries between snapshots; low values force snapshot recovery. |
+| `RESTORE_INTERVAL` | Seconds between backup/restore cycles with `--restore`. |
+
+#### Restore cycles (`--restore`, single node)
+
+`run_model_test.sh --restore` periodically quiesces the driver, takes an
+incremental backup, kills the node, bootstraps a fresh store from the backup
+(running the `RebuildDelta` replay), and relaunches on it — the driver then
+resumes and its ordinary checks validate the rebuilt state. The run fails if
+no cycle completed or any cycle failed. The driver-side knobs are
+`MODEL_RESTORE_INTERVAL` and `MODEL_RESTORE_TIMEOUT` (seconds; the timeout is
+the per-cycle lease after which the driver gives up waiting and resumes).
+
+On Antithesis (k8s template) the same driver rendezvous is serviced by the
+`restore-orchestrator` sidecar in the workload pod
+(`tests/antithesis/workload/restore-orchestrator.sh`), which drives the
+operator through a full disaster-recovery pass: quiesce-point backup, Cluster
+teardown (CR + PVCs), restore-mode round-trip (`ledgerctl restore download` /
+`finalize`), and the flip back to a full cluster. The
+`singleton_driver_model: restore cycle completed` Sometimes assertion in the
+report is the proof the path actually ran.
 
 #### Maintaining the model
 
diff --git a/tests/antithesis/k8s/workload.yaml b/tests/antithesis/k8s/workload.yaml
index 7cc95980ac..14ae55298e 100644
--- a/tests/antithesis/k8s/workload.yaml
+++ b/tests/antithesis/k8s/workload.yaml
@@ -11,14 +11,26 @@ kind: ClusterRole
 metadata:
   name: workload-ledger-updater
 rules:
+# The restore-orchestrator sidecar deletes and re-creates the Cluster (restore
+# mode round-trip), on top of the drivers' rolling updates.
 - apiGroups: ["ledger.formance.com"]
   resources: ["clusters"]
-  verbs: ["get", "list", "watch", "update", "patch"]
+  verbs: ["get", "list", "watch", "update", "patch", "create", "delete"]
 # Operational drivers (rolling_restart, quorum_recovery) delete pods and
 # observe the StatefulSet they belong to.
 - apiGroups: [""]
   resources: ["pods"]
   verbs: ["get", "list", "watch", "delete"]
+# The restore-orchestrator sidecar execs ledgerctl inside ledger pods
+# (backup at the quiesce point, restore download/finalize).
+- apiGroups: [""]
+  resources: ["pods/exec"]
+  verbs: ["create"]
+# The restore teardown deletes the cluster's volume claims so the re-created
+# cluster starts from the restored store, not stale disks.
+- apiGroups: [""]
+  resources: ["persistentvolumeclaims"]
+  verbs: ["get", "list", "delete"]
 - apiGroups: ["apps"]
   resources: ["statefulsets"]
   verbs: ["get", "list", "watch"]
@@ -93,6 +105,21 @@ spec:
         # MODEL_DUMP_BATCHES env at config-image build.
         - name: MODEL_DUMP_BATCHES
           value: "__MODEL_DUMP_BATCHES__"
+        # Restore-cycle rendezvous with the restore-orchestrator sidecar (only
+        # the model driver writes requests; other templates leave it idle).
+        # The interval and lease are k8s-sized: one cycle is a full cluster
+        # teardown + restore + rejoin, minutes under fault injection.
+        - name: MODEL_RESTORE_REQ
+          value: /var/run/restore/req
+        - name: MODEL_RESTORE_RESP
+          value: /var/run/restore/resp
+        - name: MODEL_RESTORE_INTERVAL
+          value: "180"
+        - name: MODEL_RESTORE_TIMEOUT
+          value: "900"
+        volumeMounts:
+        - name: restore-rendezvous
+          mountPath: /var/run/restore
         # Readiness probe ensures Antithesis waits for the ledger cluster
         # to be reachable before starting the Test Composer.
         readinessProbe:
@@ -106,3 +133,34 @@ spec:
             memory: 128Mi
           limits:
             memory: 512Mi
+      # Services the model driver's restore requests by driving the operator:
+      # quiesce-point backup, cluster teardown, restore-mode round-trip, and
+      # the flip back to normal. See restore-orchestrator.sh.
+      - name: restore-orchestrator
+        image: __REGISTRY__/workload-ledger-v3:__TAG__
+        command: ["/opt/antithesis/restore-orchestrator.sh"]
+        env:
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: CLUSTER_NAME
+          value: ledger
+        - name: CLUSTER_REPLICAS
+          value: "3"
+        - name: MODEL_RESTORE_REQ
+          value: /var/run/restore/req
+        - name: MODEL_RESTORE_RESP
+          value: /var/run/restore/resp
+        volumeMounts:
+        - name: restore-rendezvous
+          mountPath: /var/run/restore
+        resources:
+          requests:
+            cpu: 10m
+            memory: 64Mi
+          limits:
+            memory: 128Mi
+      volumes:
+      - name: restore-rendezvous
+        emptyDir: {}
diff --git a/tests/antithesis/workload/Dockerfile b/tests/antithesis/workload/Dockerfile
index 60c2c12f1f..13619f4246 100644
--- a/tests/antithesis/workload/Dockerfile
+++ b/tests/antithesis/workload/Dockerfile
@@ -61,10 +61,21 @@ RUN set -e; \
 # Runner
 FROM debian:trixie
 
-RUN apt-get update && apt-get install -y curl
+RUN apt-get update && apt-get install -y curl jq
+
+# kubectl, for the restore-orchestrator sidecar (drives the operator's
+# backup/restore flow from inside the workload pod).
+ARG TARGETARCH
+RUN curl -fsSLo /usr/local/bin/kubectl \
+        "https://dl.k8s.io/release/v1.31.4/bin/linux/${TARGETARCH}/kubectl" \
+    && chmod +x /usr/local/bin/kubectl
 
 COPY --from=ghcr.io/grpc-ecosystem/grpc-health-probe:v0.4.44 /ko-app/grpc-health-probe /bin/grpc_health_probe
 
+# Restore-orchestrator sidecar entrypoint — outside /opt/antithesis/test/v1 so
+# the Test Composer never discovers it as a test command.
+COPY tests/antithesis/workload/restore-orchestrator.sh /opt/antithesis/restore-orchestrator.sh
+
 COPY --from=compiler /init /init
 ENTRYPOINT ["/init"]
 
diff --git a/tests/antithesis/workload/bin/cmds/model/singleton_driver_model/config.go b/tests/antithesis/workload/bin/cmds/model/singleton_driver_model/config.go
index dea3c66c1e..b18599b229 100644
--- a/tests/antithesis/workload/bin/cmds/model/singleton_driver_model/config.go
+++ b/tests/antithesis/workload/bin/cmds/model/singleton_driver_model/config.go
@@ -131,12 +131,14 @@ const incomingBuffer = 256
 // Poll cadence while waiting for the driver to fully drain before a backup.
 const quiescePoll = 25 * time.Millisecond
 
-// Poll cadence and hard cap while waiting for the external orchestrator to finish
-// a restore. The cap is a lease: a dead orchestrator cannot park workers forever.
-const (
-	restorePoll    = 200 * time.Millisecond
-	restoreTimeout = 3 * time.Minute
-)
+// Poll cadence while waiting for the external orchestrator to finish a restore.
+const restorePoll = 200 * time.Millisecond
+
+// Default hard cap (seconds) on one restore cycle when MODEL_RESTORE_TIMEOUT is
+// unset. The cap is a lease: a dead orchestrator cannot park workers forever.
+// A k8s restore (cluster teardown, download, rejoin — under fault injection)
+// takes far longer than the local single-node swap, so the environment sets it.
+const defaultRestoreTimeoutSecs = 180
 
 // Base restore-cycle interval (seconds) when MODEL_RESTORE_INTERVAL is unset.
 const defaultRestoreIntervalSecs = 90
diff --git a/tests/antithesis/workload/bin/cmds/model/singleton_driver_model/restore.go b/tests/antithesis/workload/bin/cmds/model/singleton_driver_model/restore.go
index 077ac97dd8..d2e35af077 100644
--- a/tests/antithesis/workload/bin/cmds/model/singleton_driver_model/restore.go
+++ b/tests/antithesis/workload/bin/cmds/model/singleton_driver_model/restore.go
@@ -8,6 +8,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/antithesishq/antithesis-sdk-go/assert"
+
 	"github.com/formancehq/ledger/v3/tests/antithesis/workload/internal"
 )
 
@@ -16,8 +18,9 @@ import (
 // the RebuildDelta replay), and brings a node back up on it. The driver keeps
 // running against the restored node afterward, so its ordinary read/commit
 // checks validate the rebuilt state — no separate comparison. The driver owns
-// only the timing and the quiescence; the environment-specific work lives behind
-// RestoreTrigger (a file rendezvous locally, the k8s client on Antithesis).
+// only the timing and the quiescence; the environment-specific work lives
+// behind the file rendezvous — run_model_test.sh locally, the
+// restore-orchestrator sidecar (driving the operator) on Antithesis k8s.
 
 // awaitResume blocks while a restore cycle has paused dispatch, returning false
 // if ctx ends. Safe to call without holding mu.
@@ -118,6 +121,10 @@ func runRestoreCycle(ctx context.Context, c *Checker, trigger RestoreTrigger, in
 		if err != nil {
 			log.Printf("restore cycle: %v (continuing)", err)
 		} else {
+			// The report-visible proof that restore coverage actually ran: a
+			// green run with this never hit exercised nothing (the local
+			// runner's zero-cycles guard, for Antithesis).
+			assert.Sometimes(true, "singleton_driver_model: restore cycle completed", internal.Details{})
 			log.Printf("restore cycle: complete, resumed")
 		}
 	}
@@ -139,7 +146,8 @@ func (t *fileTrigger) Fire(ctx context.Context) error {
 		return fmt.Errorf("writing restore request: %w", err)
 	}
 
-	timeout := time.NewTimer(restoreTimeout)
+	cap := restoreTimeout()
+	timeout := time.NewTimer(cap)
 	defer timeout.Stop()
 
 	for {
@@ -156,12 +164,17 @@ func (t *fileTrigger) Fire(ctx context.Context) error {
 		case <-ctx.Done():
 			return ctx.Err()
 		case <-timeout.C:
-			return fmt.Errorf("restore timed out after %s", restoreTimeout)
+			return fmt.Errorf("restore timed out after %s", cap)
 		case <-time.After(restorePoll):
 		}
 	}
 }
 
+// restoreTimeout is the per-cycle lease, from MODEL_RESTORE_TIMEOUT seconds.
+func restoreTimeout() time.Duration {
+	return time.Duration(envInt("MODEL_RESTORE_TIMEOUT", defaultRestoreTimeoutSecs)) * time.Second
+}
+
 // selectRestoreTrigger returns the configured trigger, or nil when the restore
 // cycle is not enabled for this run. Local runs set MODEL_RESTORE_REQ/RESP.
 func selectRestoreTrigger() RestoreTrigger {
diff --git a/tests/antithesis/workload/restore-orchestrator.sh b/tests/antithesis/workload/restore-orchestrator.sh
new file mode 100755
index 0000000000..2e26e82347
--- /dev/null
+++ b/tests/antithesis/workload/restore-orchestrator.sh
@@ -0,0 +1,167 @@
+#!/usr/bin/env bash
+#
+# restore-orchestrator.sh -- services the model driver's restore rendezvous in
+# the Antithesis k8s environment, as a sidecar in the workload pod.
+#
+# The driver (singleton_driver_model) quiesces itself, writes MODEL_RESTORE_REQ
+# on the shared volume, and waits for MODEL_RESTORE_RESP. One cycle here is a
+# full disaster-recovery pass through the operator:
+#
+#   1. incremental backup at the quiesce point (exec ledgerctl in a ledger pod)
+#   2. tear the cluster down: delete the Cluster CR and every PVC
+#   3. re-create the Cluster in restore mode (1 replica), exec
+#      `ledgerctl restore download` + `restore finalize` (RebuildDelta runs
+#      inside bootstrap-from-backup on the restored store)
+#   4. flip the Cluster back to normal mode: pod 0 self-bootstraps from the
+#      restored store (RESTORED marker), the other replicas join fresh and
+#      snapshot-install the restored state
+#   5. answer ok/err; the driver resumes and its ordinary model checks validate
+#      the rebuilt state
+#
+# The declarative parts (pod restarts, mode switches, membership) converge via
+# the operator no matter what faults hit the ledger pods; the imperative execs
+# are retried here. This sidecar itself is exempt from fault injection. On any
+# failure the normal-mode Cluster spec is re-applied so the driver never
+# resumes against a cluster stuck in restore mode.
+#
+# An initial full backup is taken once at startup (mid-traffic is fine: the
+# restore replays the per-cycle incremental delta, cut at the quiesce point, on
+# top of whatever checkpoint the full backup carried).
+set -uo pipefail
+
+NS="${NAMESPACE:-default}"
+CLUSTER="${CLUSTER_NAME:-ledger}"
+REPLICAS="${CLUSTER_REPLICAS:-3}"
+REQ="${MODEL_RESTORE_REQ:?MODEL_RESTORE_REQ must be set}"
+RESP="${MODEL_RESTORE_RESP:?MODEL_RESTORE_RESP must be set}"
+
+STS="ledger-$CLUSTER"
+POD_PREFIX="ledger-$CLUSTER"
+NORMAL_SPEC=/tmp/cluster-normal.json
+RESTORE_SPEC=/tmp/cluster-restore.json
+
+S3_ARGS=(
+	--s3-bucket backups --s3-endpoint "http://minio.$NS.svc:9000"
+	--s3-region us-east-1 --s3-access-key-id minioadmin --s3-secret-access-key minioadmin
+)
+BACKUP_ARGS=(--driver s3 "${S3_ARGS[@]}")
+
+log() { echo "[restore-orchestrator] $*"; }
+
+k() { kubectl -n "$NS" "$@"; }
+
+# exec_ledgerctl POD ARGS... -- runs ledgerctl inside a ledger pod with the
+# pod-local server address and cluster credentials (the operator-e2e idiom).
+exec_ledgerctl() {
+	local pod="$1"; shift
+	k exec "$pod" -c ledger -- /bin/sh -c \
+		"./ledgerctl $* --server 127.0.0.1:8888 --insecure --auth-token \"\$CLUSTER_SECRET\" --timeout 120s"
+}
+
+# retry N CMD... -- runs CMD up to N times, 5s apart.
+retry() {
+	local n="$1" i=1; shift
+	while true; do
+		"$@" && return 0
+		[ "$i" -ge "$n" ] && return 1
+		i=$(( i + 1 ))
+		sleep 5
+	done
+}
+
+# backup_exec KIND -- runs a backup command against any live ledger pod (the
+# RPC needs a reachable server; retrying across pods covers a dead one).
+backup_exec() {
+	local kind="$1" i
+	for i in $(seq 0 $(( REPLICAS - 1 ))); do
+		if exec_ledgerctl "$POD_PREFIX-$i" "$kind" "${BACKUP_ARGS[*]}"; then return 0; fi
+		log "$kind via $POD_PREFIX-$i failed; trying next pod"
+	done
+	return 1
+}
+
+wait_ready_replicas() {
+	local want="$1" deadline=$(( $(date +%s) + ${2:-600} ))
+	while [ "$(date +%s)" -lt "$deadline" ]; do
+		[ "$(k get statefulset "$STS" -o jsonpath='{.status.readyReplicas}' 2>/dev/null)" = "$want" ] && return 0
+		sleep 5
+	done
+	return 1
+}
+
+wait_pod_running() {
+	local pod="$1" deadline=$(( $(date +%s) + ${2:-300} ))
+	while [ "$(date +%s)" -lt "$deadline" ]; do
+		[ "$(k get pod "$pod" -o jsonpath='{.status.phase}' 2>/dev/null)" = "Running" ] && return 0
+		sleep 5
+	done
+	return 1
+}
+
+# capture_specs -- snapshots the live Cluster CR as the normal-mode spec and
+# derives the restore-mode variant. Runs once; the CR is deleted and re-created
+# from these on every cycle.
+capture_specs() {
+	k get cluster "$CLUSTER" -o json \
+		| jq 'del(.status, .metadata.resourceVersion, .metadata.uid,
+		          .metadata.generation, .metadata.creationTimestamp,
+		          .metadata.managedFields)' > "$NORMAL_SPEC" || return 1
+	jq '.spec.restore = true | .spec.replicas = 1' "$NORMAL_SPEC" > "$RESTORE_SPEC"
+}
+
+teardown_cluster() {
+	k delete cluster "$CLUSTER" --wait=true --timeout=120s || return 1
+	local i
+	for i in $(seq 0 $(( REPLICAS - 1 ))); do
+		k wait --for=delete "pod/$POD_PREFIX-$i" --timeout=120s 2>/dev/null || true
+	done
+	for i in $(seq 0 $(( REPLICAS - 1 ))); do
+		k delete pvc "wal-$POD_PREFIX-$i" "data-$POD_PREFIX-$i" "cold-cache-$POD_PREFIX-$i" \
+			--ignore-not-found --wait=true --timeout=120s || return 1
+	done
+}
+
+do_one_restore() {
+	retry 3 backup_exec "store incremental-backup" || { log "incremental backup failed"; return 1; }
+
+	teardown_cluster || { log "teardown failed"; return 1; }
+
+	k apply -f "$RESTORE_SPEC" || { log "applying restore-mode cluster failed"; return 1; }
+	wait_pod_running "$POD_PREFIX-0" || { log "restore-mode pod never ran"; return 1; }
+
+	retry 5 exec_ledgerctl "$POD_PREFIX-0" "restore download" "${S3_ARGS[*]}" \
+		|| { log "restore download failed"; return 1; }
+	retry 5 exec_ledgerctl "$POD_PREFIX-0" "restore finalize --yes" \
+		|| { log "restore finalize failed"; return 1; }
+
+	k apply -f "$NORMAL_SPEC" || { log "applying normal-mode cluster failed"; return 1; }
+	# The restore-mode pod does not roll on its own after the spec flip
+	# (operator e2e does the same bounce).
+	sleep 5
+	k delete pod "$POD_PREFIX-0" --force --grace-period=0 2>/dev/null || true
+	wait_ready_replicas "$REPLICAS" || { log "cluster did not return to $REPLICAS ready replicas"; return 1; }
+	log "cluster back up on the restored (RebuildDelta) store"
+}
+
+log "waiting for $STS to reach $REPLICAS ready replicas"
+wait_ready_replicas "$REPLICAS" 1800 || log "WARNING: cluster not ready at startup; continuing"
+
+retry 10 capture_specs || { log "FATAL: cannot capture cluster spec"; exit 1; }
+
+log "taking initial full backup"
+retry 5 backup_exec "store backup" || log "WARNING: initial full backup failed; first cycle will retry"
+
+log "armed; watching $REQ"
+while true; do
+	if [ ! -f "$REQ" ]; then sleep 1; continue; fi
+	rm -f "$REQ" "$RESP"
+	log "restore cycle requested"
+	if do_one_restore; then
+		printf 'ok\n' > "$RESP"
+	else
+		# Converge back to normal mode so the driver never resumes against a
+		# cluster parked in restore mode; the driver logs the err and continues.
+		k apply -f "$NORMAL_SPEC" 2>/dev/null || true
+		printf 'err\n' > "$RESP"
+	fi
+done

From 55b400c0da3b14566371384c3b7771f7a3f0ee05 Mon Sep 17 00:00:00 2001
From: Azorlogh <bott.alix@gmail.com>
Date: Wed, 15 Jul 2026 20:35:05 +0000
Subject: [PATCH 07/17] fix(antithesis): pull kubectl as an image layer in the
 workload build

The RUN-time curl to dl.k8s.io resolves through the build stage's DNS,
which fails intermittently under qemu emulation. COPY --from an OCI
image goes through the docker host's pulls instead, like the
grpc-health-probe layer above it.
---
 tests/antithesis/workload/Dockerfile | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/tests/antithesis/workload/Dockerfile b/tests/antithesis/workload/Dockerfile
index 13619f4246..1be2482a73 100644
--- a/tests/antithesis/workload/Dockerfile
+++ b/tests/antithesis/workload/Dockerfile
@@ -63,14 +63,11 @@ FROM debian:trixie
 
 RUN apt-get update && apt-get install -y curl jq
 
+COPY --from=ghcr.io/grpc-ecosystem/grpc-health-probe:v0.4.44 /ko-app/grpc-health-probe /bin/grpc_health_probe
+
 # kubectl, for the restore-orchestrator sidecar (drives the operator's
 # backup/restore flow from inside the workload pod).
-ARG TARGETARCH
-RUN curl -fsSLo /usr/local/bin/kubectl \
-        "https://dl.k8s.io/release/v1.31.4/bin/linux/${TARGETARCH}/kubectl" \
-    && chmod +x /usr/local/bin/kubectl
-
-COPY --from=ghcr.io/grpc-ecosystem/grpc-health-probe:v0.4.44 /ko-app/grpc-health-probe /bin/grpc_health_probe
+COPY --from=bitnami/kubectl:1.31.4 /opt/bitnami/kubectl/bin/kubectl /usr/local/bin/kubectl
 
 # Restore-orchestrator sidecar entrypoint — outside /opt/antithesis/test/v1 so
 # the Test Composer never discovers it as a test command.

From 30b7f4a7dd41867505793464d86e516c481f10df Mon Sep 17 00:00:00 2001
From: Azorlogh <bott.alix@gmail.com>
Date: Wed, 15 Jul 2026 20:36:38 +0000
Subject: [PATCH 08/17] fix(antithesis): source kubectl from registry.k8s.io

The bitnami/kubectl public tags no longer exist on Docker Hub; the official registry.k8s.io/kubectl image carries the binary at /bin/kubectl.
---
 tests/antithesis/workload/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/antithesis/workload/Dockerfile b/tests/antithesis/workload/Dockerfile
index 1be2482a73..d19906285c 100644
--- a/tests/antithesis/workload/Dockerfile
+++ b/tests/antithesis/workload/Dockerfile
@@ -67,7 +67,7 @@ COPY --from=ghcr.io/grpc-ecosystem/grpc-health-probe:v0.4.44 /ko-app/grpc-health
 
 # kubectl, for the restore-orchestrator sidecar (drives the operator's
 # backup/restore flow from inside the workload pod).
-COPY --from=bitnami/kubectl:1.31.4 /opt/bitnami/kubectl/bin/kubectl /usr/local/bin/kubectl
+COPY --from=registry.k8s.io/kubectl:v1.31.4 /bin/kubectl /usr/local/bin/kubectl
 
 # Restore-orchestrator sidecar entrypoint — outside /opt/antithesis/test/v1 so
 # the Test Composer never discovers it as a test command.

From 062a135433ce0f16fefc0ca1840e9455b1de8a3f Mon Sep 17 00:00:00 2001
From: Azorlogh <bott.alix@gmail.com>
Date: Wed, 15 Jul 2026 21:36:53 +0000
Subject: [PATCH 09/17] fix(antithesis): preload the ledger image into the k8s
 environment

The environment builder mirrors images found in pod templates; the
ledger image only appears in the Cluster CR's spec.image fields, so the
hermetic environment lacked it and every ledger pod failed to pull
(no such host). A no-op preload Job carries the reference in a pod
template. The legacy Justfile solved this with the antithesis.images
webhook param, which snouty rejects by design.
---
 tests/antithesis/k8s/cluster.yaml | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/tests/antithesis/k8s/cluster.yaml b/tests/antithesis/k8s/cluster.yaml
index de32f06a03..4ce2091127 100644
--- a/tests/antithesis/k8s/cluster.yaml
+++ b/tests/antithesis/k8s/cluster.yaml
@@ -76,3 +76,25 @@ spec:
     value: minioadmin
   - name: AWS_SECRET_ACCESS_KEY
     value: minioadmin
+---
+# The ledger image is referenced only by the Cluster CR's spec.image fields,
+# which the Antithesis environment builder does not scan — it mirrors images
+# from pod templates. This no-op Job carries the image in a pod template so it
+# is baked into the hermetic environment for the operator's pods to use.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: ledger-image-preload
+spec:
+  backoffLimit: 0
+  template:
+    spec:
+      restartPolicy: Never
+      containers:
+      - name: preload
+        image: __REGISTRY__/__IMAGE_NAME__:__TAG__
+        command: ["/bin/true"]
+        resources:
+          requests:
+            cpu: 1m
+            memory: 8Mi

From 5ba005144a22c0518d8f27a16fdd8e1b6c7c9570 Mon Sep 17 00:00:00 2001
From: Azorlogh <bott.alix@gmail.com>
Date: Wed, 15 Jul 2026 22:16:17 +0000
Subject: [PATCH 10/17] fix(antithesis): harden the restore orchestrator's exec
 and error paths
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Review feedback on #1593 (NumaryBot):

1. ledgerctl restore download does not define the --timeout flag the
   shared exec wrapper appended, so every download invocation failed on
   flag parsing. The wrapper now bounds every exec with coreutils
   timeout instead — uniform across subcommands and also covering a
   hung exec stream.

2. A failed initial full backup was never retaken (cycles only run
   incrementals, which need the full base), leaving every cycle broken
   with a log line claiming otherwise. The orchestrator now tracks
   whether the full backup succeeded and retakes it at the start of the
   next cycle — the driver is quiesced there, so it is as valid a base
   as one taken at startup.

3. The failed-cycle path applied the normal-mode spec and released the
   driver immediately; it now also waits for the cluster to reach full
   readiness first, so the driver never resumes against a cluster still
   forming.
---
 .../workload/restore-orchestrator.sh          | 28 ++++++++++++++++---
 1 file changed, 24 insertions(+), 4 deletions(-)

diff --git a/tests/antithesis/workload/restore-orchestrator.sh b/tests/antithesis/workload/restore-orchestrator.sh
index 2e26e82347..91aebe9bee 100755
--- a/tests/antithesis/workload/restore-orchestrator.sh
+++ b/tests/antithesis/workload/restore-orchestrator.sh
@@ -52,10 +52,13 @@ k() { kubectl -n "$NS" "$@"; }
 
 # exec_ledgerctl POD ARGS... -- runs ledgerctl inside a ledger pod with the
 # pod-local server address and cluster credentials (the operator-e2e idiom).
+# Bounded by coreutils timeout rather than ledgerctl's --timeout: not every
+# subcommand defines that flag (restore download does not), and a uniform
+# client-side bound also covers a hung exec stream.
 exec_ledgerctl() {
 	local pod="$1"; shift
 	k exec "$pod" -c ledger -- /bin/sh -c \
-		"./ledgerctl $* --server 127.0.0.1:8888 --insecure --auth-token \"\$CLUSTER_SECRET\" --timeout 120s"
+		"timeout 300 ./ledgerctl $* --server 127.0.0.1:8888 --insecure --auth-token \"\$CLUSTER_SECRET\""
 }
 
 # retry N CMD... -- runs CMD up to N times, 5s apart.
@@ -122,6 +125,14 @@ teardown_cluster() {
 }
 
 do_one_restore() {
+	# Incrementals build on the initial full backup; if that never succeeded
+	# (or a cycle-time check finds it missing), (re)take it here — the driver
+	# is quiesced, so a cycle-time full backup is as valid a base as any.
+	if [ "$FULL_BACKUP_OK" != 1 ]; then
+		retry 3 backup_exec "store backup" || { log "full backup (retry) failed"; return 1; }
+		FULL_BACKUP_OK=1
+	fi
+
 	retry 3 backup_exec "store incremental-backup" || { log "incremental backup failed"; return 1; }
 
 	teardown_cluster || { log "teardown failed"; return 1; }
@@ -148,8 +159,13 @@ wait_ready_replicas "$REPLICAS" 1800 || log "WARNING: cluster not ready at start
 
 retry 10 capture_specs || { log "FATAL: cannot capture cluster spec"; exit 1; }
 
+FULL_BACKUP_OK=0
 log "taking initial full backup"
-retry 5 backup_exec "store backup" || log "WARNING: initial full backup failed; first cycle will retry"
+if retry 5 backup_exec "store backup"; then
+	FULL_BACKUP_OK=1
+else
+	log "WARNING: initial full backup failed; the next cycle retakes it before its incremental"
+fi
 
 log "armed; watching $REQ"
 while true; do
@@ -159,9 +175,13 @@ while true; do
 	if do_one_restore; then
 		printf 'ok\n' > "$RESP"
 	else
-		# Converge back to normal mode so the driver never resumes against a
-		# cluster parked in restore mode; the driver logs the err and continues.
+		# Converge back to normal mode and wait for the cluster to serve
+		# before releasing the driver, so it never resumes against a cluster
+		# parked in restore mode or still forming; the driver logs the err
+		# and continues.
 		k apply -f "$NORMAL_SPEC" 2>/dev/null || true
+		wait_ready_replicas "$REPLICAS" 300 \
+			|| log "WARNING: cluster not back to $REPLICAS ready replicas after failed cycle"
 		printf 'err\n' > "$RESP"
 	fi
 done

From e44ec8ee4949cf03369eb78bf1fe3b5e0d413bfe Mon Sep 17 00:00:00 2001
From: Azorlogh <bott.alix@gmail.com>
Date: Wed, 15 Jul 2026 22:44:49 +0000
Subject: [PATCH 11/17] fix(antithesis): fail restore cycles whose incremental
 exported nothing
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Review feedback on #1593 (NumaryBot): after a cycle-time full-backup
retake, the incremental taken in the same quiesce window can carry an
empty delta — the restore then replays no exports and the cycle records
ok without exercising RebuildDelta. backup_exec now keeps the command's
--json output and the cycle fails when it exported nothing, mirroring
the local runner's no-exports guard.
---
 .../workload/restore-orchestrator.sh          | 24 ++++++++++++++++---
 1 file changed, 21 insertions(+), 3 deletions(-)

diff --git a/tests/antithesis/workload/restore-orchestrator.sh b/tests/antithesis/workload/restore-orchestrator.sh
index 91aebe9bee..609926e738 100755
--- a/tests/antithesis/workload/restore-orchestrator.sh
+++ b/tests/antithesis/workload/restore-orchestrator.sh
@@ -73,12 +73,16 @@ retry() {
 }
 
 # backup_exec KIND -- runs a backup command against any live ledger pod (the
-# RPC needs a reachable server; retrying across pods covers a dead one).
+# RPC needs a reachable server; retrying across pods covers a dead one). The
+# command's --json output is left in $BACKUP_OUT for callers that inspect it.
+BACKUP_OUT=/tmp/last-backup.json
 backup_exec() {
 	local kind="$1" i
 	for i in $(seq 0 $(( REPLICAS - 1 ))); do
-		if exec_ledgerctl "$POD_PREFIX-$i" "$kind" "${BACKUP_ARGS[*]}"; then return 0; fi
-		log "$kind via $POD_PREFIX-$i failed; trying next pod"
+		if exec_ledgerctl "$POD_PREFIX-$i" "$kind" "${BACKUP_ARGS[*]}" --json > "$BACKUP_OUT" 2>&1; then
+			return 0
+		fi
+		log "$kind via $POD_PREFIX-$i failed; trying next pod: $(tail -c 300 "$BACKUP_OUT" 2>/dev/null)"
 	done
 	return 1
 }
@@ -135,6 +139,20 @@ do_one_restore() {
 
 	retry 3 backup_exec "store incremental-backup" || { log "incremental backup failed"; return 1; }
 
+	# RebuildDelta only runs when the manifest carries exports; a cycle whose
+	# incremental exported nothing (e.g. taken right after a cycle-time full
+	# backup) would restore without exercising the replay. Fail it rather than
+	# record hollow coverage — the local runner has the same guard.
+	local exports
+	exports=$(grep -o '{.*}' "$BACKUP_OUT" 2>/dev/null | tail -1 \
+		| jq -r '(.segmentsUploaded // 0) + (.logEntriesExported // 0)' 2>/dev/null)
+	case "$exports" in
+	""|0|null)
+		log "incremental produced no exports (RebuildDelta would be a no-op); failing cycle"
+		return 1
+		;;
+	esac
+
 	teardown_cluster || { log "teardown failed"; return 1; }
 
 	k apply -f "$RESTORE_SPEC" || { log "applying restore-mode cluster failed"; return 1; }

From 76683bcea06dae231f4033857f6e141f3aa6df36 Mon Sep 17 00:00:00 2001
From: Azorlogh <bott.alix@gmail.com>
Date: Wed, 15 Jul 2026 23:13:34 +0000
Subject: [PATCH 12/17] fix(antithesis): never release the driver against an
 unrestored cluster
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Review feedback on #1593 (NumaryBot): a cycle failing after
teardown_cluster left the recovery path applying the normal-mode spec —
bringing up a FRESH, EMPTY cluster (the PVCs are gone; the data exists
only in the backup) and releasing the driver into it, where every model
check would report false divergence.

The cycle is now split at the point of no return. Pre-teardown failures
(backup, exports guard) leave the running cluster untouched and report
err as before. From the teardown onward the restore choreography is
retried until it lands, clearing any half-state between attempts; the
driver's lease expiry merely parks it on transient retries until the
restored cluster serves. The verified quiesce-point backup is what
makes the teardown safe to cross in the first place.
---
 .../workload/restore-orchestrator.sh          | 70 +++++++++++++------
 1 file changed, 47 insertions(+), 23 deletions(-)

diff --git a/tests/antithesis/workload/restore-orchestrator.sh b/tests/antithesis/workload/restore-orchestrator.sh
index 609926e738..deec617a83 100755
--- a/tests/antithesis/workload/restore-orchestrator.sh
+++ b/tests/antithesis/workload/restore-orchestrator.sh
@@ -128,6 +128,32 @@ teardown_cluster() {
 	done
 }
 
+# restore_from_backup -- one attempt at the post-teardown choreography:
+# restore-mode cluster, download + finalize, flip back to the full cluster.
+restore_from_backup() {
+	k apply -f "$RESTORE_SPEC" || { log "applying restore-mode cluster failed"; return 1; }
+	wait_pod_running "$POD_PREFIX-0" || { log "restore-mode pod never ran"; return 1; }
+
+	retry 5 exec_ledgerctl "$POD_PREFIX-0" "restore download" "${S3_ARGS[*]}" \
+		|| { log "restore download failed"; return 1; }
+	retry 5 exec_ledgerctl "$POD_PREFIX-0" "restore finalize --yes" \
+		|| { log "restore finalize failed"; return 1; }
+
+	k apply -f "$NORMAL_SPEC" || { log "applying normal-mode cluster failed"; return 1; }
+	# The restore-mode pod does not roll on its own after the spec flip
+	# (operator e2e does the same bounce).
+	sleep 5
+	k delete pod "$POD_PREFIX-0" --force --grace-period=0 2>/dev/null || true
+	wait_ready_replicas "$REPLICAS" || { log "cluster did not return to $REPLICAS ready replicas"; return 1; }
+}
+
+# do_one_restore has two phases with different failure semantics. Before the
+# teardown a failure leaves the running cluster untouched, so the cycle just
+# reports err and the driver carries on. From the teardown onward the driver's
+# committed state exists ONLY in the backup — releasing the driver against a
+# fresh empty cluster would make every model check diverge — so the restore is
+# retried until it lands, however long that takes (the run's duration bounds
+# it; the driver's lease expiry just parks it on transient retries meanwhile).
 do_one_restore() {
 	# Incrementals build on the initial full backup; if that never succeeded
 	# (or a cycle-time check finds it missing), (re)take it here — the driver
@@ -153,22 +179,23 @@ do_one_restore() {
 		;;
 	esac
 
-	teardown_cluster || { log "teardown failed"; return 1; }
-
-	k apply -f "$RESTORE_SPEC" || { log "applying restore-mode cluster failed"; return 1; }
-	wait_pod_running "$POD_PREFIX-0" || { log "restore-mode pod never ran"; return 1; }
-
-	retry 5 exec_ledgerctl "$POD_PREFIX-0" "restore download" "${S3_ARGS[*]}" \
-		|| { log "restore download failed"; return 1; }
-	retry 5 exec_ledgerctl "$POD_PREFIX-0" "restore finalize --yes" \
-		|| { log "restore finalize failed"; return 1; }
-
-	k apply -f "$NORMAL_SPEC" || { log "applying normal-mode cluster failed"; return 1; }
-	# The restore-mode pod does not roll on its own after the spec flip
-	# (operator e2e does the same bounce).
-	sleep 5
-	k delete pod "$POD_PREFIX-0" --force --grace-period=0 2>/dev/null || true
-	wait_ready_replicas "$REPLICAS" || { log "cluster did not return to $REPLICAS ready replicas"; return 1; }
+	# Point of no return: the quiesce-point backup above is verified, so the
+	# data survives whatever happens to the cluster from here.
+	teardown_cluster || log "teardown failed; continuing into restore recovery"
+
+	local attempt=1
+	until restore_from_backup; do
+		attempt=$(( attempt + 1 ))
+		log "restore attempt failed; the data now lives only in the backup — retrying (attempt $attempt)"
+		# Clear whatever half-state the attempt left so the next one starts clean.
+		k delete cluster "$CLUSTER" --wait=true --timeout=120s 2>/dev/null || true
+		local i
+		for i in $(seq 0 $(( REPLICAS - 1 ))); do
+			k delete pvc "wal-$POD_PREFIX-$i" "data-$POD_PREFIX-$i" "cold-cache-$POD_PREFIX-$i" \
+				--ignore-not-found --wait=true --timeout=120s 2>/dev/null || true
+		done
+		sleep 10
+	done
 	log "cluster back up on the restored (RebuildDelta) store"
 }
 
@@ -193,13 +220,10 @@ while true; do
 	if do_one_restore; then
 		printf 'ok\n' > "$RESP"
 	else
-		# Converge back to normal mode and wait for the cluster to serve
-		# before releasing the driver, so it never resumes against a cluster
-		# parked in restore mode or still forming; the driver logs the err
-		# and continues.
-		k apply -f "$NORMAL_SPEC" 2>/dev/null || true
-		wait_ready_replicas "$REPLICAS" 300 \
-			|| log "WARNING: cluster not back to $REPLICAS ready replicas after failed cycle"
+		# Only pre-teardown steps report err (backup / exports guard), and
+		# those leave the running cluster untouched — the driver just logs
+		# the err and continues against it. Post-teardown failures never
+		# reach here: do_one_restore retries the restore until it lands.
 		printf 'err\n' > "$RESP"
 	fi
 done

From 1c06a78e763302dc39850dd3e3c10bdead715b58 Mon Sep 17 00:00:00 2001
From: Azorlogh <bott.alix@gmail.com>
Date: Wed, 15 Jul 2026 23:39:11 +0000
Subject: [PATCH 13/17] fix(antithesis): recognise lost-response
 download/finalize as success
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Observed in the Antithesis run: a network partition ate the exec
response of a restore download that had completed server-side, and
every retry then failed on FailedPrecondition "backup already
downloaded" until the cycle gave up. The download and finalize retries
now recognise the FailedPrecondition a repeat hits as prior success —
"already downloaded" for download, and "no backup downloaded" for
finalize, which consumed the staging the download step already proved
existed.
---
 .../workload/restore-orchestrator.sh          | 28 ++++++++++++++++---
 1 file changed, 24 insertions(+), 4 deletions(-)

diff --git a/tests/antithesis/workload/restore-orchestrator.sh b/tests/antithesis/workload/restore-orchestrator.sh
index deec617a83..98c97ab9e6 100755
--- a/tests/antithesis/workload/restore-orchestrator.sh
+++ b/tests/antithesis/workload/restore-orchestrator.sh
@@ -128,16 +128,36 @@ teardown_cluster() {
 	done
 }
 
+# The exec's response can be lost mid-step (network partition, bounded client)
+# while the server-side operation completed, so the download/finalize retries
+# must recognise the FailedPrecondition a repeat then hits as prior success:
+# a repeated download reports "already downloaded"; a repeated finalize
+# reports "no backup downloaded" because finalize consumed the staging — and
+# the download step has already proven the staging existed.
+download_step() {
+	local out
+	out=$(exec_ledgerctl "$POD_PREFIX-0" "restore download" "${S3_ARGS[*]}" 2>&1) && return 0
+	case "$out" in *"already downloaded"*) return 0 ;; esac
+	log "download attempt failed: $(printf '%s' "$out" | tail -c 200)"
+	return 1
+}
+
+finalize_step() {
+	local out
+	out=$(exec_ledgerctl "$POD_PREFIX-0" "restore finalize --yes" 2>&1) && return 0
+	case "$out" in *"no backup downloaded"*) return 0 ;; esac
+	log "finalize attempt failed: $(printf '%s' "$out" | tail -c 200)"
+	return 1
+}
+
 # restore_from_backup -- one attempt at the post-teardown choreography:
 # restore-mode cluster, download + finalize, flip back to the full cluster.
 restore_from_backup() {
 	k apply -f "$RESTORE_SPEC" || { log "applying restore-mode cluster failed"; return 1; }
 	wait_pod_running "$POD_PREFIX-0" || { log "restore-mode pod never ran"; return 1; }
 
-	retry 5 exec_ledgerctl "$POD_PREFIX-0" "restore download" "${S3_ARGS[*]}" \
-		|| { log "restore download failed"; return 1; }
-	retry 5 exec_ledgerctl "$POD_PREFIX-0" "restore finalize --yes" \
-		|| { log "restore finalize failed"; return 1; }
+	retry 5 download_step || { log "restore download failed"; return 1; }
+	retry 5 finalize_step || { log "restore finalize failed"; return 1; }
 
 	k apply -f "$NORMAL_SPEC" || { log "applying normal-mode cluster failed"; return 1; }
 	# The restore-mode pod does not roll on its own after the spec flip

From 70c152c18f93562b71d87734159b2aac90531ce1 Mon Sep 17 00:00:00 2001
From: Azorlogh <bott.alix@gmail.com>
Date: Wed, 15 Jul 2026 23:41:52 +0000
Subject: [PATCH 14/17] fix(antithesis): parse the whole backup JSON output for
 the exports guard
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Review feedback on #1593 (NumaryBot): ledgerctl's --json output can be
pretty-printed across lines, which the single-line grep never matched —
the exports guard would then read empty and fail every cycle. The
backup capture now separates stdout (pure JSON, fed to jq whole, the
same way the local runner parses it) from stderr (kept for failure
logs).
---
 tests/antithesis/workload/restore-orchestrator.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/antithesis/workload/restore-orchestrator.sh b/tests/antithesis/workload/restore-orchestrator.sh
index 98c97ab9e6..bb918aed6a 100755
--- a/tests/antithesis/workload/restore-orchestrator.sh
+++ b/tests/antithesis/workload/restore-orchestrator.sh
@@ -79,10 +79,11 @@ BACKUP_OUT=/tmp/last-backup.json
 backup_exec() {
 	local kind="$1" i
 	for i in $(seq 0 $(( REPLICAS - 1 ))); do
-		if exec_ledgerctl "$POD_PREFIX-$i" "$kind" "${BACKUP_ARGS[*]}" --json > "$BACKUP_OUT" 2>&1; then
+		if exec_ledgerctl "$POD_PREFIX-$i" "$kind" "${BACKUP_ARGS[*]}" --json \
+			> "$BACKUP_OUT" 2> "$BACKUP_OUT.err"; then
 			return 0
 		fi
-		log "$kind via $POD_PREFIX-$i failed; trying next pod: $(tail -c 300 "$BACKUP_OUT" 2>/dev/null)"
+		log "$kind via $POD_PREFIX-$i failed; trying next pod: $(tail -c 300 "$BACKUP_OUT.err" 2>/dev/null)"
 	done
 	return 1
 }
@@ -190,8 +191,7 @@ do_one_restore() {
 	# backup) would restore without exercising the replay. Fail it rather than
 	# record hollow coverage — the local runner has the same guard.
 	local exports
-	exports=$(grep -o '{.*}' "$BACKUP_OUT" 2>/dev/null | tail -1 \
-		| jq -r '(.segmentsUploaded // 0) + (.logEntriesExported // 0)' 2>/dev/null)
+	exports=$(jq -r '(.segmentsUploaded // 0) + (.logEntriesExported // 0)' "$BACKUP_OUT" 2>/dev/null)
 	case "$exports" in
 	""|0|null)
 		log "incremental produced no exports (RebuildDelta would be a no-op); failing cycle"

From e01cdee3d56eecf7684fa8911b4e55c70f32da14 Mon Sep 17 00:00:00 2001
From: Azorlogh <bott.alix@gmail.com>
Date: Thu, 16 Jul 2026 01:28:24 +0000
Subject: [PATCH 15/17] fix(antithesis): keep test templates out of the
 restore-orchestrator sidecar
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The sidecar reused the workload image, and any container whose image
carries /opt/antithesis/test/v1 hosts those templates too: the Composer
scheduled main/first_default_ledger into the sidecar, where the workload
env (LEDGER_GRPC_ADDR, ...) is absent, so it dialed the localhost
fallback and died — failing the "Commands finish with zero exit code"
Always property on every k8s run of this branch (present in takes 3-4,
absent in pre-sidecar baselines). The sidecar now uses a --target
sidecar image variant that strips the test tree.
---
 .claude/skills/antithesis-run/SKILL.md | 10 ++++++++--
 tests/antithesis/k8s/workload.yaml     |  7 +++++--
 tests/antithesis/workload/Dockerfile   | 13 ++++++++++++-
 tests/antithesis/workload/Justfile     |  2 ++
 4 files changed, 27 insertions(+), 5 deletions(-)

diff --git a/.claude/skills/antithesis-run/SKILL.md b/.claude/skills/antithesis-run/SKILL.md
index 139f16b5d5..ecf09f4f0c 100644
--- a/.claude/skills/antithesis-run/SKILL.md
+++ b/.claude/skills/antithesis-run/SKILL.md
@@ -98,13 +98,19 @@ docker build --platform linux/amd64 \
   -t "$ANTITHESIS_REPOSITORY/ledger-operator:$TAG" misc/operator
 docker push "$ANTITHESIS_REPOSITORY/ledger-operator:$TAG"
 
-# 4. Build & push workload.
+# 4. Build & push workload (+ the sidecar variant, which ships without the
+#    test tree so the Composer never schedules test commands into it).
 ( cd tests/antithesis/workload && \
   docker build --platform linux/amd64 \
     --build-arg GOARCH=amd64 --build-arg GOOS=linux \
     -f Dockerfile \
     -t "$ANTITHESIS_REPOSITORY/workload-ledger-v3:$TAG" ../../.. && \
-  docker push "$ANTITHESIS_REPOSITORY/workload-ledger-v3:$TAG" )
+  docker push "$ANTITHESIS_REPOSITORY/workload-ledger-v3:$TAG" && \
+  docker build --platform linux/amd64 \
+    --build-arg GOARCH=amd64 --build-arg GOOS=linux \
+    -f Dockerfile --target sidecar \
+    -t "$ANTITHESIS_REPOSITORY/workload-ledger-v3-sidecar:$TAG" ../../.. && \
+  docker push "$ANTITHESIS_REPOSITORY/workload-ledger-v3-sidecar:$TAG" )
 ```
 
 **Compose mode** — same image build/push, plus the etcd retag:
diff --git a/tests/antithesis/k8s/workload.yaml b/tests/antithesis/k8s/workload.yaml
index 14ae55298e..6e4f46a12a 100644
--- a/tests/antithesis/k8s/workload.yaml
+++ b/tests/antithesis/k8s/workload.yaml
@@ -135,9 +135,12 @@ spec:
             memory: 512Mi
       # Services the model driver's restore requests by driving the operator:
       # quiesce-point backup, cluster teardown, restore-mode round-trip, and
-      # the flip back to normal. See restore-orchestrator.sh.
+      # the flip back to normal. See restore-orchestrator.sh. The sidecar image
+      # variant carries no /opt/antithesis/test tree, so the Composer never
+      # schedules test commands into this container (they would run without
+      # the workload env above).
       - name: restore-orchestrator
-        image: __REGISTRY__/workload-ledger-v3:__TAG__
+        image: __REGISTRY__/workload-ledger-v3-sidecar:__TAG__
         command: ["/opt/antithesis/restore-orchestrator.sh"]
         env:
         - name: NAMESPACE
diff --git a/tests/antithesis/workload/Dockerfile b/tests/antithesis/workload/Dockerfile
index d19906285c..37ebb7c774 100644
--- a/tests/antithesis/workload/Dockerfile
+++ b/tests/antithesis/workload/Dockerfile
@@ -59,7 +59,7 @@ RUN set -e; \
         | xargs -P"$(nproc)" -I{} sh -c 'mkdir -p "/cmds/$(dirname "$1")" && go build -race -o "/cmds/$1" "./bin/cmds/$1"' _ {}
 
 # Runner
-FROM debian:trixie
+FROM debian:trixie AS runner
 
 RUN apt-get update && apt-get install -y curl jq
 
@@ -84,3 +84,14 @@ ENTRYPOINT ["/init"]
 # own template (model), separate from the rest of the suite (main).
 COPY --from=compiler /cmds/ /opt/antithesis/test/v1/
 COPY --from=compiler /instrumented/symbols/ /symbols
+
+# Sidecar image (build with --target sidecar): any container whose image
+# carries /opt/antithesis/test/v1 is treated by the Composer as another host of
+# those templates, and commands scheduled into the sidecar run without the
+# workload env (LEDGER_GRPC_ADDR, ...). The restore-orchestrator image
+# therefore ships without the test tree.
+FROM runner AS sidecar
+RUN rm -rf /opt/antithesis/test
+
+# Default target: the workload image, test templates included.
+FROM runner AS workload
diff --git a/tests/antithesis/workload/Justfile b/tests/antithesis/workload/Justfile
index 485516ea29..d8b066a3fe 100644
--- a/tests/antithesis/workload/Justfile
+++ b/tests/antithesis/workload/Justfile
@@ -6,6 +6,8 @@ tag := 'antithesis'
 # where the Antithesis composer should focus on a specific subset.
 build include='':
 	docker build --platform linux/amd64 --build-arg GOARCH=amd64 --build-arg GOOS=linux --build-arg INCLUDED_DRIVERS={{quote(include)}} -f Dockerfile --progress=plain -t "$ANTITHESIS_REPOSITORY/workload-ledger-v3:{{tag}}" ../../../
+	docker build --platform linux/amd64 --build-arg GOARCH=amd64 --build-arg GOOS=linux --build-arg INCLUDED_DRIVERS={{quote(include)}} -f Dockerfile --target sidecar --progress=plain -t "$ANTITHESIS_REPOSITORY/workload-ledger-v3-sidecar:{{tag}}" ../../../
 
 push include='': (build include)
 	docker push "$ANTITHESIS_REPOSITORY/workload-ledger-v3:{{tag}}"
+	docker push "$ANTITHESIS_REPOSITORY/workload-ledger-v3-sidecar:{{tag}}"

From 1a5ff5806e9f20b2bc5c43cfed037e5a6fae0a23 Mon Sep 17 00:00:00 2001
From: Azorlogh <bott.alix@gmail.com>
Date: Thu, 16 Jul 2026 08:30:23 +0000
Subject: [PATCH 16/17] fix(antithesis): atomic claim/withdraw on the restore
 rendezvous request
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Review feedback on #1593 (NumaryBot): a request the driver had given up
on (lease expiry) stayed on disk, so a late orchestrator could pick it
up and cut the backup while the driver was committing again — the
restore would then erase committed state and poison every model check.

Both sides now take the request file by rename, so exactly one wins:
the orchestrators claim it to start a cycle, the driver withdraws it on
lease expiry or shutdown. If the claim already happened, the driver
stays parked until the orchestrator answers — resuming mid-cycle has
the same poisoning effect (the cycle may already be past the teardown).
---
 tests/antithesis/run_model_test.sh            |  7 ++++--
 .../model/singleton_driver_model/restore.go   | 23 ++++++++++++++++++-
 .../workload/restore-orchestrator.sh          |  7 ++++--
 3 files changed, 32 insertions(+), 5 deletions(-)

diff --git a/tests/antithesis/run_model_test.sh b/tests/antithesis/run_model_test.sh
index 094c8cb338..eea7f2338d 100755
--- a/tests/antithesis/run_model_test.sh
+++ b/tests/antithesis/run_model_test.sh
@@ -248,8 +248,11 @@ do_one_restore() {
 # service_restore_request handles one pending request, if any. Runs in the main
 # shell (not a subshell) so start_node's SERVER_PIDS update is visible to cleanup.
 service_restore_request() {
-	[ "$RESTORE" = 1 ] && [ -f "$RESTORE_REQ" ] || return 0
-	rm -f "$RESTORE_REQ"
+	[ "$RESTORE" = 1 ] || return 0
+	# Atomic claim, mirroring the driver's withdraw-by-rename on lease expiry:
+	# exactly one side gets the request.
+	mv "$RESTORE_REQ" "$RESTORE_REQ.claimed" 2>/dev/null || return 0
+	rm -f "$RESTORE_REQ.claimed"
 	log "restore: cycle requested"
 	if do_one_restore; then
 		RESTORE_CYCLES=$(( RESTORE_CYCLES + 1 ))
diff --git a/tests/antithesis/workload/bin/cmds/model/singleton_driver_model/restore.go b/tests/antithesis/workload/bin/cmds/model/singleton_driver_model/restore.go
index d2e35af077..7949b108f7 100644
--- a/tests/antithesis/workload/bin/cmds/model/singleton_driver_model/restore.go
+++ b/tests/antithesis/workload/bin/cmds/model/singleton_driver_model/restore.go
@@ -162,14 +162,35 @@ func (t *fileTrigger) Fire(ctx context.Context) error {
 
 		select {
 		case <-ctx.Done():
+			t.withdraw()
 			return ctx.Err()
 		case <-timeout.C:
-			return fmt.Errorf("restore timed out after %s", cap)
+			if t.withdraw() {
+				return fmt.Errorf("restore timed out after %s (request never claimed)", cap)
+			}
+			// The orchestrator claimed the request and is mid-cycle — possibly
+			// past the teardown, where the driver's state exists only in the
+			// backup. Resuming unquiesced would cut commits the restore then
+			// erases, poisoning every model check, so keep waiting.
+			log.Printf("restore cycle: lease %s expired but the request is claimed; waiting for the orchestrator", cap)
+			timeout.Reset(cap)
 		case <-time.After(restorePoll):
 		}
 	}
 }
 
+// withdraw atomically takes the request back, returning false if the
+// orchestrator claimed it first (its claim is a rename too, so exactly one
+// side wins). A withdrawn request can never trigger a cycle later, when the
+// driver is no longer quiesced.
+func (t *fileTrigger) withdraw() bool {
+	if os.Rename(t.reqPath, t.reqPath+".expired") != nil {
+		return false
+	}
+	_ = os.Remove(t.reqPath + ".expired")
+	return true
+}
+
 // restoreTimeout is the per-cycle lease, from MODEL_RESTORE_TIMEOUT seconds.
 func restoreTimeout() time.Duration {
 	return time.Duration(envInt("MODEL_RESTORE_TIMEOUT", defaultRestoreTimeoutSecs)) * time.Second
diff --git a/tests/antithesis/workload/restore-orchestrator.sh b/tests/antithesis/workload/restore-orchestrator.sh
index bb918aed6a..328088c7a6 100755
--- a/tests/antithesis/workload/restore-orchestrator.sh
+++ b/tests/antithesis/workload/restore-orchestrator.sh
@@ -234,8 +234,11 @@ fi
 
 log "armed; watching $REQ"
 while true; do
-	if [ ! -f "$REQ" ]; then sleep 1; continue; fi
-	rm -f "$REQ" "$RESP"
+	# Atomic claim: the driver withdraws an expired request with a rename of
+	# its own, so exactly one side wins — a request the driver has withdrawn
+	# (it is no longer quiesced) can never start a cycle here.
+	if ! mv "$REQ" "$REQ.claimed" 2>/dev/null; then sleep 1; continue; fi
+	rm -f "$REQ.claimed" "$RESP"
 	log "restore cycle requested"
 	if do_one_restore; then
 		printf 'ok\n' > "$RESP"

From bf2d0bd1b2cc9f92c1e5e251bb27844e521a35dc Mon Sep 17 00:00:00 2001
From: Azorlogh <bott.alix@gmail.com>
Date: Thu, 16 Jul 2026 08:48:40 +0000
Subject: [PATCH 17/17] fix(antithesis): enter the restore only on a
 verified-complete teardown
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Review feedback on #1593 (flemzord): the point-of-no-return crossed
into the restore even when the teardown had failed partway, so a
lingering PVC could hand the restore-mode pod its old store back —
stale data served as "restored", hollow RebuildDelta coverage.

teardown_cluster is now idempotent (--ignore-not-found on the CR),
verifies every claim is actually gone before reporting success, and is
retried until it does — both at the point of no return and between
restore attempts, where the inline half-state cleanup duplicated it.
---
 .../workload/restore-orchestrator.sh          | 30 ++++++++++++-------
 1 file changed, 20 insertions(+), 10 deletions(-)

diff --git a/tests/antithesis/workload/restore-orchestrator.sh b/tests/antithesis/workload/restore-orchestrator.sh
index 328088c7a6..db1db4824b 100755
--- a/tests/antithesis/workload/restore-orchestrator.sh
+++ b/tests/antithesis/workload/restore-orchestrator.sh
@@ -117,8 +117,12 @@ capture_specs() {
 	jq '.spec.restore = true | .spec.replicas = 1' "$NORMAL_SPEC" > "$RESTORE_SPEC"
 }
 
+# teardown_cluster is idempotent and succeeds only once the CR, pods, and
+# volume claims are verifiably gone — callers retry it until then. A lingering
+# claim would hand the re-created pod its old store back, silently voiding the
+# RebuildDelta coverage the cycle claims.
 teardown_cluster() {
-	k delete cluster "$CLUSTER" --wait=true --timeout=120s || return 1
+	k delete cluster "$CLUSTER" --ignore-not-found --wait=true --timeout=120s || return 1
 	local i
 	for i in $(seq 0 $(( REPLICAS - 1 ))); do
 		k wait --for=delete "pod/$POD_PREFIX-$i" --timeout=120s 2>/dev/null || true
@@ -127,6 +131,10 @@ teardown_cluster() {
 		k delete pvc "wal-$POD_PREFIX-$i" "data-$POD_PREFIX-$i" "cold-cache-$POD_PREFIX-$i" \
 			--ignore-not-found --wait=true --timeout=120s || return 1
 	done
+	for i in $(seq 0 $(( REPLICAS - 1 ))); do
+		[ -z "$(k get pvc "wal-$POD_PREFIX-$i" "data-$POD_PREFIX-$i" "cold-cache-$POD_PREFIX-$i" \
+			--ignore-not-found -o name 2>/dev/null)" ] || return 1
+	done
 }
 
 # The exec's response can be lost mid-step (network partition, bounded client)
@@ -200,20 +208,22 @@ do_one_restore() {
 	esac
 
 	# Point of no return: the quiesce-point backup above is verified, so the
-	# data survives whatever happens to the cluster from here.
-	teardown_cluster || log "teardown failed; continuing into restore recovery"
-
+	# data survives whatever happens to the cluster from here. The restore is
+	# only entered on a verified-complete teardown — booting the restore-mode
+	# pod with any old storage left would serve stale data as "restored".
 	local attempt=1
+	until teardown_cluster; do
+		attempt=$(( attempt + 1 ))
+		log "teardown incomplete; retrying until the old store is verifiably gone (attempt $attempt)"
+		sleep 10
+	done
+
+	attempt=1
 	until restore_from_backup; do
 		attempt=$(( attempt + 1 ))
 		log "restore attempt failed; the data now lives only in the backup — retrying (attempt $attempt)"
 		# Clear whatever half-state the attempt left so the next one starts clean.
-		k delete cluster "$CLUSTER" --wait=true --timeout=120s 2>/dev/null || true
-		local i
-		for i in $(seq 0 $(( REPLICAS - 1 ))); do
-			k delete pvc "wal-$POD_PREFIX-$i" "data-$POD_PREFIX-$i" "cold-cache-$POD_PREFIX-$i" \
-				--ignore-not-found --wait=true --timeout=120s 2>/dev/null || true
-		done
+		until teardown_cluster; do sleep 10; done
 		sleep 10
 	done
 	log "cluster back up on the restored (RebuildDelta) store"