[WIP] evetest: kvm→k boot-disk conversion tests + volverify app

evetest/broker/provider/qemu.go

@@ -1482,6 +1482,15 @@ func (dev *qemuDevice) buildArgs() []string {
 	args := []string{
 		"-enable-kvm",
 		"-machine", "q35",
+		// Let the chipset watchdog actually reset the guest. q35 brings the ICH9
+		// LPC bridge and with it the iTCO watchdog EVE drives through
+		// /dev/watchdog, but the bridge only latches a status bit on timeout
+		// unless noreboot is cleared. Without these two a guest silently survives
+		// a watchdog it should have died on, which hides watchdog regressions and
+		// quietly disarms any test that injects one. eden configures its QEMU the
+		// same way.
+		"-global", "ICH9-LPC.noreboot=false",
+		"-watchdog-action", "reset",
 		"-cpu", "host",
 		"-smp", fmt.Sprintf("%d", dev.spec.CPUs),
 		"-m", fmt.Sprintf("%d", dev.spec.MemoryBytes>>20),

evetest/devconfig.go

@@ -886,9 +886,20 @@ type ApplicationInstanceConfig struct {
 	UserData            string
 	NetworkAdapters     []AppNetworkAdapter
 	EnforceNetIntfOrder bool
+	// DataVolumes are additional blank (empty) volumes attached to the app beyond
+	// its image. Each has no content tree — EVE creates an empty disk of the given
+	// size (VCOT_BLANK) and mounts it at MountDir.
+	DataVolumes []DataVolumeConfig
 	// Many more parameters can be configured; they will be added later as needed.
 }
 
+// DataVolumeConfig describes one blank data volume attached to an application.
+type DataVolumeConfig struct {
+	SizeBytes uint64 // volume size in bytes (Maxsizebytes)
+	MountDir  string // mount point inside the app (default: /mnt/vol<N>)
+	ReadOnly  bool
+}
+
 func (config ApplicationInstanceConfig) toProto(th *TestHarness, devName string,
 	appUUID, volumeUUID uuid.UUID) *eveconfig.AppInstanceConfig {
 	vmConfig := &eveconfig.VmConfig{
@@ -2272,6 +2283,32 @@ func (dc *EdgeDeviceConfig) addApplicationWithUUIDs(
 		contentTreeUUID, datastoreUUID, config.DisplayName)
 	dc.ContentInfo = append(dc.ContentInfo, contentTree)
 	dc.Datastores = append(dc.Datastores, dsConfig)
+
+	// Attach any blank data volumes: a VCOT_BLANK Volume (no content tree or
+	// datastore) plus a VolumeRef mounting it into the app. appInstConfig is a
+	// pointer already stored in dc.Apps, so extending its VolumeRefList here is
+	// reflected in the stored config.
+	for i, dv := range config.DataVolumes {
+		dvUUID := dc.th.newUUID("application data volume")
+		mountDir := dv.MountDir
+		if mountDir == "" {
+			mountDir = fmt.Sprintf("/mnt/vol%d", i)
+		}
+		appInstConfig.VolumeRefList = append(appInstConfig.VolumeRefList,
+			&eveconfig.VolumeRef{
+				Uuid:     dvUUID.String(),
+				MountDir: mountDir,
+			})
+		dc.Volumes = append(dc.Volumes, &eveconfig.Volume{
+			Uuid: dvUUID.String(),
+			Origin: &eveconfig.VolumeContentOrigin{
+				Type: eveconfig.VolumeContentOriginType_VCOT_BLANK,
+			},
+			Maxsizebytes: int64(dv.SizeBytes),
+			Readonly:     dv.ReadOnly,
+			DisplayName:  fmt.Sprintf("%s-data%d", config.DisplayName, i),
+		})
+	}
 }
 
 // UpdateApplication updates an existing application instance identified

evetest/edgedevice.go

@@ -38,6 +38,9 @@ type EdgeDevice struct {
 	// Set of app UUIDs (strings) for which WaitUntilAppIsRunning is active.
 	// Used by WatchAppInfo to suppress duplicate state logging.
 	appsBeingWaited sync.Map
+	// Per-handle override of how long to wait for an upgrade; see
+	// SetUpgradeTimeout. Zero means use the package default.
+	upgradeTimeout time.Duration
 }
 
 // GetEdgeDevice returns a handle to an onboarded EdgeDevice identified by devName.
@@ -483,7 +486,7 @@ func (d *EdgeDevice) waitForUpgrade(targetShortVersion string) {
 	}
 	defer unsub()
 
-	ctx, cancel := context.WithTimeout(d.th.ctx, eveUpgradeTimeout)
+	ctx, cancel := context.WithTimeout(d.th.ctx, d.upgradeWaitTimeout())
 	defer cancel()
 
 	var lastLoggedState, lastLoggedStatus string
@@ -626,6 +629,40 @@ func (d *EdgeDevice) HardReboot(waitUntilRebooted bool) {
 	})
 }
 
+// ExpectAdditionalReboots tells the harness to expect n more device-initiated
+// reboots beyond those it already accounts for (UpgradeEVE counts one reboot per
+// upgrade). A cross-flavor boot-disk conversion that runs an offline shrink/grow
+// reboots additional times; the number is known only to the test driving it (a
+// plain kvm<->k conversion with no resize reboots differently than a shrink+grow),
+// so the test declares it here to keep the teardown reboot-count check accurate.
+func (d *EdgeDevice) ExpectAdditionalReboots(n int) {
+	for i := 0; i < n; i++ {
+		d.th.incExpectedRebootCount(d.devName)
+	}
+}
+
+// SetUpgradeTimeout overrides how long UpgradeEVE waits for the device to come
+// back running the target version, for this handle only.
+//
+// The default is sized for an ordinary base-OS upgrade: download, one reboot,
+// done. A cross-flavor boot-disk conversion is a different animal — it also runs
+// an offline shrink+grow across several reboots and then brings up a whole
+// container-cluster stack — and it lands close enough to the default that a
+// healthy conversion and an expired budget are decided by a couple of minutes of
+// host load. A test driving one should raise the timeout, otherwise it reports a
+// conversion that was still progressing as a failure.
+func (d *EdgeDevice) SetUpgradeTimeout(timeout time.Duration) {
+	d.upgradeTimeout = timeout
+}
+
+// upgradeWaitTimeout is the effective upgrade wait for this handle.
+func (d *EdgeDevice) upgradeWaitTimeout() time.Duration {
+	if d.upgradeTimeout > 0 {
+		return d.upgradeTimeout
+	}
+	return eveUpgradeTimeout
+}
+
 // rebootAndWait executes triggerFn to initiate a device reboot and, if
 // wait is true, blocks until the device confirms the reboot by reporting
 // a ZInfoDevice.lastRebootTime strictly after the moment triggerFn was called.

evetest/testapps/volverify/Dockerfile

@@ -0,0 +1,38 @@
+# Copyright (c) 2026 Zededa, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+ARG GOLANG_VERSION=1.25
+
+# hadolint ignore=DL3029
+FROM --platform=$BUILDPLATFORM golang:${GOLANG_VERSION} AS build
+
+ARG TARGETOS
+ARG TARGETARCH
+
+WORKDIR /app
+
+COPY go.mod ./
+RUN go mod download
+
+COPY . .
+RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -o /volverify ./cmd/volverify
+
+FROM ubuntu:24.04
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+# hadolint ignore=DL3008
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends openssh-server e2fsprogs && \
+    rm -rf /var/lib/apt/lists/* && \
+    mkdir /run/sshd
+
+RUN echo 'root:testpassword' | chpasswd
+RUN sed -i 's/#PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config && \
+    sed -i 's/#PasswordAuthentication.*/PasswordAuthentication yes/' /etc/ssh/sshd_config
+EXPOSE 22
+
+COPY --from=build /volverify /usr/local/bin/volverify
+COPY init.sh /
+
+CMD ["/bin/bash", "/init.sh"]

evetest/testapps/volverify/Makefile

@@ -0,0 +1,16 @@
+# Copyright (c) 2026 Zededa, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+EVETEST_ORG ?= lfedge
+IMAGE = $(EVETEST_ORG)/evetest-volverify
+# Update VERSION whenever there is a change made to this app.
+VERSION ?= 1.0
+
+DOCKER_TARGET ?= load
+DOCKER_PLATFORM ?= $(shell uname -s | tr '[A-Z]' '[a-z]')/$(subst aarch64,arm64,$(subst x86_64,amd64,$(shell uname -m)))
+
+build:
+	docker buildx build \
+		--$(DOCKER_TARGET) \
+		--platform $(DOCKER_PLATFORM) \
+		-t $(IMAGE):$(VERSION) .

evetest/testapps/volverify/README.md

@@ -0,0 +1,49 @@
+# volverify — application-volume corruption verifier
+
+A test-app tool that writes a deterministic, self-verifying fill/delete pattern to
+an application volume and later checks it, to detect corruption caused by a
+watchdog-interrupted EVE-kvm→EVE-k offline filesystem shrink. It is the ground
+truth the soak harness pairs with the resize fsck marker.
+
+Design: `~/notes/kvm-to-k-appvol-shrink-soak-design.md` (§4). Deployed inside the
+evetest ubuntu app and driven over SSH via `RunShellScriptInsideApp`.
+
+## What it does
+
+- **Layer 1** — every 4 KiB block is `AES-CTR(key=derive(fileID), iv=blockIndex)`
+  plus a header carrying the *logical* identity `(fileID, blockIndex)` and CRCs.
+  The identity is logical (file offset ÷ block size), never physical disk
+  placement — placement changes by design when the shrink relocates the P3 tail,
+  and the verifier checks that each logical read still yields the identity's
+  bytes. Reproducible, incompressible, non-zero, so a zeroed/torn/misplaced block
+  is unambiguous.
+- **Layer 2** — a `masterSeed`-seeded PRNG drives a deterministic
+  create/delete/mkdir/rmdir op stream; the writer fsyncs and advances a 2-slot
+  ping-pong committed-index every `--commit-every` ops. The verifier replays the
+  stream to the committed index and classifies each expected file:
+  `ok / present-corrupt / orphaned (in lost+found) / lost / resurrected`.
+
+## Usage
+
+```sh
+volverify write  --dir /mnt/data --seed 42 --ops 100000   # crash-safe, resumable
+volverify verify --dir /mnt/data --seed 42 --ops 100000   # exits non-zero on any anomaly
+```
+
+Both invocations must use the same `--seed` and size flags. `write` is idempotent
+across reboots (it resumes from the committed index).
+
+`verify --expect-committed <N>` supplies an off-volume floor on the committed op
+index. The on-volume commit slots live on the same volume being shrunk, so fsck can
+clear them along with the last files' data — which would make the verifier expect
+nothing and mask the loss. Since the soak harness runs `write` to completion before
+the shrink, it knows the true high-water mark and passes it here, so the last work is
+still expected (and its loss flagged).
+
+## Build
+
+```sh
+make build                       # docker image lfedge/evetest-volverify:1.0
+GOWORK=off go test ./...         # unit tests (fault-injection classification)
+sudo ./scripts/loopback-ext4-test.sh   # on-fs fidelity check (real ext4 + e2fsck)
+```

evetest/testapps/volverify/cmd/volverify/main.go

@@ -0,0 +1,117 @@
+// Copyright (c) 2026 Zededa, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+// Command volverify writes and later verifies a deterministic, self-describing
+// fill/delete pattern on an application volume, to detect corruption caused by a
+// watchdog-interrupted EVE-kvm→EVE-k offline filesystem shrink.
+//
+// It is deployed inside the evetest test app and driven over SSH:
+//
+//	volverify write  --dir /mnt/data --seed 42 --ops 100000
+//	volverify verify --dir /mnt/data --seed 42 --ops 100000
+//
+// write is crash-safe and resumable: run it repeatedly across reboots. verify
+// exits non-zero when it finds any anomaly and prints a machine-readable summary.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+
+	"github.com/lf-edge/eve/evetest/testapps/volverify/internal/verify"
+)
+
+func main() {
+	if len(os.Args) < 2 {
+		usage()
+		os.Exit(2)
+	}
+	cmd := os.Args[1]
+	fs := flag.NewFlagSet(cmd, flag.ExitOnError)
+	dir := fs.String("dir", "", "volume mount point to operate on (required)")
+	def := verify.DefaultConfig()
+	seed := fs.Uint64("seed", def.Seed, "master seed for the op stream")
+	ops := fs.Uint64("ops", def.Ops, "number of ops to apply / expect")
+	commitEvery := fs.Uint64("commit-every", def.CommitEvery, "fsync + commit cadence in ops")
+	blockSize := fs.Int("block-size", def.BlockSize, "on-disk block size in bytes")
+	dirFanout := fs.Int("dir-fanout", def.DirFanout, "per-level file-tree fan-out")
+	smallBlocks := fs.Int("small-blocks", def.SmallBlocks, "max blocks for a small file")
+	medBlocks := fs.Int("med-blocks", def.MedBlocks, "max blocks for a medium file")
+	maxBlocks := fs.Int("max-blocks", def.MaxBlocks, "max blocks for a large file")
+	expectCommitted := fs.Int64("expect-committed", def.ExpectCommitted,
+		"verify: floor on the committed op index (harness high-water mark); -1 = trust on-volume commit only")
+	_ = fs.Parse(os.Args[2:])
+
+	if *dir == "" {
+		fmt.Fprintln(os.Stderr, "error: --dir is required")
+		os.Exit(2)
+	}
+	cfg := verify.Config{
+		Seed:            *seed,
+		BlockSize:       *blockSize,
+		Ops:             *ops,
+		CommitEvery:     *commitEvery,
+		DirFanout:       *dirFanout,
+		SmallBlocks:     *smallBlocks,
+		MedBlocks:       *medBlocks,
+		MaxBlocks:       *maxBlocks,
+		ExpectCommitted: *expectCommitted,
+	}
+
+	switch cmd {
+	case "write":
+		w, err := verify.NewWriter(*dir, cfg)
+		if err != nil {
+			fatal(err)
+		}
+		committed, err := w.Run()
+		if err != nil {
+			fatal(err)
+		}
+		fmt.Printf("write: complete committed=%d\n", committed)
+	case "verify":
+		rep, err := verify.Verify(*dir, cfg)
+		if err != nil {
+			fatal(err)
+		}
+		fmt.Println(rep.String())
+		for _, a := range rep.Anomalies {
+			fmt.Printf("  ANOMALY file=%d verdict=%s path=%s expBlocks=%d sizeMismatch=%v blocks=%v\n",
+				a.FileID, a.Verdict, a.Path, a.ExpectBlocks, a.SizeMismatch, blockCountsString(a.BlockCounts))
+		}
+		for _, id := range rep.Resurrected {
+			fmt.Printf("  ANOMALY resurrected file=%d\n", id)
+		}
+		if !rep.Clean() {
+			os.Exit(1)
+		}
+		fmt.Println("verify: clean")
+	default:
+		usage()
+		os.Exit(2)
+	}
+}
+
+func blockCountsString(m map[verify.BlockStatus]int) string {
+	out := ""
+	for s, n := range m {
+		if s == verify.BlockOK {
+			continue
+		}
+		out += fmt.Sprintf("%s=%d ", s, n)
+	}
+	if out == "" {
+		return "-"
+	}
+	return out
+}
+
+func fatal(err error) {
+	fmt.Fprintln(os.Stderr, "error:", err)
+	os.Exit(1)
+}
+
+func usage() {
+	fmt.Fprintln(os.Stderr, "usage: volverify <write|verify> --dir <mount> [--seed N --ops N ...]")
+}

evetest/testapps/volverify/go.mod

@@ -0,0 +1,3 @@
+module github.com/lf-edge/eve/evetest/testapps/volverify
+
+go 1.25

evetest/testapps/volverify/init.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# Copyright (c) 2026 Zededa, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+# The volverify binary is not a daemon: the test drives it on demand over SSH
+# (RunShellScriptInsideApp). Start sshd and keep the container alive.
+
+/usr/sbin/sshd
+exec sleep infinity