diff --git a/evetest/broker/provider/qemu.go b/evetest/broker/provider/qemu.go index 5325c7e5af9..8faada23c72 100644 --- a/evetest/broker/provider/qemu.go +++ b/evetest/broker/provider/qemu.go @@ -1482,6 +1482,15 @@ func (dev *qemuDevice) buildArgs() []string { args := []string{ "-enable-kvm", "-machine", "q35", + // Let the chipset watchdog actually reset the guest. q35 brings the ICH9 + // LPC bridge and with it the iTCO watchdog EVE drives through + // /dev/watchdog, but the bridge only latches a status bit on timeout + // unless noreboot is cleared. Without these two a guest silently survives + // a watchdog it should have died on, which hides watchdog regressions and + // quietly disarms any test that injects one. eden configures its QEMU the + // same way. + "-global", "ICH9-LPC.noreboot=false", + "-watchdog-action", "reset", "-cpu", "host", "-smp", fmt.Sprintf("%d", dev.spec.CPUs), "-m", fmt.Sprintf("%d", dev.spec.MemoryBytes>>20), diff --git a/evetest/cmd/list-tests/main.go b/evetest/cmd/list-tests/main.go index 77b9f5c26d0..f7c51d08616 100644 --- a/evetest/cmd/list-tests/main.go +++ b/evetest/cmd/list-tests/main.go @@ -72,6 +72,7 @@ type pkgContext struct { constStrings map[string]string // const name → String() return value (for variant display) allConsts map[string]string // all simple const name → literal value (for variant display) varParams map[string]paramInfo + funcDecls map[string]*ast.FuncDecl // package-local func name → declaration } func main() { @@ -244,6 +245,17 @@ func buildPkgContext(files []*ast.File) pkgContext { constStrings: map[string]string{}, allConsts: map[string]string{}, varParams: map[string]paramInfo{}, + funcDecls: map[string]*ast.FuncDecl{}, + } + + for _, f := range files { + for _, decl := range f.Decls { + fd, ok := decl.(*ast.FuncDecl) + if !ok || fd.Name == nil || fd.Recv != nil { + continue + } + ctx.funcDecls[fd.Name.Name] = fd + } } // Pass 1: collect const values (package-level and function-local). @@ -428,26 +440,44 @@ func returnsParamDef(fd *ast.FuncDecl) bool { } // extractParams finds the evetest.DefineTestParameters call in fd and returns -// the resolved parameter definitions. +// the resolved parameter definitions. Calls to package-local functions are +// followed, so that a test which delegates its body to a shared helper — the +// shape two tests take when they differ only in a setup argument — still reports +// the parameters that helper defines. func extractParams( fd *ast.FuncDecl, ctx pkgContext, paramFuncs map[string]paramInfo) []paramInfo { + return extractParamsFrom(fd, ctx, paramFuncs, map[string]bool{}) +} + +// extractParamsFrom is extractParams over one function body; visited records the +// functions already descended into so that recursion terminates. +func extractParamsFrom(fd *ast.FuncDecl, ctx pkgContext, + paramFuncs map[string]paramInfo, visited map[string]bool) []paramInfo { + if fd.Body == nil || fd.Name == nil || visited[fd.Name.Name] { + return nil + } + visited[fd.Name.Name] = true var params []paramInfo ast.Inspect(fd.Body, func(n ast.Node) bool { call, ok := n.(*ast.CallExpr) if !ok { return true } - sel, ok := call.Fun.(*ast.SelectorExpr) - if !ok { - return true - } - pkg, ok := sel.X.(*ast.Ident) - if !ok || pkg.Name != "evetest" || sel.Sel.Name != "DefineTestParameters" { - return true - } - for _, arg := range call.Args { - if pi, ok := resolveParamArg(arg, ctx, paramFuncs); ok { - params = append(params, pi) + switch fun := call.Fun.(type) { + case *ast.SelectorExpr: + pkg, ok := fun.X.(*ast.Ident) + if !ok || pkg.Name != "evetest" || fun.Sel.Name != "DefineTestParameters" { + return true + } + for _, arg := range call.Args { + if pi, ok := resolveParamArg(arg, ctx, paramFuncs); ok { + params = append(params, pi) + } + } + case *ast.Ident: + if callee, ok := ctx.funcDecls[fun.Name]; ok { + params = append(params, + extractParamsFrom(callee, ctx, paramFuncs, visited)...) } } return true diff --git a/evetest/devconfig.go b/evetest/devconfig.go index 5a2a752c43d..0a9490ab1d1 100644 --- a/evetest/devconfig.go +++ b/evetest/devconfig.go @@ -886,9 +886,20 @@ type ApplicationInstanceConfig struct { UserData string NetworkAdapters []AppNetworkAdapter EnforceNetIntfOrder bool + // DataVolumes are additional blank (empty) volumes attached to the app beyond + // its image. Each has no content tree — EVE creates an empty disk of the given + // size (VCOT_BLANK) and mounts it at MountDir. + DataVolumes []DataVolumeConfig // Many more parameters can be configured; they will be added later as needed. } +// DataVolumeConfig describes one blank data volume attached to an application. +type DataVolumeConfig struct { + SizeBytes uint64 // volume size in bytes (Maxsizebytes) + MountDir string // mount point inside the app (default: /mnt/vol) + ReadOnly bool +} + func (config ApplicationInstanceConfig) toProto(th *TestHarness, devName string, appUUID, volumeUUID uuid.UUID) *eveconfig.AppInstanceConfig { vmConfig := &eveconfig.VmConfig{ @@ -2272,6 +2283,32 @@ func (dc *EdgeDeviceConfig) addApplicationWithUUIDs( contentTreeUUID, datastoreUUID, config.DisplayName) dc.ContentInfo = append(dc.ContentInfo, contentTree) dc.Datastores = append(dc.Datastores, dsConfig) + + // Attach any blank data volumes: a VCOT_BLANK Volume (no content tree or + // datastore) plus a VolumeRef mounting it into the app. appInstConfig is a + // pointer already stored in dc.Apps, so extending its VolumeRefList here is + // reflected in the stored config. + for i, dv := range config.DataVolumes { + dvUUID := dc.th.newUUID("application data volume") + mountDir := dv.MountDir + if mountDir == "" { + mountDir = fmt.Sprintf("/mnt/vol%d", i) + } + appInstConfig.VolumeRefList = append(appInstConfig.VolumeRefList, + &eveconfig.VolumeRef{ + Uuid: dvUUID.String(), + MountDir: mountDir, + }) + dc.Volumes = append(dc.Volumes, &eveconfig.Volume{ + Uuid: dvUUID.String(), + Origin: &eveconfig.VolumeContentOrigin{ + Type: eveconfig.VolumeContentOriginType_VCOT_BLANK, + }, + Maxsizebytes: int64(dv.SizeBytes), + Readonly: dv.ReadOnly, + DisplayName: fmt.Sprintf("%s-data%d", config.DisplayName, i), + }) + } } // UpdateApplication updates an existing application instance identified diff --git a/evetest/edgedevice.go b/evetest/edgedevice.go index 45ac71dffb7..0955700668d 100644 --- a/evetest/edgedevice.go +++ b/evetest/edgedevice.go @@ -38,6 +38,9 @@ type EdgeDevice struct { // Set of app UUIDs (strings) for which WaitUntilAppIsRunning is active. // Used by WatchAppInfo to suppress duplicate state logging. appsBeingWaited sync.Map + // Per-handle override of how long to wait for an upgrade; see + // SetUpgradeTimeout. Zero means use the package default. + upgradeTimeout time.Duration } // GetEdgeDevice returns a handle to an onboarded EdgeDevice identified by devName. @@ -483,7 +486,7 @@ func (d *EdgeDevice) waitForUpgrade(targetShortVersion string) { } defer unsub() - ctx, cancel := context.WithTimeout(d.th.ctx, eveUpgradeTimeout) + ctx, cancel := context.WithTimeout(d.th.ctx, d.upgradeWaitTimeout()) defer cancel() var lastLoggedState, lastLoggedStatus string @@ -626,6 +629,40 @@ func (d *EdgeDevice) HardReboot(waitUntilRebooted bool) { }) } +// ExpectAdditionalReboots tells the harness to expect n more device-initiated +// reboots beyond those it already accounts for (UpgradeEVE counts one reboot per +// upgrade). A cross-flavor boot-disk conversion that runs an offline shrink/grow +// reboots additional times; the number is known only to the test driving it (a +// plain kvm<->k conversion with no resize reboots differently than a shrink+grow), +// so the test declares it here to keep the teardown reboot-count check accurate. +func (d *EdgeDevice) ExpectAdditionalReboots(n int) { + for i := 0; i < n; i++ { + d.th.incExpectedRebootCount(d.devName) + } +} + +// SetUpgradeTimeout overrides how long UpgradeEVE waits for the device to come +// back running the target version, for this handle only. +// +// The default is sized for an ordinary base-OS upgrade: download, one reboot, +// done. A cross-flavor boot-disk conversion is a different animal — it also runs +// an offline shrink+grow across several reboots and then brings up a whole +// container-cluster stack — and it lands close enough to the default that a +// healthy conversion and an expired budget are decided by a couple of minutes of +// host load. A test driving one should raise the timeout, otherwise it reports a +// conversion that was still progressing as a failure. +func (d *EdgeDevice) SetUpgradeTimeout(timeout time.Duration) { + d.upgradeTimeout = timeout +} + +// upgradeWaitTimeout is the effective upgrade wait for this handle. +func (d *EdgeDevice) upgradeWaitTimeout() time.Duration { + if d.upgradeTimeout > 0 { + return d.upgradeTimeout + } + return eveUpgradeTimeout +} + // rebootAndWait executes triggerFn to initiate a device reboot and, if // wait is true, blocks until the device confirms the reboot by reporting // a ZInfoDevice.lastRebootTime strictly after the moment triggerFn was called. diff --git a/evetest/testapps/volverify/Dockerfile b/evetest/testapps/volverify/Dockerfile new file mode 100644 index 00000000000..957f55ddc00 --- /dev/null +++ b/evetest/testapps/volverify/Dockerfile @@ -0,0 +1,38 @@ +# Copyright (c) 2026 Zededa, Inc. +# SPDX-License-Identifier: Apache-2.0 + +ARG GOLANG_VERSION=1.25 + +# hadolint ignore=DL3029 +FROM --platform=$BUILDPLATFORM golang:${GOLANG_VERSION} AS build + +ARG TARGETOS +ARG TARGETARCH + +WORKDIR /app + +COPY go.mod ./ +RUN go mod download + +COPY . . +RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -o /volverify ./cmd/volverify + +FROM ubuntu:24.04 + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# hadolint ignore=DL3008 +RUN apt-get update && \ + apt-get install -y --no-install-recommends openssh-server e2fsprogs && \ + rm -rf /var/lib/apt/lists/* && \ + mkdir /run/sshd + +RUN echo 'root:testpassword' | chpasswd +RUN sed -i 's/#PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config && \ + sed -i 's/#PasswordAuthentication.*/PasswordAuthentication yes/' /etc/ssh/sshd_config +EXPOSE 22 + +COPY --from=build /volverify /usr/local/bin/volverify +COPY init.sh / + +CMD ["/bin/bash", "/init.sh"] diff --git a/evetest/testapps/volverify/Makefile b/evetest/testapps/volverify/Makefile new file mode 100644 index 00000000000..eacb99e9226 --- /dev/null +++ b/evetest/testapps/volverify/Makefile @@ -0,0 +1,16 @@ +# Copyright (c) 2026 Zededa, Inc. +# SPDX-License-Identifier: Apache-2.0 + +EVETEST_ORG ?= lfedge +IMAGE = $(EVETEST_ORG)/evetest-volverify +# Update VERSION whenever there is a change made to this app. +VERSION ?= 1.0 + +DOCKER_TARGET ?= load +DOCKER_PLATFORM ?= $(shell uname -s | tr '[A-Z]' '[a-z]')/$(subst aarch64,arm64,$(subst x86_64,amd64,$(shell uname -m))) + +build: + docker buildx build \ + --$(DOCKER_TARGET) \ + --platform $(DOCKER_PLATFORM) \ + -t $(IMAGE):$(VERSION) . diff --git a/evetest/testapps/volverify/README.md b/evetest/testapps/volverify/README.md new file mode 100644 index 00000000000..bb72c40acf8 --- /dev/null +++ b/evetest/testapps/volverify/README.md @@ -0,0 +1,49 @@ +# volverify — application-volume corruption verifier + +A test-app tool that writes a deterministic, self-verifying fill/delete pattern to +an application volume and later checks it, to detect corruption caused by a +watchdog-interrupted EVE-kvm→EVE-k offline filesystem shrink. It is the ground +truth the soak harness pairs with the resize fsck marker. + +Design: `~/notes/kvm-to-k-appvol-shrink-soak-design.md` (§4). Deployed inside the +evetest ubuntu app and driven over SSH via `RunShellScriptInsideApp`. + +## What it does + +- **Layer 1** — every 4 KiB block is `AES-CTR(key=derive(fileID), iv=blockIndex)` + plus a header carrying the *logical* identity `(fileID, blockIndex)` and CRCs. + The identity is logical (file offset ÷ block size), never physical disk + placement — placement changes by design when the shrink relocates the P3 tail, + and the verifier checks that each logical read still yields the identity's + bytes. Reproducible, incompressible, non-zero, so a zeroed/torn/misplaced block + is unambiguous. +- **Layer 2** — a `masterSeed`-seeded PRNG drives a deterministic + create/delete/mkdir/rmdir op stream; the writer fsyncs and advances a 2-slot + ping-pong committed-index every `--commit-every` ops. The verifier replays the + stream to the committed index and classifies each expected file: + `ok / present-corrupt / orphaned (in lost+found) / lost / resurrected`. + +## Usage + +```sh +volverify write --dir /mnt/data --seed 42 --ops 100000 # crash-safe, resumable +volverify verify --dir /mnt/data --seed 42 --ops 100000 # exits non-zero on any anomaly +``` + +Both invocations must use the same `--seed` and size flags. `write` is idempotent +across reboots (it resumes from the committed index). + +`verify --expect-committed ` supplies an off-volume floor on the committed op +index. The on-volume commit slots live on the same volume being shrunk, so fsck can +clear them along with the last files' data — which would make the verifier expect +nothing and mask the loss. Since the soak harness runs `write` to completion before +the shrink, it knows the true high-water mark and passes it here, so the last work is +still expected (and its loss flagged). + +## Build + +```sh +make build # docker image lfedge/evetest-volverify:1.0 +GOWORK=off go test ./... # unit tests (fault-injection classification) +sudo ./scripts/loopback-ext4-test.sh # on-fs fidelity check (real ext4 + e2fsck) +``` diff --git a/evetest/testapps/volverify/cmd/volverify/main.go b/evetest/testapps/volverify/cmd/volverify/main.go new file mode 100644 index 00000000000..c539f2e22a0 --- /dev/null +++ b/evetest/testapps/volverify/cmd/volverify/main.go @@ -0,0 +1,117 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Command volverify writes and later verifies a deterministic, self-describing +// fill/delete pattern on an application volume, to detect corruption caused by a +// watchdog-interrupted EVE-kvm→EVE-k offline filesystem shrink. +// +// It is deployed inside the evetest test app and driven over SSH: +// +// volverify write --dir /mnt/data --seed 42 --ops 100000 +// volverify verify --dir /mnt/data --seed 42 --ops 100000 +// +// write is crash-safe and resumable: run it repeatedly across reboots. verify +// exits non-zero when it finds any anomaly and prints a machine-readable summary. +package main + +import ( + "flag" + "fmt" + "os" + + "github.com/lf-edge/eve/evetest/testapps/volverify/internal/verify" +) + +func main() { + if len(os.Args) < 2 { + usage() + os.Exit(2) + } + cmd := os.Args[1] + fs := flag.NewFlagSet(cmd, flag.ExitOnError) + dir := fs.String("dir", "", "volume mount point to operate on (required)") + def := verify.DefaultConfig() + seed := fs.Uint64("seed", def.Seed, "master seed for the op stream") + ops := fs.Uint64("ops", def.Ops, "number of ops to apply / expect") + commitEvery := fs.Uint64("commit-every", def.CommitEvery, "fsync + commit cadence in ops") + blockSize := fs.Int("block-size", def.BlockSize, "on-disk block size in bytes") + dirFanout := fs.Int("dir-fanout", def.DirFanout, "per-level file-tree fan-out") + smallBlocks := fs.Int("small-blocks", def.SmallBlocks, "max blocks for a small file") + medBlocks := fs.Int("med-blocks", def.MedBlocks, "max blocks for a medium file") + maxBlocks := fs.Int("max-blocks", def.MaxBlocks, "max blocks for a large file") + expectCommitted := fs.Int64("expect-committed", def.ExpectCommitted, + "verify: floor on the committed op index (harness high-water mark); -1 = trust on-volume commit only") + _ = fs.Parse(os.Args[2:]) + + if *dir == "" { + fmt.Fprintln(os.Stderr, "error: --dir is required") + os.Exit(2) + } + cfg := verify.Config{ + Seed: *seed, + BlockSize: *blockSize, + Ops: *ops, + CommitEvery: *commitEvery, + DirFanout: *dirFanout, + SmallBlocks: *smallBlocks, + MedBlocks: *medBlocks, + MaxBlocks: *maxBlocks, + ExpectCommitted: *expectCommitted, + } + + switch cmd { + case "write": + w, err := verify.NewWriter(*dir, cfg) + if err != nil { + fatal(err) + } + committed, err := w.Run() + if err != nil { + fatal(err) + } + fmt.Printf("write: complete committed=%d\n", committed) + case "verify": + rep, err := verify.Verify(*dir, cfg) + if err != nil { + fatal(err) + } + fmt.Println(rep.String()) + for _, a := range rep.Anomalies { + fmt.Printf(" ANOMALY file=%d verdict=%s path=%s expBlocks=%d sizeMismatch=%v blocks=%v\n", + a.FileID, a.Verdict, a.Path, a.ExpectBlocks, a.SizeMismatch, blockCountsString(a.BlockCounts)) + } + for _, id := range rep.Resurrected { + fmt.Printf(" ANOMALY resurrected file=%d\n", id) + } + if !rep.Clean() { + os.Exit(1) + } + fmt.Println("verify: clean") + default: + usage() + os.Exit(2) + } +} + +func blockCountsString(m map[verify.BlockStatus]int) string { + out := "" + for s, n := range m { + if s == verify.BlockOK { + continue + } + out += fmt.Sprintf("%s=%d ", s, n) + } + if out == "" { + return "-" + } + return out +} + +func fatal(err error) { + fmt.Fprintln(os.Stderr, "error:", err) + os.Exit(1) +} + +func usage() { + fmt.Fprintln(os.Stderr, "usage: volverify --dir [--seed N --ops N ...]") +} diff --git a/evetest/testapps/volverify/go.mod b/evetest/testapps/volverify/go.mod new file mode 100644 index 00000000000..efbed6df6cd --- /dev/null +++ b/evetest/testapps/volverify/go.mod @@ -0,0 +1,3 @@ +module github.com/lf-edge/eve/evetest/testapps/volverify + +go 1.25 diff --git a/evetest/testapps/volverify/init.sh b/evetest/testapps/volverify/init.sh new file mode 100755 index 00000000000..9ed55971073 --- /dev/null +++ b/evetest/testapps/volverify/init.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# Copyright (c) 2026 Zededa, Inc. +# SPDX-License-Identifier: Apache-2.0 + +# The volverify binary is not a daemon: the test drives it on demand over SSH +# (RunShellScriptInsideApp). Start sshd and keep the container alive. + +/usr/sbin/sshd +exec sleep infinity diff --git a/evetest/testapps/volverify/internal/verify/block.go b/evetest/testapps/volverify/internal/verify/block.go new file mode 100644 index 00000000000..641071f2c39 --- /dev/null +++ b/evetest/testapps/volverify/internal/verify/block.go @@ -0,0 +1,188 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Package verify implements the two-layer self-verifying fill/delete pattern used +// to detect application-volume corruption caused by a watchdog-interrupted +// EVE-kvm→EVE-k offline filesystem shrink (design: kvm-to-k-appvol-shrink-soak). +// +// Layer 1 (this file) gives every on-disk block content that is a pure function +// of the file's *logical* identity — (fileID, logical block index within the +// file), never the physical disk placement. Physical placement changes by design +// when resize2fs relocates the P3 tail, so the verifier reads a file back through +// the filesystem and checks that each logical block still yields the bytes its +// identity implies. A block that reads back as another file's or another index's +// content (an extent tree torn during relocation) is caught by the identity in +// its header. +package verify + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/sha256" + "encoding/binary" + "hash/crc32" +) + +const ( + // blockMagic marks a well-formed block header ("VVB1" little-endian). + blockMagic = 0x31425656 + // headerLen is the fixed prefix each block reserves for its header. + headerLen = 32 +) + +var castagnoli = crc32.MakeTable(crc32.Castagnoli) + +// deriveKey returns the 16-byte AES key for a file's block content. It is a pure +// function of fileID, so a block's expected bytes depend only on file identity +// and logical index — never on stored state, so no content manifest is needed. +func deriveKey(fileID uint64) []byte { + var b [8]byte + binary.LittleEndian.PutUint64(b[:], fileID) + sum := sha256.Sum256(append([]byte("volverify-key-v1"), b[:]...)) + return sum[:16] +} + +// blockBody fills dst with the deterministic, incompressible, non-zero body for +// (fileID, blockIndex): an AES-CTR keystream over zeros. blockIndex occupies the +// high half of the IV so per-block counter ranges never overlap. +func blockBody(fileID, blockIndex uint64, dst []byte) { + blk, err := aes.NewCipher(deriveKey(fileID)) + if err != nil { + panic(err) // deriveKey always yields a valid 16-byte key + } + var iv [aes.BlockSize]byte + binary.BigEndian.PutUint64(iv[0:8], blockIndex) + ctr := cipher.NewCTR(blk, iv[:]) + for i := range dst { + dst[i] = 0 + } + ctr.XORKeyStream(dst, dst) +} + +// BuildBlock returns a blockSize-byte block for the logical (fileID, blockIndex). +func BuildBlock(fileID, blockIndex uint64, blockSize int) []byte { + buf := make([]byte, blockSize) + body := buf[headerLen:] + blockBody(fileID, blockIndex, body) + binary.LittleEndian.PutUint32(buf[0:4], blockMagic) + binary.LittleEndian.PutUint64(buf[4:12], fileID) + binary.LittleEndian.PutUint64(buf[12:20], blockIndex) + binary.LittleEndian.PutUint32(buf[20:24], uint32(len(body))) + binary.LittleEndian.PutUint32(buf[24:28], crc32.Checksum(body, castagnoli)) + binary.LittleEndian.PutUint32(buf[28:32], crc32.Checksum(buf[0:28], castagnoli)) + return buf +} + +// BlockStatus classifies one on-disk block against its expected logical identity. +type BlockStatus int + +const ( + // BlockOK means the block content matches the expected keystream. + BlockOK BlockStatus = iota + // BlockZeroed means the block is all zero — fsck cleared it or a torn/short write. + BlockZeroed + // BlockGarbage means there is no valid header (bad magic/header-CRC/length). + BlockGarbage + // BlockMisplaced means a valid header carrying another file's or index's identity. + BlockMisplaced + // BlockTorn means the header is intact but the body CRC disagrees with the body. + BlockTorn + // BlockAltered means the block is self-consistent but its body != expected keystream. + BlockAltered +) + +func (s BlockStatus) String() string { + switch s { + case BlockOK: + return "ok" + case BlockZeroed: + return "zeroed" + case BlockGarbage: + return "garbage" + case BlockMisplaced: + return "misplaced" + case BlockTorn: + return "torn" + case BlockAltered: + return "altered" + } + return "unknown" +} + +// BlockResult is a classified block plus, for a misplaced block, the logical +// identity its header actually carried (so cross-file smearing can be traced). +type BlockResult struct { + Status BlockStatus + FoundFile uint64 + FoundIndex uint64 +} + +// parseHeader extracts a block's declared logical identity. ok is false when the +// block has no usable header (bad magic, header-CRC mismatch, or wrong body length). +func parseHeader(buf []byte, blockSize int) (fileID, blockIndex uint64, ok bool) { + if len(buf) < headerLen { + return 0, 0, false + } + if binary.LittleEndian.Uint32(buf[0:4]) != blockMagic { + return 0, 0, false + } + if binary.LittleEndian.Uint32(buf[28:32]) != crc32.Checksum(buf[0:28], castagnoli) { + return 0, 0, false + } + if int(binary.LittleEndian.Uint32(buf[20:24])) != blockSize-headerLen { + return 0, 0, false + } + return binary.LittleEndian.Uint64(buf[4:12]), binary.LittleEndian.Uint64(buf[12:20]), true +} + +// VerifyBlock classifies buf against the block expected at logical (expFile, expIndex). +func VerifyBlock(buf []byte, expFile, expIndex uint64, blockSize int) BlockResult { + if isZero(buf) { + return BlockResult{Status: BlockZeroed} + } + fid, bidx, ok := parseHeader(buf, blockSize) + if !ok { + return BlockResult{Status: BlockGarbage} + } + if fid != expFile || bidx != expIndex { + return BlockResult{Status: BlockMisplaced, FoundFile: fid, FoundIndex: bidx} + } + body := buf[headerLen:] + if binary.LittleEndian.Uint32(buf[24:28]) != crc32.Checksum(body, castagnoli) { + return BlockResult{Status: BlockTorn} + } + exp := make([]byte, len(body)) + blockBody(expFile, expIndex, exp) + if !bytesEqual(exp, body) { + return BlockResult{Status: BlockAltered} + } + return BlockResult{Status: BlockOK} +} + +// headerFileID reports the file identity a block claims, for the lost+found scan +// (design §4.3): an orphaned file is identified by reading its first block header. +func headerFileID(buf []byte, blockSize int) (uint64, bool) { + fid, _, ok := parseHeader(buf, blockSize) + return fid, ok +} + +func isZero(b []byte) bool { + for _, x := range b { + if x != 0 { + return false + } + } + return true +} + +func bytesEqual(a, b []byte) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} diff --git a/evetest/testapps/volverify/internal/verify/block_test.go b/evetest/testapps/volverify/internal/verify/block_test.go new file mode 100644 index 00000000000..722570be064 --- /dev/null +++ b/evetest/testapps/volverify/internal/verify/block_test.go @@ -0,0 +1,68 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package verify + +import "testing" + +const testBlockSize = 256 + +func TestBlockRoundTrip(t *testing.T) { + b := BuildBlock(7, 3, testBlockSize) + if res := VerifyBlock(b, 7, 3, testBlockSize); res.Status != BlockOK { + t.Fatalf("roundtrip: got %v, want ok", res.Status) + } + // The body must be non-zero and reproducible. + b2 := BuildBlock(7, 3, testBlockSize) + if !bytesEqual(b, b2) { + t.Fatal("block content not reproducible for same identity") + } + if isZero(b[headerLen:]) { + t.Fatal("block body is all-zero") + } +} + +func TestBlockZeroed(t *testing.T) { + b := make([]byte, testBlockSize) + if res := VerifyBlock(b, 1, 0, testBlockSize); res.Status != BlockZeroed { + t.Fatalf("got %v, want zeroed", res.Status) + } +} + +func TestBlockGarbage(t *testing.T) { + b := BuildBlock(1, 0, testBlockSize) + b[1] ^= 0xff // corrupt the magic + if res := VerifyBlock(b, 1, 0, testBlockSize); res.Status != BlockGarbage { + t.Fatalf("got %v, want garbage", res.Status) + } +} + +func TestBlockMisplaced(t *testing.T) { + // A block written for file 9 read where file 1 is expected. + b := BuildBlock(9, 0, testBlockSize) + res := VerifyBlock(b, 1, 0, testBlockSize) + if res.Status != BlockMisplaced || res.FoundFile != 9 { + t.Fatalf("got %v foundFile=%d, want misplaced foundFile=9", res.Status, res.FoundFile) + } + // Same file, wrong index is also misplaced. + b2 := BuildBlock(1, 5, testBlockSize) + if res := VerifyBlock(b2, 1, 0, testBlockSize); res.Status != BlockMisplaced || res.FoundIndex != 5 { + t.Fatalf("got %v foundIndex=%d, want misplaced foundIndex=5", res.Status, res.FoundIndex) + } +} + +func TestBlockTorn(t *testing.T) { + b := BuildBlock(1, 0, testBlockSize) + // Flip a body byte without updating the body CRC: the header still parses. + b[headerLen+10] ^= 0xff + if res := VerifyBlock(b, 1, 0, testBlockSize); res.Status != BlockTorn { + t.Fatalf("got %v, want torn", res.Status) + } +} + +func TestBlockHeaderFileID(t *testing.T) { + b := BuildBlock(42, 0, testBlockSize) + if id, ok := headerFileID(b, testBlockSize); !ok || id != 42 { + t.Fatalf("got id=%d ok=%v, want 42 true", id, ok) + } +} diff --git a/evetest/testapps/volverify/internal/verify/commit.go b/evetest/testapps/volverify/internal/verify/commit.go new file mode 100644 index 00000000000..39d29c81fd1 --- /dev/null +++ b/evetest/testapps/volverify/internal/verify/commit.go @@ -0,0 +1,129 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package verify + +import ( + "encoding/binary" + "errors" + "fmt" + "hash/crc32" + "os" + "path/filepath" + "syscall" +) + +// commitDirName holds the two ping-pong committed-index slots on the volume. +const commitDirName = ".vv-commit" + +const ( + commitMagic = 0x56564349 // "VVCI" little-endian + commitRecLen = 24 // magic(4) + generation(8) + index(8) + crc(4) +) + +// commitRecord is the durable "everything through op Index is fsynced" marker. +// generation strictly increases so recovery can pick the newest valid slot even +// when a crash tore the most recent write (design §4.2). +type commitRecord struct { + generation uint64 + index int64 +} + +func slotPath(volDir string, slot int) string { + return filepath.Join(volDir, commitDirName, fmt.Sprintf("commit.%d", slot)) +} + +// writeCommit atomically publishes rec into the ping-pong slot chosen by its +// generation parity, fsyncing the file and its directory before returning. +func writeCommit(volDir string, rec commitRecord) error { + dir := filepath.Join(volDir, commitDirName) + if err := os.MkdirAll(dir, 0o755); err != nil { + return err + } + var buf [commitRecLen]byte + binary.LittleEndian.PutUint32(buf[0:4], commitMagic) + binary.LittleEndian.PutUint64(buf[4:12], rec.generation) + binary.LittleEndian.PutUint64(buf[12:20], uint64(rec.index)) + binary.LittleEndian.PutUint32(buf[20:24], crc32.Checksum(buf[0:20], castagnoli)) + + path := slotPath(volDir, int(rec.generation%2)) + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) + if err != nil { + return err + } + if _, err := f.Write(buf[:]); err != nil { + f.Close() + return err + } + if err := f.Sync(); err != nil { + f.Close() + return err + } + if err := f.Close(); err != nil { + return err + } + return fsyncDir(dir) +} + +// readCommit returns the highest-generation valid slot's index, or -1 when no +// valid slot exists (fresh volume, or both slots torn). +func readCommit(volDir string) int64 { + best := int64(-1) + bestGen := int64(-1) + for slot := 0; slot < 2; slot++ { + rec, ok := readSlot(slotPath(volDir, slot)) + if !ok { + continue + } + if int64(rec.generation) > bestGen { + bestGen = int64(rec.generation) + best = rec.index + } + } + return best +} + +func readSlot(path string) (commitRecord, bool) { + data, err := os.ReadFile(path) + if err != nil || len(data) != commitRecLen { + return commitRecord{}, false + } + if binary.LittleEndian.Uint32(data[0:4]) != commitMagic { + return commitRecord{}, false + } + if binary.LittleEndian.Uint32(data[20:24]) != crc32.Checksum(data[0:20], castagnoli) { + return commitRecord{}, false + } + return commitRecord{ + generation: binary.LittleEndian.Uint64(data[4:12]), + index: int64(binary.LittleEndian.Uint64(data[12:20])), + }, true +} + +// nextGeneration returns the generation to use for the next commit write. +func nextGeneration(volDir string) uint64 { + var maxGen int64 = -1 + for slot := 0; slot < 2; slot++ { + if rec, ok := readSlot(slotPath(volDir, slot)); ok { + if int64(rec.generation) > maxGen { + maxGen = int64(rec.generation) + } + } + } + return uint64(maxGen + 1) +} + +// fsyncDir flushes a directory entry so a rename/create is durable. +func fsyncDir(dir string) error { + d, err := os.Open(dir) + if err != nil { + return err + } + defer d.Close() + err = d.Sync() + // Some filesystems reject fsync on a directory; that is not fatal here. + if err != nil && (errors.Is(err, os.ErrInvalid) || errors.Is(err, syscall.EINVAL)) { + return nil + } + return err +} diff --git a/evetest/testapps/volverify/internal/verify/commit_test.go b/evetest/testapps/volverify/internal/verify/commit_test.go new file mode 100644 index 00000000000..2e2f22e68f7 --- /dev/null +++ b/evetest/testapps/volverify/internal/verify/commit_test.go @@ -0,0 +1,46 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package verify + +import ( + "os" + "testing" +) + +func TestCommitPingPong(t *testing.T) { + dir := t.TempDir() + if got := readCommit(dir); got != -1 { + t.Fatalf("fresh volume: got committed=%d, want -1", got) + } + for i := 0; i < 5; i++ { + gen := nextGeneration(dir) + if err := writeCommit(dir, commitRecord{generation: gen, index: int64(i * 10)}); err != nil { + t.Fatal(err) + } + } + if got := readCommit(dir); got != 40 { + t.Fatalf("got committed=%d, want 40", got) + } +} + +func TestCommitTornNewestSlotFallsBack(t *testing.T) { + dir := t.TempDir() + // generation 0 -> slot 0 (index 100), generation 1 -> slot 1 (index 200). + if err := writeCommit(dir, commitRecord{generation: 0, index: 100}); err != nil { + t.Fatal(err) + } + if err := writeCommit(dir, commitRecord{generation: 1, index: 200}); err != nil { + t.Fatal(err) + } + if got := readCommit(dir); got != 200 { + t.Fatalf("got %d, want 200", got) + } + // Tear the newest slot (slot 1); recovery must fall back to slot 0. + if err := os.WriteFile(slotPath(dir, 1), []byte("torn"), 0o644); err != nil { + t.Fatal(err) + } + if got := readCommit(dir); got != 100 { + t.Fatalf("after tearing newest slot: got %d, want 100", got) + } +} diff --git a/evetest/testapps/volverify/internal/verify/config.go b/evetest/testapps/volverify/internal/verify/config.go new file mode 100644 index 00000000000..e2544373c38 --- /dev/null +++ b/evetest/testapps/volverify/internal/verify/config.go @@ -0,0 +1,54 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package verify + +// Config parameterizes an op stream. The same Config (and Seed) must be supplied +// to the writer and to the later verifier, since both replay the identical stream. +type Config struct { + Seed uint64 // master seed for the deterministic op stream + BlockSize int // on-disk block size in bytes (must be > headerLen) + Ops uint64 // total number of ops the writer applies + CommitEvery uint64 // fsync + advance the committed index every this many ops + DirFanout int // per-level fan-out of the file placement tree + SmallBlocks int // upper bound (inclusive) on a "small" file, in blocks + MedBlocks int // upper bound on a "medium" file, in blocks + MaxBlocks int // upper bound on a "large" file, in blocks + + // ExpectCommitted is an off-volume floor on the committed op index for the + // verifier: the effective committed index is max(on-volume commit, + // ExpectCommitted). It closes the blind spot where fsck clears both the last + // files' data and the on-volume commit slots — the harness, which ran the + // writer to completion, supplies the true high-water mark so the verifier + // still expects (and thus flags loss of) the last work. -1 means "trust the + // on-volume commit only". Authoritative only when the caller knows the writer + // finished; a value above reality would over-expect. Ignored by the writer. + ExpectCommitted int64 +} + +// DefaultConfig returns a Config sized for on-device churn against a large blank +// volume: 4 KiB blocks and a heavy-tailed size mix (mostly KB, occasional +// MB/hundreds-MB) so allocated blocks scatter through the shrink evacuation zone. +func DefaultConfig() Config { + return Config{ + Seed: 1, + BlockSize: 4096, + Ops: 100000, + CommitEvery: 64, + DirFanout: 16, + SmallBlocks: 4, // <= 16 KiB + MedBlocks: 256, // <= 1 MiB + MaxBlocks: 65536, // <= 256 MiB + ExpectCommitted: -1, + } +} + +// valid reports whether the Config is self-consistent enough to run. +func (c Config) valid() bool { + return c.BlockSize > headerLen && + c.CommitEvery > 0 && + c.DirFanout > 0 && + c.SmallBlocks >= 1 && + c.MedBlocks >= c.SmallBlocks && + c.MaxBlocks >= c.MedBlocks +} diff --git a/evetest/testapps/volverify/internal/verify/engine.go b/evetest/testapps/volverify/internal/verify/engine.go new file mode 100644 index 00000000000..c6b13c9d472 --- /dev/null +++ b/evetest/testapps/volverify/internal/verify/engine.go @@ -0,0 +1,311 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package verify + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + "syscall" +) + +// Writer applies the deterministic op stream to a volume, fsyncing and advancing +// the committed index every Config.CommitEvery ops so that, after an abrupt power +// loss, everything through the last committed index is durable (design §4.2). +type Writer struct { + volDir string + cfg Config +} + +// NewWriter returns a Writer that operates on the volume mounted at volDir. +func NewWriter(volDir string, cfg Config) (*Writer, error) { + if !cfg.valid() { + return nil, fmt.Errorf("invalid config: %+v", cfg) + } + return &Writer{volDir: volDir, cfg: cfg}, nil +} + +// Run applies ops until Config.Ops (or until the volume fills), resuming after a +// crash from the committed index. It is safe to call repeatedly across reboots on +// the same volume. It returns the final committed op index — the high-water mark +// the verifier should expect (equal to Ops-1 on full completion, or lower when the +// volume filled first). Filling the volume is the expected end state for the +// corruption soak, so ENOSPC is a clean stop, not an error. +func (w *Writer) Run() (int64, error) { + gen := newGenerator(w.cfg) + m := newModel(w.cfg) + + // Rebuild in-memory state up to the committed index without touching disk; + // this also advances the generator to the first uncommitted op. + committed := readCommit(w.volDir) + for n := uint64(0); int64(n) <= committed; n++ { + m.apply(gen.next(m, n)) + } + gen64 := nextGeneration(w.volDir) + lastCommitted := committed + + touchedFiles := make(map[string]bool) + touchedDirs := make(map[string]bool) + + commit := func(index uint64) error { + for p := range touchedFiles { + if err := fsyncFile(p); err != nil { + return err + } + } + for d := range touchedDirs { + if err := fsyncDir(d); err != nil { + return err + } + } + if err := writeCommit(w.volDir, commitRecord{generation: gen64, index: int64(index)}); err != nil { + return err + } + gen64++ + lastCommitted = int64(index) + touchedFiles = make(map[string]bool) + touchedDirs = make(map[string]bool) + return nil + } + + start := uint64(committed + 1) + for n := start; n < w.cfg.Ops; n++ { + o := gen.next(m, n) + if err := w.applyOp(o, touchedFiles, touchedDirs); err != nil { + if errors.Is(err, syscall.ENOSPC) { + // Volume full: stop cleanly at the last fully-written op. Drop the + // partial file (it is an uncommitted op the verifier ignores) and + // commit everything through n-1, then report that index. + if o.typ == opCreate { + _ = os.Remove(filepath.Join(w.volDir, filePathFor(w.cfg, o.fileID))) + } + if n > start { + if err := commit(n - 1); err != nil { + return lastCommitted, err + } + } + return lastCommitted, nil + } + return lastCommitted, fmt.Errorf("op %d (%v): %w", n, o.typ, err) + } + m.apply(o) + if (n+1)%w.cfg.CommitEvery == 0 { + if err := commit(n); err != nil { + return lastCommitted, err + } + } + } + if w.cfg.Ops > start { + if err := commit(w.cfg.Ops - 1); err != nil { + return lastCommitted, err + } + } + return lastCommitted, nil +} + +// applyOp performs one op against the volume and records what it touched so the +// next commit can fsync it. +func (w *Writer) applyOp(o op, touchedFiles, touchedDirs map[string]bool) error { + switch o.typ { + case opCreate: + rel := filePathFor(w.cfg, o.fileID) + path := filepath.Join(w.volDir, rel) + parent := filepath.Dir(path) + if err := os.MkdirAll(parent, 0o755); err != nil { + return err + } + if err := w.writeFile(path, o.fileID, o.nblocks); err != nil { + return err + } + touchedFiles[path] = true + touchedDirs[parent] = true + case opDelete: + rel := filePathFor(w.cfg, o.fileID) + path := filepath.Join(w.volDir, rel) + if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + return err + } + touchedDirs[filepath.Dir(path)] = true + case opMkdir: + path := filepath.Join(w.volDir, o.dir) + if err := os.MkdirAll(path, 0o755); err != nil { + return err + } + touchedDirs[filepath.Dir(path)] = true + case opRmdir: + path := filepath.Join(w.volDir, o.dir) + if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + // A non-empty scratch dir cannot be removed; leave it in place. + if !isNotEmpty(err) { + return err + } + } + touchedDirs[filepath.Dir(path)] = true + } + return nil +} + +// writeFile writes nblocks Layer-1 blocks for fileID, replacing any prior content. +func (w *Writer) writeFile(path string, fileID uint64, nblocks int) error { + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) + if err != nil { + return err + } + defer f.Close() + for i := 0; i < nblocks; i++ { + if _, err := f.Write(BuildBlock(fileID, uint64(i), w.cfg.BlockSize)); err != nil { + return err + } + } + return nil +} + +// Verify reconstructs the expected committed state and checks the on-disk tree +// against it, classifying every expected file (design §4.2, §4.3). +func Verify(volDir string, cfg Config) (Report, error) { + if !cfg.valid() { + return Report{}, fmt.Errorf("invalid config: %+v", cfg) + } + gen := newGenerator(cfg) + m := newModel(cfg) + committed := readCommit(volDir) + if cfg.ExpectCommitted > committed { + // Trust the harness-supplied high-water mark over on-volume bookkeeping + // that fsck may have cleared alongside the data (see Config.ExpectCommitted). + committed = cfg.ExpectCommitted + } + for n := uint64(0); int64(n) <= committed; n++ { + m.apply(gen.next(m, n)) + } + + orphans := scanLostFound(volDir, cfg.BlockSize) + + rep := Report{ + CommittedIndex: committed, + LiveExpected: len(m.liveIDs), + DeletedExpected: len(m.deleted), + } + + for _, id := range m.liveIDs { + meta := m.files[id] + rel := m.filePath(id) + path := filepath.Join(volDir, rel) + anom, ok := checkFile(path, id, meta.nblocks, cfg.BlockSize) + if ok { + rep.FilesOK++ + continue + } + if anom.Verdict == FileLost && orphans[id] { + anom.Verdict = FileOrphaned + } + anom.FileID = id + anom.Path = rel + anom.ExpectBlocks = meta.nblocks + rep.Anomalies = append(rep.Anomalies, anom) + } + + for id := range m.deleted { + path := filepath.Join(volDir, m.filePath(id)) + if _, err := os.Stat(path); err == nil { + rep.Resurrected = append(rep.Resurrected, id) + } + } + return rep, nil +} + +// checkFile classifies one expected-live file. ok is true only for a fully-clean +// file (present, correct size, every block OK). +func checkFile(path string, fileID uint64, nblocks, blockSize int) (FileAnomaly, bool) { + f, err := os.Open(path) + if err != nil { + if os.IsNotExist(err) { + return FileAnomaly{Verdict: FileLost}, false + } + return FileAnomaly{Verdict: FileLost}, false + } + defer f.Close() + + anom := FileAnomaly{Verdict: FilePresentCorrupt, BlockCounts: make(map[BlockStatus]int)} + if fi, err := f.Stat(); err == nil { + if fi.Size() != int64(nblocks)*int64(blockSize) { + anom.SizeMismatch = true + } + } + + buf := make([]byte, blockSize) + allOK := !anom.SizeMismatch + for i := 0; i < nblocks; i++ { + n, err := io.ReadFull(f, buf) + if err == io.EOF || err == io.ErrUnexpectedEOF { + // Fewer blocks than expected: treat the missing tail as zeroed. + for ; n < blockSize; n++ { + buf[n] = 0 + } + res := VerifyBlock(buf, fileID, uint64(i), blockSize) + anom.BlockCounts[res.Status]++ + allOK = false + continue + } + if err != nil { + anom.BlockCounts[BlockGarbage]++ + allOK = false + continue + } + res := VerifyBlock(buf, fileID, uint64(i), blockSize) + anom.BlockCounts[res.Status]++ + if res.Status != BlockOK { + allOK = false + } + } + if allOK { + return FileAnomaly{}, true + } + return anom, false +} + +// scanLostFound returns the set of fileIDs recoverable from a lost+found +// directory at the volume root, identified by reading each entry's first block +// header (design §4.3). It is best-effort: a missing or unreadable lost+found +// yields an empty set. +func scanLostFound(volDir string, blockSize int) map[uint64]bool { + out := make(map[uint64]bool) + lf := filepath.Join(volDir, "lost+found") + entries, err := os.ReadDir(lf) + if err != nil { + return out + } + buf := make([]byte, blockSize) + for _, e := range entries { + if e.IsDir() { + continue + } + f, err := os.Open(filepath.Join(lf, e.Name())) + if err != nil { + continue + } + n, _ := io.ReadFull(f, buf) + f.Close() + if n < blockSize { + continue + } + if id, ok := headerFileID(buf, blockSize); ok { + out[id] = true + } + } + return out +} + +func fsyncFile(path string) error { + f, err := os.Open(path) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + defer f.Close() + return f.Sync() +} diff --git a/evetest/testapps/volverify/internal/verify/engine_test.go b/evetest/testapps/volverify/internal/verify/engine_test.go new file mode 100644 index 00000000000..b0cf29550a9 --- /dev/null +++ b/evetest/testapps/volverify/internal/verify/engine_test.go @@ -0,0 +1,233 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package verify + +import ( + "os" + "path/filepath" + "testing" +) + +// committedModel rebuilds the expected state up to the committed index, mirroring +// what Verify does, so a test can locate specific live/deleted files to corrupt. +func committedModel(dir string, cfg Config) *model { + gen := newGenerator(cfg) + m := newModel(cfg) + c := readCommit(dir) + for n := uint64(0); int64(n) <= c; n++ { + m.apply(gen.next(m, n)) + } + return m +} + +func writeVolume(t *testing.T, cfg Config) (string, *model) { + t.Helper() + dir := t.TempDir() + w, err := NewWriter(dir, cfg) + if err != nil { + t.Fatal(err) + } + if _, err := w.Run(); err != nil { + t.Fatal(err) + } + return dir, committedModel(dir, cfg) +} + +func aLiveFile(t *testing.T, m *model) (uint64, int) { + t.Helper() + if len(m.liveIDs) == 0 { + t.Fatal("no live files after write") + } + id := m.liveIDs[0] + return id, m.files[id].nblocks +} + +func TestWriteVerifyClean(t *testing.T) { + cfg := testConfig() + dir, m := writeVolume(t, cfg) + rep, err := Verify(dir, cfg) + if err != nil { + t.Fatal(err) + } + if !rep.Clean() { + t.Fatalf("expected clean, got %s\n%+v", rep, rep.Anomalies) + } + if rep.FilesOK != len(m.liveIDs) { + t.Fatalf("FilesOK=%d, want %d", rep.FilesOK, len(m.liveIDs)) + } +} + +func TestTruncateDetected(t *testing.T) { + cfg := testConfig() + dir, m := writeVolume(t, cfg) + id, _ := aLiveFile(t, m) + if err := os.Truncate(filepath.Join(dir, filePathFor(cfg, id)), 0); err != nil { + t.Fatal(err) + } + rep, _ := Verify(dir, cfg) + a := findAnomaly(rep, id) + if a == nil || a.Verdict != FilePresentCorrupt || !a.SizeMismatch { + t.Fatalf("truncate not detected as present-corrupt: %+v", a) + } +} + +func TestZeroBlockDetected(t *testing.T) { + cfg := testConfig() + dir, m := writeVolume(t, cfg) + id, _ := aLiveFile(t, m) + f, err := os.OpenFile(filepath.Join(dir, filePathFor(cfg, id)), os.O_WRONLY, 0) + if err != nil { + t.Fatal(err) + } + if _, err := f.WriteAt(make([]byte, cfg.BlockSize), 0); err != nil { + t.Fatal(err) + } + f.Close() + rep, _ := Verify(dir, cfg) + a := findAnomaly(rep, id) + if a == nil || a.Verdict != FilePresentCorrupt || a.BlockCounts[BlockZeroed] == 0 { + t.Fatalf("zeroed block not detected: %+v", a) + } +} + +func TestMisplacedDetected(t *testing.T) { + cfg := testConfig() + dir, m := writeVolume(t, cfg) + id, nblocks := aLiveFile(t, m) + other := id + 100000 // an identity distinct from every live/deleted file + f, err := os.OpenFile(filepath.Join(dir, filePathFor(cfg, id)), os.O_WRONLY|os.O_TRUNC, 0o644) + if err != nil { + t.Fatal(err) + } + for i := 0; i < nblocks; i++ { + if _, err := f.Write(BuildBlock(other, uint64(i), cfg.BlockSize)); err != nil { + t.Fatal(err) + } + } + f.Close() + rep, _ := Verify(dir, cfg) + a := findAnomaly(rep, id) + if a == nil || a.BlockCounts[BlockMisplaced] == 0 { + t.Fatalf("misplaced data not detected: %+v", a) + } +} + +func TestLostThenOrphaned(t *testing.T) { + cfg := testConfig() + dir, m := writeVolume(t, cfg) + id, _ := aLiveFile(t, m) + if err := os.Remove(filepath.Join(dir, filePathFor(cfg, id))); err != nil { + t.Fatal(err) + } + rep, _ := Verify(dir, cfg) + if a := findAnomaly(rep, id); a == nil || a.Verdict != FileLost { + t.Fatalf("removed file not reported lost: %+v", a) + } + // Now place the file in lost+found (identified by its first block header). + lf := filepath.Join(dir, "lost+found") + if err := os.MkdirAll(lf, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(lf, "#131074"), BuildBlock(id, 0, cfg.BlockSize), 0o644); err != nil { + t.Fatal(err) + } + rep, _ = Verify(dir, cfg) + if a := findAnomaly(rep, id); a == nil || a.Verdict != FileOrphaned { + t.Fatalf("lost+found file not reported orphaned: %+v", a) + } +} + +func TestResurrectedDetected(t *testing.T) { + cfg := testConfig() + dir, m := writeVolume(t, cfg) + var delID uint64 + found := false + for id := range m.deleted { + delID = id + found = true + break + } + if !found { + t.Skip("no deleted files in this op stream") + } + path := filepath.Join(dir, filePathFor(cfg, delID)) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, BuildBlock(delID, 0, cfg.BlockSize), 0o644); err != nil { + t.Fatal(err) + } + rep, _ := Verify(dir, cfg) + if len(rep.Resurrected) == 0 { + t.Fatal("resurrected deleted file not detected") + } +} + +func TestCrashResumeIdempotent(t *testing.T) { + cfg := testConfig() + dir, _ := writeVolume(t, cfg) + // Running the writer again must resume from the commit and leave the volume + // clean (idempotent re-application of already-committed ops). + w, _ := NewWriter(dir, cfg) + if _, err := w.Run(); err != nil { + t.Fatal(err) + } + rep, _ := Verify(dir, cfg) + if !rep.Clean() { + t.Fatalf("resume left volume dirty: %s", rep) + } +} + +func TestInFlightExtraFileTolerated(t *testing.T) { + cfg := testConfig() + dir, _ := writeVolume(t, cfg) + // A file whose id was never part of the committed op stream stands in for an + // in-flight (op > committed index) create. It is outside the expected set, so + // the verifier must tolerate it rather than flag it (design §4.2). + extra := cfg.Ops + 5 + path := filepath.Join(dir, filePathFor(cfg, extra)) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, BuildBlock(extra, 0, cfg.BlockSize), 0o644); err != nil { + t.Fatal(err) + } + rep, _ := Verify(dir, cfg) + if !rep.Clean() { + t.Fatalf("in-flight extra file should be tolerated, got %s\n%+v", rep, rep.Anomalies) + } +} + +func TestExpectCommittedSurvivesMetadataLoss(t *testing.T) { + cfg := testConfig() + dir, m := writeVolume(t, cfg) + id, _ := aLiveFile(t, m) + // Simulate fsck clearing both the on-volume commit metadata and a last file. + if err := os.RemoveAll(filepath.Join(dir, commitDirName)); err != nil { + t.Fatal(err) + } + if err := os.Remove(filepath.Join(dir, filePathFor(cfg, id))); err != nil { + t.Fatal(err) + } + // Without the floor the verifier reads no commit, expects nothing, and the + // loss is masked. + if rep, _ := Verify(dir, cfg); !rep.Clean() { + t.Fatalf("baseline: expected masked-clean without floor, got %s", rep) + } + // With the harness high-water mark as a floor, the loss is caught. + cfg.ExpectCommitted = int64(cfg.Ops - 1) + rep, _ := Verify(dir, cfg) + if a := findAnomaly(rep, id); a == nil || a.Verdict != FileLost { + t.Fatalf("floor: loss after metadata clearing not caught: %+v", a) + } +} + +func findAnomaly(rep Report, id uint64) *FileAnomaly { + for i := range rep.Anomalies { + if rep.Anomalies[i].FileID == id { + return &rep.Anomalies[i] + } + } + return nil +} diff --git a/evetest/testapps/volverify/internal/verify/model.go b/evetest/testapps/volverify/internal/verify/model.go new file mode 100644 index 00000000000..38132b2cf6f --- /dev/null +++ b/evetest/testapps/volverify/internal/verify/model.go @@ -0,0 +1,103 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package verify + +import ( + "sort" +) + +// fileMeta records a live file's placement and logical length (in blocks). +type fileMeta struct { + nblocks int +} + +// model is the expected filesystem state reconstructed by replaying the op +// stream. The writer advances it as it applies ops; the verifier rebuilds it up +// to the committed index and then checks the on-disk tree against it. +// +// liveIDs and dirList are kept sorted so that op selection (which live file to +// delete, which scratch dir to remove) is a pure function of the PRNG draw and +// therefore identical in the writer and the verifier — Go map iteration order is +// randomized and must never drive a selection. +type model struct { + cfg Config + files map[uint64]fileMeta + liveIDs []uint64 + deleted map[uint64]bool + dirs map[string]bool + dirList []string +} + +func newModel(cfg Config) *model { + return &model{ + cfg: cfg, + files: make(map[uint64]fileMeta), + deleted: make(map[uint64]bool), + dirs: make(map[string]bool), + } +} + +// filePath returns the volume-relative path of a file (see filePathFor). +func (m *model) filePath(fileID uint64) string { + return filePathFor(m.cfg, fileID) +} + +func (m *model) addFile(fileID uint64, nblocks int) { + if _, ok := m.files[fileID]; !ok { + i := sort.Search(len(m.liveIDs), func(i int) bool { return m.liveIDs[i] >= fileID }) + m.liveIDs = append(m.liveIDs, 0) + copy(m.liveIDs[i+1:], m.liveIDs[i:]) + m.liveIDs[i] = fileID + } + m.files[fileID] = fileMeta{nblocks: nblocks} + delete(m.deleted, fileID) +} + +func (m *model) removeFile(fileID uint64) { + if _, ok := m.files[fileID]; !ok { + return + } + delete(m.files, fileID) + i := sort.Search(len(m.liveIDs), func(i int) bool { return m.liveIDs[i] >= fileID }) + if i < len(m.liveIDs) && m.liveIDs[i] == fileID { + m.liveIDs = append(m.liveIDs[:i], m.liveIDs[i+1:]...) + } + m.deleted[fileID] = true +} + +func (m *model) addDir(dir string) { + if m.dirs[dir] { + return + } + m.dirs[dir] = true + i := sort.SearchStrings(m.dirList, dir) + m.dirList = append(m.dirList, "") + copy(m.dirList[i+1:], m.dirList[i:]) + m.dirList[i] = dir +} + +func (m *model) removeDir(dir string) { + if !m.dirs[dir] { + return + } + delete(m.dirs, dir) + i := sort.SearchStrings(m.dirList, dir) + if i < len(m.dirList) && m.dirList[i] == dir { + m.dirList = append(m.dirList[:i], m.dirList[i+1:]...) + } +} + +// apply advances the model by one op. +func (m *model) apply(o op) { + switch o.typ { + case opCreate: + m.addFile(o.fileID, o.nblocks) + case opDelete: + m.removeFile(o.fileID) + case opMkdir: + m.addDir(o.dir) + case opRmdir: + m.removeDir(o.dir) + } +} diff --git a/evetest/testapps/volverify/internal/verify/opstream.go b/evetest/testapps/volverify/internal/verify/opstream.go new file mode 100644 index 00000000000..951e944276c --- /dev/null +++ b/evetest/testapps/volverify/internal/verify/opstream.go @@ -0,0 +1,81 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package verify + +import "fmt" + +// opType is the kind of filesystem mutation an op performs. +type opType int + +const ( + opCreate opType = iota + opDelete + opMkdir + opRmdir +) + +// op is one entry in the deterministic op stream (design §4.2). fileID equals the +// op index for a create. +type op struct { + n uint64 + typ opType + fileID uint64 + nblocks int + dir string +} + +// generator produces the op stream. Given the same seed it yields the same ops, +// provided it is driven against a model advanced by those same ops — the two +// selection ops (delete/rmdir) consult the model, so the writer and verifier must +// both replay from op 0 to stay in lockstep. +type generator struct { + rng *prng + cfg Config +} + +func newGenerator(cfg Config) *generator { + return &generator{rng: newPRNG(cfg.Seed), cfg: cfg} +} + +// scratchDir names one of a bounded set of churn directories, kept disjoint from +// the file-placement tree so mkdir/rmdir churn never removes a live file's parent. +func scratchDir(k int) string { + return fmt.Sprintf("scratch/s%03d", k) +} + +// pickNBlocks draws a file length from a heavy-tailed distribution: mostly small, +// occasionally medium, rarely large. +func (g *generator) pickNBlocks() int { + r := g.rng.next() % 1000 + switch { + case r < 850: + return 1 + g.rng.intn(g.cfg.SmallBlocks) + case r < 990: + return 1 + g.rng.intn(g.cfg.MedBlocks) + default: + return 1 + g.rng.intn(g.cfg.MaxBlocks) + } +} + +// next returns op n for the current model state and advances the generator. +func (g *generator) next(m *model, n uint64) op { + choice := g.rng.next() % 100 + switch { + case choice < 65: + return op{n: n, typ: opCreate, fileID: n, nblocks: g.pickNBlocks()} + case choice < 85: + if len(m.liveIDs) == 0 { + return op{n: n, typ: opCreate, fileID: n, nblocks: g.pickNBlocks()} + } + id := m.liveIDs[g.rng.intn(len(m.liveIDs))] + return op{n: n, typ: opDelete, fileID: id} + case choice < 93: + return op{n: n, typ: opMkdir, dir: scratchDir(g.rng.intn(g.cfg.DirFanout * g.cfg.DirFanout))} + default: + if len(m.dirList) == 0 { + return op{n: n, typ: opCreate, fileID: n, nblocks: g.pickNBlocks()} + } + return op{n: n, typ: opRmdir, dir: m.dirList[g.rng.intn(len(m.dirList))]} + } +} diff --git a/evetest/testapps/volverify/internal/verify/opstream_test.go b/evetest/testapps/volverify/internal/verify/opstream_test.go new file mode 100644 index 00000000000..1690ebca13d --- /dev/null +++ b/evetest/testapps/volverify/internal/verify/opstream_test.go @@ -0,0 +1,75 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package verify + +import "testing" + +func testConfig() Config { + return Config{ + Seed: 1234, + BlockSize: testBlockSize, + Ops: 300, + CommitEvery: 8, + DirFanout: 4, + SmallBlocks: 2, + MedBlocks: 4, + MaxBlocks: 8, + ExpectCommitted: -1, + } +} + +// replayOps drives the generator against a fresh model, returning the full stream. +func replayOps(cfg Config, n uint64) []op { + gen := newGenerator(cfg) + m := newModel(cfg) + ops := make([]op, 0, n) + for i := uint64(0); i < n; i++ { + o := gen.next(m, i) + ops = append(ops, o) + m.apply(o) + } + return ops +} + +func TestOpStreamDeterministic(t *testing.T) { + cfg := testConfig() + a := replayOps(cfg, cfg.Ops) + b := replayOps(cfg, cfg.Ops) + if len(a) != len(b) { + t.Fatalf("length mismatch %d vs %d", len(a), len(b)) + } + for i := range a { + if a[i] != b[i] { + t.Fatalf("op %d differs: %+v vs %+v", i, a[i], b[i]) + } + } +} + +func TestOpStreamSeedSensitive(t *testing.T) { + cfg := testConfig() + a := replayOps(cfg, cfg.Ops) + cfg.Seed++ + b := replayOps(cfg, cfg.Ops) + same := true + for i := range a { + if a[i] != b[i] { + same = false + break + } + } + if same { + t.Fatal("different seeds produced identical op streams") + } +} + +func TestOpStreamHasMix(t *testing.T) { + cfg := testConfig() + var counts [4]int + for _, o := range replayOps(cfg, cfg.Ops) { + counts[o.typ]++ + } + if counts[opCreate] == 0 || counts[opDelete] == 0 { + t.Fatalf("expected creates and deletes, got %+v", counts) + } +} diff --git a/evetest/testapps/volverify/internal/verify/report.go b/evetest/testapps/volverify/internal/verify/report.go new file mode 100644 index 00000000000..bb476dab855 --- /dev/null +++ b/evetest/testapps/volverify/internal/verify/report.go @@ -0,0 +1,84 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package verify + +import "fmt" + +// FileVerdict is the outcome for one expected-live file. +type FileVerdict int + +const ( + // FileOK means the file is present, correctly sized, and every block verified. + FileOK FileVerdict = iota + // FilePresentCorrupt means the file is present but has bad/misplaced/short + // blocks — the dangerous case EVE serves as-is (design §2.4, §4.3). + FilePresentCorrupt + // FileOrphaned means the file is missing from its path but was recovered in + // lost+found — self-heals to a blank/content-tree recreate (safe). + FileOrphaned + // FileLost means the file is missing and not in lost+found — data loss. + FileLost +) + +func (v FileVerdict) String() string { + switch v { + case FileOK: + return "ok" + case FilePresentCorrupt: + return "present-corrupt" + case FileOrphaned: + return "orphaned" + case FileLost: + return "lost" + } + return "unknown" +} + +// FileAnomaly describes one non-OK expected file. +type FileAnomaly struct { + FileID uint64 + Path string + Verdict FileVerdict + ExpectBlocks int + // BlockCounts tallies each block status seen in a present-corrupt file. + BlockCounts map[BlockStatus]int + // SizeMismatch is set when the on-disk size differs from expected. + SizeMismatch bool +} + +// Report summarizes a verification pass. It is the volume-content ground truth +// the soak harness pairs with the resize fsck marker. +type Report struct { + CommittedIndex int64 + LiveExpected int + DeletedExpected int + FilesOK int + Anomalies []FileAnomaly + // Resurrected lists committed-deleted files that reappeared (e.g. from + // lost+found) — an anomaly in its own right. + Resurrected []uint64 +} + +// Clean reports whether the pass found no anomalies of any kind. +func (r Report) Clean() bool { + return len(r.Anomalies) == 0 && len(r.Resurrected) == 0 +} + +// Counts returns the number of anomalies of each verdict. +func (r Report) Counts() map[FileVerdict]int { + c := make(map[FileVerdict]int) + for _, a := range r.Anomalies { + c[a.Verdict]++ + } + return c +} + +// String renders a compact human-readable summary. +func (r Report) String() string { + c := r.Counts() + return fmt.Sprintf( + "committed=%d live=%d deleted=%d ok=%d present-corrupt=%d orphaned=%d lost=%d resurrected=%d", + r.CommittedIndex, r.LiveExpected, r.DeletedExpected, r.FilesOK, + c[FilePresentCorrupt], c[FileOrphaned], c[FileLost], len(r.Resurrected)) +} diff --git a/evetest/testapps/volverify/internal/verify/rng.go b/evetest/testapps/volverify/internal/verify/rng.go new file mode 100644 index 00000000000..68fdaf9d004 --- /dev/null +++ b/evetest/testapps/volverify/internal/verify/rng.go @@ -0,0 +1,28 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package verify + +// prng is a splitmix64 stream. It is deterministic given its seed, so the writer +// and verifier reproduce the identical op stream by seeding from the same +// masterSeed and drawing in the same order (design §4.2). +type prng struct { + state uint64 +} + +func newPRNG(seed uint64) *prng { + return &prng{state: seed} +} + +func (p *prng) next() uint64 { + p.state += 0x9E3779B97F4A7C15 + z := p.state + z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9 + z = (z ^ (z >> 27)) * 0x94D049BB133111EB + return z ^ (z >> 31) +} + +// intn returns a value in [0,n). n must be positive. +func (p *prng) intn(n int) int { + return int(p.next() % uint64(n)) +} diff --git a/evetest/testapps/volverify/internal/verify/util.go b/evetest/testapps/volverify/internal/verify/util.go new file mode 100644 index 00000000000..aac62b08c2f --- /dev/null +++ b/evetest/testapps/volverify/internal/verify/util.go @@ -0,0 +1,25 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package verify + +import ( + "errors" + "fmt" + "syscall" +) + +// filePathFor returns the volume-relative path of a file, scattering files across +// a bounded two-level tree derived from the fileID alone. Writer and verifier +// both use it so a file always lands at the same place. +func filePathFor(cfg Config, fileID uint64) string { + fan := uint64(cfg.DirFanout) + top := fileID % fan + sub := (fileID / fan) % fan + return fmt.Sprintf("d%02d/d%02d/f%d.blk", top, sub, fileID) +} + +// isNotEmpty reports whether err is a "directory not empty" error. +func isNotEmpty(err error) bool { + return errors.Is(err, syscall.ENOTEMPTY) || errors.Is(err, syscall.EEXIST) +} diff --git a/evetest/testapps/volverify/scripts/loopback-ext4-test.sh b/evetest/testapps/volverify/scripts/loopback-ext4-test.sh new file mode 100755 index 00000000000..1ed4a34170d --- /dev/null +++ b/evetest/testapps/volverify/scripts/loopback-ext4-test.sh @@ -0,0 +1,91 @@ +#!/bin/bash +# Copyright (c) 2026 Zededa, Inc. +# SPDX-License-Identifier: Apache-2.0 +# +# On-filesystem fidelity check for volverify: round-trips the write/verify pattern +# on a real loopback ext4 (not tmpfs), then injects truncation and block-zeroing +# and confirms the verifier detects them. A final stage corrupts filesystem data +# and runs e2fsck so the lost+found / orphaned-vs-present-corrupt path exercises +# the real fsck behavior the soak depends on. +# +# Requires root (losetup/mkfs/mount). Run: sudo ./scripts/loopback-ext4-test.sh + +set -euo pipefail + +if [ "$(id -u)" -ne 0 ]; then + echo "must run as root (losetup/mount); re-run with sudo" >&2 + exit 1 +fi + +HERE="$(cd "$(dirname "$0")/.." && pwd)" +WORK="$(mktemp -d)" +IMG="$WORK/vol.img" +MNT="$WORK/mnt" +BIN="$WORK/volverify" +LOOP="" + +cleanup() { + mountpoint -q "$MNT" && umount "$MNT" || true + [ -n "$LOOP" ] && losetup -d "$LOOP" 2>/dev/null || true + rm -rf "$WORK" +} +trap cleanup EXIT + +# Deterministic op stream small enough to run quickly but large enough to spill +# files across many block groups. +SEED=20260723 +OPS=4000 +COMMON="--seed $SEED --ops $OPS --block-size 4096 --small-blocks 8 --med-blocks 512 --max-blocks 4096" + +echo "== build volverify ==" +( cd "$HERE" && GOWORK=off go build -o "$BIN" ./cmd/volverify ) + +echo "== create + mount loopback ext4 (1 GiB) ==" +mkdir -p "$MNT" +truncate -s 1G "$IMG" +mkfs.ext4 -q -F "$IMG" +LOOP="$(losetup --find --show "$IMG")" +mount "$LOOP" "$MNT" + +echo "== write pattern ==" +# shellcheck disable=SC2086 +"$BIN" write --dir "$MNT" $COMMON + +echo "== verify (expect clean on a real ext4) ==" +# shellcheck disable=SC2086 +"$BIN" verify --dir "$MNT" $COMMON +echo "PASS: clean round-trip on ext4" + +echo "== inject truncation + block-zeroing on two files ==" +# Collect matches with mapfile rather than `find | head` — under pipefail the +# early pipe close makes find exit 141 (SIGPIPE) and set -e then aborts. +mapfile -t FILES < <(find "$MNT" -name 'f*.blk') +if [ "${#FILES[@]}" -lt 2 ]; then + echo "FAIL: need >=2 data files to corrupt, found ${#FILES[@]}" >&2 + exit 1 +fi +VICTIM="${FILES[0]}" +VICTIM2="${FILES[1]}" +truncate -s 0 "$VICTIM" +dd if=/dev/zero of="$VICTIM2" bs=4096 count=1 conv=notrunc status=none + +echo "== verify (expect anomalies) ==" +# shellcheck disable=SC2086 +if "$BIN" verify --dir "$MNT" $COMMON; then + echo "FAIL: verifier reported clean after injected corruption" >&2 + exit 1 +fi +echo "PASS: verifier detected injected truncation/zeroing" + +echo "== fsck path: corrupt fs data then e2fsck, re-verify (informational) ==" +umount "$MNT" +# Zero a 4 MiB span in the data area to force e2fsck into real repairs. +dd if=/dev/zero of="$IMG" bs=1M count=4 seek=200 conv=notrunc status=none +e2fsck -fy "$IMG" || true +mount "$LOOP" "$MNT" +echo "-- lost+found after e2fsck:"; ls -1 "$MNT/lost+found" 2>/dev/null || true +echo "-- verifier report after e2fsck:" +# shellcheck disable=SC2086 +"$BIN" verify --dir "$MNT" $COMMON || true + +echo "ALL DONE" diff --git a/evetest/tests/resize/appvolshrink_test.go b/evetest/tests/resize/appvolshrink_test.go new file mode 100644 index 00000000000..012dc8ed12f --- /dev/null +++ b/evetest/tests/resize/appvolshrink_test.go @@ -0,0 +1,1395 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package resize_test + +import ( + "encoding/base64" + "fmt" + "strings" + "testing" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + uuid "github.com/satori/go.uuid" + + eveconfig "github.com/lf-edge/eve-api/go/config" + "github.com/lf-edge/eve-api/go/evecommon" + eveinfo "github.com/lf-edge/eve-api/go/info" + pillartypes "github.com/lf-edge/eve/pkg/pillar/types" + + "github.com/lf-edge/eve/evetest" + "github.com/lf-edge/eve/evetest/constants" + "github.com/lf-edge/eve/evetest/matchers" + "github.com/lf-edge/eve/evetest/netmodels" +) + +const ( + seedParamKey = "VOLVERIFY_SEED" + opsParamKey = "VOLVERIFY_OPS" + volverifyImageParamKey = "VOLVERIFY_IMAGE" + + // defaultVolverifyImage is where the volverify testapp is currently published. + // The canonical lfedge/evetest-volverify repository does not exist yet; point + // VOLVERIFY_IMAGE at it once it does. + defaultVolverifyImage = "eriknordmark/evetest-volverify" + + // volverifyBlockSize is the on-disk block size of the verification pattern; it + // must be identical for the write and the verify, which replay the same stream. + volverifyBlockSize = 4096 + + // volverifyCommitDir is volverify's on-volume committed-index directory. Its + // presence identifies the data volume among the guest's block devices after + // the conversion, when the volume is no longer mounted at its MountDir. + volverifyCommitDir = ".vv-commit" + + fillPeakPctParamKey = "FILL_PEAK_PCT" + fillKeepGiBParamKey = "FILL_KEEP_GIB" + devsideOnlyParamKey = "DEVSIDE_ONLY" + + // Fill /persist to this percentage before the app is deployed, then trim back + // to this many GiB once its volume is written. The peak has to sit well above + // the shrink boundary (~38 GiB of a 61.7 GiB /persist on a 64 GiB boot disk) so + // that the volume, allocated last, lands above it; the keep figure has to stay + // under the resizer's own limit (~34 GiB) and leave room for EVE-k's images + // (~8 GiB measured), while still leaving enough high-residing data that the + // shrink runs long enough for the watchdog to interrupt it. + defaultFillPeakPct = 90 + defaultFillKeepGiB = 15 + + // stageCDataVolMiB is the data-volume size this test defaults to. The app + // redeploy on EVE-k wedges in a Longhorn CSI CreateVolume race above ~256 MiB + // (sweep: 100/256 MiB pass, >=512 MiB wedge), which would mask the corruption + // result behind an unrelated failure, so stay under that ceiling by default. + stageCDataVolMiB = 256 +) + +// TestAppVolumeShrinkCorruption checks whether the EVE-kvm→EVE-k offline boot-disk +// shrink corrupts an application data volume when it is interrupted part-way, and +// whether the corruption is detectable. +// +// It follows the same conversion sequence as TestKvmToKResize — that ordering and +// those readiness gates are load-bearing, see the comments there — and swaps the +// plain container app for volverify with a blank data volume. The volume is filled +// with a deterministic self-verifying pattern on EVE-kvm, the conversion relocates +// its blocks, and the pattern is re-verified on EVE-k. +// +// The fault is baked into the target EVE image rather than driven from here: the +// fork#7 stress build runs the offline resizer under a no-pet hardware watchdog +// whose timeout escalates with the retry count, so early attempts are cut mid +// shrink/grow and a later one converges. Run this against a non-stress build to +// get the same measurement with no fault injected. +// +// Parameters: as TestKvmToKResize, plus +// - DATAVOL_MB: blank data-volume size (default 256; see stageCDataVolMiB). +// - VOLVERIFY_SEED / VOLVERIFY_OPS: pattern seed and op count. The writer stops +// early when the volume fills and reports the committed high-water mark, which +// the verify is then held to. +// - VOLVERIFY_IMAGE: Docker repo of the volverify testapp. +// +// The device is provisioned from the LIVE image; see +// TestAppVolumeShrinkCorruptionFromInstaller for the installer-written variant. +func TestAppVolumeShrinkCorruption(test *testing.T) { + runAppVolumeShrinkCorruption(test, evetest.CreateFromScratchWithLiveImage) +} + +// TestAppVolumeShrinkCorruptionFromInstaller takes the same corruption +// measurement against a boot disk the EVE installer laid out, instead of a +// pre-built live image written to the disk whole. An installer-written ESP carries +// a zero-length boot/.boot_repository, which the offline grow's FAT32 copy can only +// read with diskfs/go-diskfs#419 ("invalid start cluster: 0" without it); a +// live-image ESP has no such file, so only this variant covers that path. +// +// Same parameters as TestAppVolumeShrinkCorruption. Expect a longer setup: the +// installer VM boots and writes the disk before the test's own device is up. +func TestAppVolumeShrinkCorruptionFromInstaller(test *testing.T) { + runAppVolumeShrinkCorruption(test, evetest.CreateFromScratchWithInstaller) +} + +// runAppVolumeShrinkCorruption is the body shared by both variants. +// provisionPolicy selects how the device's boot disk comes into existence; +// everything after Setup is identical. +func runAppVolumeShrinkCorruption(test *testing.T, + provisionPolicy evetest.ExistingEdgeDeviceReusePolicy) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.EVEVersionParameter(), + evetest.HypervisorParameter(), + evetest.TPMParameter(), + evetest.DiskSizeMiBParameter(), + evetest.TestParameterDefinition{ + Key: ramSizeMiBParamKey, + DefaultValue: uint32(minDeviceRAMInMiB), + Description: evetest.TestParameterDescription{ + Summary: "Device RAM in MiB (EVE-k + Longhorn need >= 16 GiB)", + Default: "16384 (16 GiB)", + }, + }, + evetest.TestParameterDefinition{ + Key: cpusParamKey, + DefaultValue: uint8(minDeviceCPUs), + Description: evetest.TestParameterDescription{ + Summary: "Device vCPUs (EVE-k + Longhorn need >= 8)", + Default: "8", + }, + }, + evetest.TestParameterDefinition{ + Key: dataVolMiBParamKey, + DefaultValue: uint32(stageCDataVolMiB), + Description: evetest.TestParameterDescription{ + Summary: "App data-volume size in MiB (stay <= 256 to avoid the EVE-k CSI create race)", + Default: "256", + }, + }, + evetest.TestParameterDefinition{ + Key: fillPeakPctParamKey, + DefaultValue: uint32(defaultFillPeakPct), + Description: evetest.TestParameterDescription{ + Summary: "Fill /persist to this % before deploying the app, so its volume lands in the blocks the shrink evacuates (0 disables)", + Default: "90", + }, + }, + evetest.TestParameterDefinition{ + Key: fillKeepGiBParamKey, + DefaultValue: uint32(defaultFillKeepGiB), + Description: evetest.TestParameterDescription{ + Summary: "Trim /persist back to this many GiB after the volume is written (must leave room for EVE-k)", + Default: "15", + }, + }, + evetest.TestParameterDefinition{ + Key: devsideOnlyParamKey, + DefaultValue: false, + Description: evetest.TestParameterDescription{ + Summary: "Stop once the volume verdict is taken, skipping the app-side checks", + Default: "false", + }, + }, + evetest.TestParameterDefinition{ + Key: seedParamKey, + DefaultValue: uint64(20260723), + Description: evetest.TestParameterDescription{ + Summary: "volverify master seed for the fill/delete pattern", + Default: "20260723", + }, + }, + evetest.TestParameterDefinition{ + Key: opsParamKey, + DefaultValue: uint64(200000), + Description: evetest.TestParameterDescription{ + Summary: "volverify op count; the writer stops early once the volume fills", + Default: "200000", + }, + }, + evetest.TestParameterDefinition{ + Key: volverifyImageParamKey, + DefaultValue: defaultVolverifyImage, + Description: evetest.TestParameterDescription{ + Summary: "Docker repo of the volverify test app", + Default: defaultVolverifyImage, + }, + }, + evetest.TestParameterDefinition{ + Key: initialEVEVersionParamKey, + DefaultValue: "16.6.0", + Description: evetest.TestParameterDescription{ + Summary: "SMALL-geometry EVE-kvm base to start on (pre-large-geometry)", + Default: "16.6.0", + }, + }, + evetest.TestParameterDefinition{ + Key: initialHypervisorParamKey, + DefaultValue: evetest.HypervisorKVM, + Description: evetest.TestParameterDescription{ + Summary: "Hypervisor of the initial (small) base", + Default: "kvm", + AllowedValues: "kvm", + }, + }, + ) + + withTPM := evetest.GetTPMParameterValue() + diskSizeMiB := evetest.GetDiskSizeMiBParameterValue() + initialVersion := evetest.GetTestParameter[string](initialEVEVersionParamKey) + if initialVersion == "" { + evetestT.Fatalf("%s%s is required", constants.EnvPrefix, initialEVEVersionParamKey) + } + initialHypervisor := evetest.GetTestParameter[evetest.Hypervisor](initialHypervisorParamKey) + convVersion := evetest.GetEVEVersionParameterValue() + targetHypervisor := evetest.GetHypervisorParameterValue() + + effectiveDiskMiB := diskSizeMiB + if effectiveDiskMiB == 0 { + effectiveDiskMiB = constants.DefaultEVEDeviceDiskSizeInMiB + } + if effectiveDiskMiB < constants.DefaultEVEDeviceDiskSizeInMiB { + evetestT.Fatalf("boot disk %d MiB is too small for the kvm→k conversion; "+ + "need at least %d MiB (64 GiB) — set DISK_SIZE_MB accordingly", + effectiveDiskMiB, constants.DefaultEVEDeviceDiskSizeInMiB) + } + effectiveRAMMiB := evetest.GetTestParameter[uint32](ramSizeMiBParamKey) + if effectiveRAMMiB == 0 { + effectiveRAMMiB = constants.DefaultEVEDeviceRAMInMiB + } + if effectiveRAMMiB < minDeviceRAMInMiB { + evetestT.Fatalf("device RAM %d MiB is too small for the kvm→k conversion "+ + "(EVE-k + Longhorn); need at least %d MiB (16 GiB) — set RAM_SIZE_MB accordingly", + effectiveRAMMiB, minDeviceRAMInMiB) + } + effectiveCPUs := evetest.GetTestParameter[uint8](cpusParamKey) + if effectiveCPUs == 0 { + effectiveCPUs = constants.DefaultEVEDeviceCPUs + } + if effectiveCPUs < minDeviceCPUs { + evetestT.Fatalf("device vCPUs %d is too few for the kvm→k conversion "+ + "(EVE-k + Longhorn); need at least %d — set CPUS accordingly", + effectiveCPUs, minDeviceCPUs) + } + dataVolMiB := evetest.GetTestParameter[uint32](dataVolMiBParamKey) + dataVolBytes := uint64(dataVolMiB) * evetest.MiB + seed := evetest.GetTestParameter[uint64](seedParamKey) + ops := evetest.GetTestParameter[uint64](opsParamKey) + volverifyImage := evetest.GetTestParameter[string](volverifyImageParamKey) + fillPeakPct := evetest.GetTestParameter[uint32](fillPeakPctParamKey) + fillKeepGiB := evetest.GetTestParameter[uint32](fillKeepGiBParamKey) + if fillPeakPct > 0 { + t.Expect(fillKeepGiB).To(BeNumerically(">", 0), "FILL_KEEP_GIB must be set when filling") + } + + // Cap a single file at a sixteenth of the volume. volverify's default large-file + // bound is 256 MiB, which on a volume this size would let one op consume the + // whole thing and leave the pattern with almost no files to place — and so + // almost no coverage of the relocated block range. + maxBlocks := uint64(dataVolMiB) * evetest.MiB / 16 / volverifyBlockSize + if maxBlocks < 256 { + maxBlocks = 256 + } + volverifyArgs := fmt.Sprintf("--dir %s --seed %d --ops %d --block-size %d --max-blocks %d", + dataMountDir, seed, ops, volverifyBlockSize, maxBlocks) + + const devName = "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithEVEVersion: initialVersion, + WithHypervisor: initialHypervisor, + WithTPM: withTPM, + MinDiskSizeInMiB: diskSizeMiB, + MinRAMInMiB: effectiveRAMMiB, + MinCPUs: effectiveCPUs, + DeviceReusePolicy: provisionPolicy, + }, + evetest.RequireNetworkModel{NetworkModel: netmodels.SingleEthWithDHCP}, + ) + device := evetest.GetEdgeDevice(devName) + log := evetest.Logger() + log.Infof("volverify app %s:1.0 on a %d MiB data volume (max file %d blocks)", + volverifyImage, dataVolMiB, maxBlocks) + + devConfig := evetest.NewEdgeDeviceConfig(devName) + if fillPeakPct > 0 { + // The volume is deliberately created on a nearly-full /persist, which EVE + // would otherwise refuse: volumemgr declines a volume whose size exceeds the + // remaining space, counting a dom0 reservation of 20% on top. The fill is + // transient and trimmed away before the conversion, so the check is what is + // wrong here, not the request. + devConfig.ConfigItems = append(devConfig.ConfigItems, &eveconfig.ConfigItem{ + Key: string(pillartypes.IgnoreDiskCheckForApps), + Value: "true", + }) + } + networkUUID := devConfig.AddNetwork(evetest.DHCPNetworkConfig{NetworkType: evecommon.NetworkType_V4}) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "eth0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: networkUUID, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + device.ApplyConfig(devConfig, false, false) + + // The conversion is several reboots plus an EVE-k bring-up and runs close to + // the framework's default upgrade budget, so give it room; without this a + // conversion that is merely slow is reported as a failed one. + device.SetUpgradeTimeout(45 * time.Minute) + + log.Infof("baseline: asserting SMALL boot-disk geometry") + assertSmallGeometry(t, device) + assertWatchdogDriverBound(t, device) + evetest.Checkpoint("baseline-small") + + log.Infof("kvm→kvm hop: upgrading to the conversion-capable build %s (kvm)", convVersion) + device.UpgradeEVE(convVersion, evetest.HypervisorKVM, true, false) + assertSmallGeometry(t, device) + assertCheckDecision(t, device, "shrink") + evetest.Checkpoint("kvm-hop-done") + + log.Infof("settling vault to a local TPM unlock") + settleVaultLocal(t, device) + evetest.Checkpoint("vault-settled") + + // Fill /persist before the app exists, so its data volume is allocated at the + // top of the filesystem — inside the range the shrink has to evacuate. On an + // almost-empty /persist the volume lands low, the shrink finishes in about a + // second, and the watchdog only ever interrupts the grow. + if fillPeakPct > 0 { + log.Infof("filling /persist to %d%% so the app's volume lands in the shrink's evacuation zone", fillPeakPct) + fillPersistToPct(t, device, int(fillPeakPct), dataVolMiB) + evetest.Checkpoint("persist-filled") + } + + // Switch (L2-bridged) NI: the carried app gets its own DHCP address on the eth0 + // segment and is reached directly. A local NI does not reconverge onto the + // kubevirt VMI after the conversion (verified stuck, not slow). + niUUID := devConfig.AddNetworkInstance(evetest.SwitchNetworkInstanceConfig{ + DisplayName: "switch-ni", + Port: "eth0", + }) + appUUID := devConfig.AddApplication(evetest.ApplicationInstanceConfig{ + DisplayName: "volverify-app", + Activate: true, + Image: evetest.DockerContainer{ImageName: volverifyImage, Tag: "1.0"}, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 512 * evetest.MiB, + NetworkAdapters: []evetest.AppNetworkAdapter{ + evetest.VirtualNetworkAdapter{ + LogicalLabel: "vif0", + NetworkInstanceUUID: niUUID, + PortFwdRules: []evetest.PortFwdRule{ + {Protocol: evetest.NetworkProtocolTCP, EdgeNodePort: appSSHFwdPort, AppPort: 22}, + }, + ACLAllowRules: []evetest.ACLAllowRule{ + {Protocol: evetest.NetworkProtocolAny, RemoteSubnet: evetest.IPSubnet("0.0.0.0/0")}, + }, + }, + }, + DataVolumes: []evetest.DataVolumeConfig{ + {SizeBytes: dataVolBytes, MountDir: dataMountDir}, + }, + }) + device.ApplyConfig(devConfig, false, false) + device.WaitUntilAppIsRunning(appUUID, 5*time.Minute) + appAuth := evetest.UsernamePasswordAuth{Username: appSSHUser, Password: appSSHPassword} + assertAppSSH(t, device, appUUID, appAuth) + captureAppNet(device, "pre-conversion EVE-kvm (SSH OK)") + + // Fill the volume on EVE-kvm, where the runx shim has formatted and mounted it + // at MountDir. The writer stops at Ops or when the volume fills, whichever comes + // first, and reports the committed high-water mark the verify is held to. + log.Infof("pre-conversion: filling the data volume with the volverify pattern") + writtenCommitted := writeVolverifyPattern(t, device, appUUID, appAuth, volverifyArgs) + log.Infof("volume filled through committed op %d", writtenCommitted) + + // Free the low blocks now that the volume is placed. This is what lets the + // shrink fit at all, and it leaves the relocation work — including the volume — + // concentrated above the boundary. + if fillPeakPct > 0 { + log.Infof("trimming /persist back to %d GiB (lowest blocks first)", fillKeepGiB) + trimPersistToGiB(t, device, int(fillKeepGiB)) + log.Infof("asserting the data volume actually lies above the shrink boundary") + assertVolumeAboveShrinkBoundary(t, device) + evetest.Checkpoint("volume-placed-high") + } + capturePersist(device, "pre-conversion EVE-kvm (volume filled)") + evetest.Checkpoint("volume-filled") + + // EVE's post-resize check hashes each volume against a pre-resize manifest and + // DELETES any that mismatches, so a torn volume is gone before this test can look + // at it — which is why nine 8 GiB corruptions were recorded as "nothing to + // measure". The marker makes baseosmgr quarantine it as .corrupt instead, + // leaving the damaged bytes for fsck and volverify to characterize. + markerOut, _, markerErr := device.RunShellScript( + `eve exec pillar touch /persist/volmanifest-keep-corrupt`, eveShellTimeout, 0) + t.Expect(markerErr).NotTo(HaveOccurred(), + "could not arm the corrupt-volume quarantine marker:\n%s", markerOut) + log.Infof("armed /persist/volmanifest-keep-corrupt so a torn volume is kept, not deleted") + + log.Infof("kvm→k conversion: upgrading to %s (%s) — triggers the offline shrink+grow", convVersion, targetHypervisor) + conversionOK := false + defer func() { + if !conversionOK { + dumpConversionFailure(device) + } + }() + // Do not wait for EVE to commit the new partition. Committing is a trial + // period that runs long after the device is already up on the target, and + // nothing this test asserts depends on it: the geometry is grown by then and + // what matters is whether the app and its volume come back. Waiting for it + // only delays — and can fail — a conversion that is doing fine. The commit is + // observed at the end instead, where it costs nothing. + device.UpgradeEVE(convVersion, targetHypervisor, false, false) + waitDeviceOnTarget(t, device, targetHypervisor, 45*time.Minute) + conversionOK = true + device.ExpectAdditionalReboots(1) + captureResizeEvidence(device) + evetest.Checkpoint("conversion-complete") + + assertGrownShrink(t, device) + evetest.Checkpoint("geometry-grown") + + stopPersistSampler := startPersistSampler(device, 45*time.Second) + defer stopPersistSampler() + + // Take the verdict here, before Longhorn can ingest the volume and long before + // the app is asked to boot. This is the only reading that survives the + // post-conversion app wedge, which has cost every verdict it has hit. + verifyRelocatedVolumeOnDevice(t, device, dataVolMiB, seed, ops, maxBlocks, writtenCommitted) + + // The volume verdict is complete here, about 32 minutes into a run whose + // remaining 48 minutes are the app-side checks. At data-volume sizes where the + // post-conversion app reliably wedges, those 48 minutes re-derive a known + // failure, so a corruption soak can stop here and collect roughly two and a + // half times as many volume verdicts per day. + // + // This forfeits the wedge diagnostics, which are produced by the app-failure + // path and nowhere else: leave it off when the run is meant to investigate why + // the app does not come back. + if evetest.GetTestParameter[bool](devsideOnlyParamKey) { + log.Infof("DEVSIDE_ONLY: volume verdict taken; skipping the app-side checks") + evetest.Checkpoint("devside-only-complete") + return + } + + if targetHypervisor == evetest.HypervisorKubevirt { + log.Infof("(a) waiting for k3s node ready") + if !isK3sReady(device.GetClusterInfo()) { + clusterUpdates, stop := device.WatchClusterInfo() + defer stop() + t.Eventually(clusterUpdates, 30*time.Minute).Should(Receive( + matchers.SatisfyPredicate("K3s node is ready", isK3sReady))) + } + log.Infof("(b) waiting for volumemgr Initialized") + waitVolumemgrReady(t, device) + log.Infof("(c) waiting for longhorn StorageClass ready") + waitLonghornSC(t, device) + } + // Registered above the wait, not below it: the app failing to reach RUNNING is the + // most common way this test fails, and a defer set up after the wait never runs on + // that path — which is exactly where the wedge evidence is needed. + appRunning := false + appSSHOK := false + defer func() { + if appSSHOK { + return + } + stage := "never reached RUNNING" + if appRunning { + stage = "RUNNING but unreachable" + } + captureAppNet(device, "post-conversion EVE-k (app "+stage+")") + captureAppNetFailure(device, appUUID) + }() + log.Infof("(d) waiting for the app to reach RUNNING on the target") + waitAppRunningWithPVCRecovery(t, device, appUUID, dataVolMiB) + appRunning = true + log.Infof("(e0) post-conversion: waiting up to 10m for the app to report a routable IPv4") + waitAppHasRoutableIPv4(t, device, appUUID, 10*time.Minute) + log.Infof("(e) post-conversion: app must be SSH-reachable") + assertAppSSH(t, device, appUUID, appAuth) + appSSHOK = true + + log.Infof("(f) post-conversion: re-verifying the data-volume pattern") + captureVolManifest(device) + + // Check the filesystem before anything mounts it, so the structural verdict can + // be set against the content verdict below. A mount would replay the journal and + // could repair what is being measured. + volDev := findDataVolumeDevice(t, device, appUUID, appAuth, dataVolMiB) + fsckRC, fsckOut := fsckDataVolume(device, appUUID, appAuth, volDev) + + state := mountDataVolumeRO(t, device, appUUID, appAuth, dataMountDir) + if state == volumeStateBlank { + recordVolumeOutcome(dataVolMiB, state, fsckRC, "", fsckOut, -1, "") + // The pattern is gone but the volume is intact and empty: either the + // post-resize manifest check found the volume torn and removed it, so EVE + // recreated it blank, or the shrink destroyed the filesystem outright. The + // manifest capture above says which. Either way nothing corrupt is being + // served to the app, so this is not the failure this test gates on. + log.Errorf("data volume came back BLANK — the pattern did not survive; " + + "check the volume-manifest capture for whether the detector removed it") + evetest.Checkpoint("volume-recreated-blank") + return + } + + report := verifyVolverifyPattern(t, device, appUUID, appAuth, volverifyArgs, writtenCommitted) + log.Infof("volverify report:\n%s", report) + + // Content is captured, so the filesystem can now be checked with the journal + // replayed — the only reading that distinguishes real damage from the stale + // accounting an unclean unmount leaves behind. + log.Infof("re-checking the volume filesystem with the journal replayed") + replayRC, replayOut := fsckDataVolumeAfterVerify(device, appUUID, appAuth, volDev, dataMountDir) + recordVolumeOutcome(dataVolMiB, state, fsckRC, report, fsckOut, replayRC, replayOut) + + // present-corrupt is the silent case: a torn-but-present volume that EVE would + // serve to the app as-is, invisible to fsck and to qemu-img. orphaned and lost + // files are the recoverable modes — EVE recreates a missing volume file blank — + // so surface them but do not fail on them. + presentCorrupt := reportField(report, "present-corrupt") + t.Expect(presentCorrupt).To(BeNumerically(">=", 0), + "could not parse present-corrupt from the volverify report:\n%s", report) + t.Expect(presentCorrupt).To(Equal(0), + "data volume served corrupt-but-present after the interrupted shrink:\n%s", report) + evetest.Checkpoint("volume-verified") + + // Whether EVE committed the new partition is recorded rather than asserted: + // the volume result above does not depend on it, but a target still sitting at + // "inprogress" here means the trial period had not finished, and one that + // reverted would explain an otherwise puzzling later failure. + logPartitionState(device) +} + +// devsideVerifyScript checks the carried-over volume on the device itself. +// +// upgradeconverter moves the pre-conversion app volumes out of +// /persist/vault/volumes — which Longhorn must own on EVE-k — into +// /persist/vault/volumes-kvm, and volumemgr rolls them into PVCs lazily once the +// cluster is up. In between, and for as long as that import keeps failing, the file +// sits there as a plain ext4 image holding the volume exactly as the shrink left it. +// The data-volume path in csihandler returns on a failed RolloutDiskToPVC without +// removing the source, so a wedged import preserves it rather than consuming it. +// +// volverify does not have to be shipped in: containerd has already unpacked the app +// image, so the same statically-linked binary the app would run is on disk. Args: +// seed, ops, max-blocks, expect-committed. +const devsideVerifyScript = `set -u +# Both artifacts appear on their own schedule after the conversion: containerd unpacks +# the app image, and the post-vault phase of upgradeconverter moves the volumes. The +# device reports itself on the target before either is guaranteed, so wait for both +# together — checking one first and exiting early skips the wait on the other. +# Encrypted volumes land in vault/volumes-kvm; clear ones stay in clear/volumes, which +# is where kvmMigratedSourcePath looks for them. +VV="" +F="" +i=0 +while [ "$i" -lt 60 ]; do + [ -n "$VV" ] || VV=$(find /persist/vault/containerd -path '*usr/local/bin/volverify' 2>/dev/null | head -1) + # A *.raw.corrupt is EVE's post-resize hash check having caught the interrupted + # shrink tearing this volume, quarantined instead of deleted (see the marker this + # test drops before the conversion). That is the finding, not a missing volume: + # verify it exactly like an intact one, so fsck and volverify can say whether the + # damage is structurally visible or silent. + # Quarantined copies are listed first: a *.raw.corrupt IS the finding, so picking + # an intact sibling ahead of it would report a clean verdict for a conversion that + # had already been caught tearing a volume. + [ -n "$F" ] || F=$(ls /persist/vault/volumes-kvm/*.raw.corrupt /persist/clear/volumes/*.raw.corrupt \ + /persist/vault/volumes-kvm/*.raw /persist/clear/volumes/*.raw \ + 2>/dev/null | head -1) + [ -n "$VV" ] && [ -n "$F" ] && break + i=$((i + 1)); sleep 10 +done +echo "DEVSIDE-WAITED=${i}0s" +# Giving up is only actionable with the layout that defeated the search: an 8 GiB run +# put ~8 GiB somewhere under /persist/vault inside the wait window, yet matched nothing +# here, and du -d1 could not say where. List the candidates so the next occurrence +# reports the actual path instead of only that it is not the expected one. +if [ -z "$VV" ] || [ -z "$F" ]; then + echo "DEVSIDE-LAYOUT:" + for d in /persist/vault/volumes-kvm /persist/vault/volumes /persist/clear/volumes; do + echo " $d:" + ls -l "$d" 2>&1 + done + echo " largest under /persist/vault:" + du -x -B1 -d2 /persist/vault 2>/dev/null | sort -rn | head -12 +fi +[ -n "$VV" ] || { echo "DEVSIDE=no-volverify"; exit 0; } +[ -n "$F" ] || { echo "DEVSIDE=no-relocated-volume"; exit 0; } +echo "DEVSIDE-FILE=$F size=$(stat -c %s "$F")" +# Whether the selection above had a choice at all: >1 means head -1 discarded a +# candidate and the verdict may describe the wrong volume. +echo "DEVSIDE-CANDIDATES=$(ls /persist/vault/volumes-kvm/*.raw.corrupt \ + /persist/clear/volumes/*.raw.corrupt /persist/vault/volumes-kvm/*.raw \ + /persist/clear/volumes/*.raw 2>/dev/null | wc -l)" +# Read the filesystem before anything mounts it: a mount replays the journal and can +# repair the very damage being measured. +e2fsck -fn "$F" > /tmp/dsfsck.out 2>&1 +echo "DEVSIDE-FSCK-RC=$?" +sed -n '1,40p' /tmp/dsfsck.out +mkdir -p /tmp/dsmnt +L=$(losetup -r -f --show "$F" 2>&1) || { echo "DEVSIDE=losetup-failed:$L"; exit 0; } +if mount -o ro,noload "$L" /tmp/dsmnt 2>&1; then + # Redirect rather than pipe, so the status is volverify's and not the pager's. + "$VV" verify --dir /tmp/dsmnt --seed "$1" --ops "$2" \ + --block-size 4096 --max-blocks "$3" --expect-committed "$4" > /tmp/dsvv.out 2>&1 + echo "DEVSIDE-VERIFY-RC=$?" + grep -aE 'committed=|verify:' /tmp/dsvv.out | tail -4 + umount /tmp/dsmnt || true +else + echo "DEVSIDE=mount-failed" +fi +losetup -d "$L" || true +rm -rf /tmp/dsmnt /tmp/dsvv.out /tmp/dsfsck.out` + +// verifyRelocatedVolumeOnDevice records the volume verdict without involving the app, +// Longhorn, CDI or a completed PVC import — see devsideVerifyScript. +// +// Not finding anything to check is non-fatal ONLY when the app-side verdict is still to +// come: on a clear-volume or ZFS layout the file may not be where this looks. Under +// DEVSIDE_ONLY there is no second verdict, so the same silence means the iteration +// measured nothing — and a run that measures nothing must not report PASS, or a soak +// banks clean-looking rows that never examined a volume. +// A volume it does read and find corrupt fails the test either way; that is the finding +// being hunted. +func verifyRelocatedVolumeOnDevice(t Gomega, device *evetest.EdgeDevice, dataVolMiB uint32, + seed, ops uint64, maxBlocks uint64, expectCommitted int) { + log := evetest.Logger() + out, err := runOnEVEScript(device, devsideVerifyScript, 15*time.Minute, + fmt.Sprintf("%d", seed), fmt.Sprintf("%d", ops), + fmt.Sprintf("%d", maxBlocks), fmt.Sprintf("%d", expectCommitted)) + if err != nil { + log.Errorf("device-side volume verify could not run:\n%s", out) + return + } + log.Infof("device-side volume verify:\n%s", strings.TrimSpace(out)) + + if i := strings.Index(out, "DEVSIDE="); i >= 0 { + var reason string + fmt.Sscanf(out[i:], "DEVSIDE=%s", &reason) + log.Errorf("device-side volume verify skipped: %s", reason) + // The volume was present and written before the conversion, so its absence + // afterwards is upgradeconverter's account to give: stageKvmVolumes logs + // "relocated N, skipped M" per run, plus a line for each entry it declined to + // move and for the sentinel short-circuit. Without this the layout dump can + // only say the volume is not there, not whether it was moved elsewhere, + // skipped, or never seen. + for _, p := range []struct{ what, pipeline string }{ + {"upgradeconverter volume relocation", "grep -a stageKvmVolumes | tail -20"}, + {"upgradeconverter kube entry", "grep -a relocateKvmVolumesForKube | tail -10"}, + // stageKvmVolumes reports relocating everything it finds, so the volume is + // already gone when it runs. Grep for the file itself rather than any one + // agent's wording: whoever removed it had to name it. + {"every mention of a volume file", `grep -a "#0.raw" | tail -40`}, + {"volumemgr delete path", `grep -a volumemgr | grep -aiE "delete|destroy|purge|remov" | tail -30`}, + } { + o, _, e := device.RunShellScript(newlogProbe(p.pipeline), 90*time.Second, 0) + log.Errorf("[devside-miss] %s:\n%s(err=%v)", p.what, strings.TrimSpace(o), e) + } + t.Expect(evetest.GetTestParameter[bool](devsideOnlyParamKey)).To(BeFalse(), + "DEVSIDE_ONLY run banked no volume verdict (%s): this iteration measured "+ + "nothing about shrink corruption:\n%s", reason, out) + return + } + + fsckRC, verifyRC := -1, -1 + if i := strings.Index(out, "DEVSIDE-FSCK-RC="); i >= 0 { + fmt.Sscanf(out[i:], "DEVSIDE-FSCK-RC=%d", &fsckRC) + } + if i := strings.Index(out, "DEVSIDE-VERIFY-RC="); i >= 0 { + fmt.Sscanf(out[i:], "DEVSIDE-VERIFY-RC=%d", &verifyRC) + } + // eve-detected records whether EVE's own post-resize hash check had already + // condemned this volume: the file it hands us is then the quarantined copy. A + // clean volverify on such a file would mean the two checks disagree, so the + // distinction has to survive into the row rather than being inferred from a path. + eveDetected := strings.Contains(out, ".raw"+".corrupt") + evetest.Logger().Infof("[DEVICE-VOLUME-OUTCOME] datavolMiB=%d app-independent=true "+ + "eve-detected-corrupt=%t fsck-rc=%d structural-damage=%t verify-rc=%d verify=[%s]", + dataVolMiB, eveDetected, fsckRC, fsckFoundStructuralDamage(out), verifyRC, + volverifySummary(out)) + evetest.Checkpoint("devside-volume-verified") + + // volverify exits non-zero on ANY anomaly, so its status is the catch-all: a + // verdict class not named individually below still fails the run. + t.Expect(verifyRC).To(BeNumerically("==", 0), + "volverify rejected the relocated volume (DEVSIDE-VERIFY-RC=%d):\n%s", verifyRC, out) + + presentCorrupt := reportField(out, "present-corrupt") + // An absent field reads as -1, which would satisfy the bound below and bank a + // clean-looking row for a run that measured nothing. + t.Expect(presentCorrupt).To(BeNumerically(">=", 0), + "volverify produced no present-corrupt count, so this iteration measured "+ + "nothing about shrink corruption:\n%s", out) + t.Expect(presentCorrupt).To(BeNumerically("<=", 0), + "the relocated pre-conversion volume is present but CORRUPT (present-corrupt=%d) — "+ + "the interrupted shrink tore data the filesystem check cannot see:\n%s", + presentCorrupt, out) + // Losing a committed file outright, or resurrecting a committed-deleted one, is + // data loss just as much as torn content. Orphaned is excluded: recovery into + // lost+found self-heals to a blank/content-tree recreate. + for _, key := range []string{"lost", "resurrected"} { + n := reportField(out, key) + t.Expect(n).To(BeNumerically("<=", 0), + "the relocated pre-conversion volume lost committed data (%s=%d):\n%s", + key, n, out) + } +} + +// volverifySummary pulls volverify's one-line tally out of the device-side output so +// the outcome record carries the counts rather than the whole transcript. +func volverifySummary(out string) string { + for _, line := range strings.Split(out, "\n") { + if strings.Contains(line, "committed=") && strings.Contains(line, "ok=") { + return strings.TrimSpace(line) + } + } + return "no-summary" +} + +// writeVolverifyPattern fills the app's data volume with the deterministic pattern +// and returns the committed op index the writer reached. It requires the volume to +// be mounted at its MountDir, which the runx shim does on EVE-kvm. +func writeVolverifyPattern(t Gomega, device *evetest.EdgeDevice, appUUID uuid.UUID, + auth evetest.AuthMethod, args string) int { + script := fmt.Sprintf( + "grep -q ' %s ' /proc/mounts || { echo NOT-MOUNTED; cat /proc/mounts; exit 1; }; "+ + "volverify write %s && sync", dataMountDir, args) + committed := -1 + t.Eventually(func(t Gomega) { + stdout, stderr, err := device.RunShellScriptInsideApp(appUUID, auth, script, 60*time.Minute, 0) + t.Expect(err).NotTo(HaveOccurred(), "volverify write failed:\n%s%s", stdout, stderr) + committed = reportField(stdout, "committed") + t.Expect(committed).To(BeNumerically(">=", 0), + "volverify write reported no committed index:\n%s", stdout) + }, 65*time.Minute, 10*time.Second).Should(Succeed()) + return committed +} + +// Outcomes of locating the app's data volume after the conversion. +const ( + volumeStatePattern = "PATTERN" // mounted and still carrying the volverify pattern + volumeStateBlank = "BLANK" // the volume exists but the pattern is gone +) + +// recordVolumeOutcome emits one line pairing what the filesystem check concluded +// with what the content verify found, which is the row a soak accumulates over +// many iterations. Read on its own, either verdict is ambiguous; together they say +// whether corruption occurred and whether a structural check would have noticed. +// +// The line is deliberately single and grep-friendly, since the point is to compare +// hundreds of these rather than to read one. +func recordVolumeOutcome(dataVolMiB uint32, state string, fsckRC int, report, fsckOut string, + replayRC int, replayOut string) { + // Before the journal is replayed, count mismatches are expected and mean + // nothing; after it, anything found is real. + dirty := fmt.Sprintf("rc=%d", fsckRC) + switch fsckRC { + case 0: + dirty += "(clean)" + case 4: + dirty += "(errors-unreplayed)" + } + if strings.Contains(fsckOut, "skipping journal recovery") { + dirty += "+journal-not-replayed" + } + replayed := "not-run" + if replayRC >= 0 { + replayed = fmt.Sprintf("rc=%d", replayRC) + switch replayRC { + case 0: + replayed += "(clean)" + case 1, 2: + // A non-zero status is not by itself damage. A volume captured while it + // was still being written to has stale superblock counters, an orphan + // flag and extent trees e2fsck would rather rewrite, and it repairs all + // of those on every run while saying nothing about the data. Only + // findings that imply lost, crossed or unreachable blocks mean the + // interrupted shrink actually hurt the volume. + if fsckFoundStructuralDamage(replayOut) { + replayed += "(STRUCTURAL-DAMAGE)" + } else { + replayed += "(accounting-only)" + } + case 4: + replayed += "(errors-left)" + } + if strings.Contains(replayOut, "FILE SYSTEM WAS MODIFIED") { + replayed += "+modified" + } + } + if report == "" { + report = "no-content-verify" + } + evetest.Logger().Errorf("[VOLUME-OUTCOME] datavolMiB=%d state=%s fsck-dirty=%s fsck-replayed=%s verify=[%s]", + dataVolMiB, state, dirty, replayed, strings.ReplaceAll(report, "\n", " | ")) +} + +// fsckDataVolume runs a read-only filesystem check on the data volume's block +// device and returns e2fsck's exit status and output. +// +// This is the counterpart to the content verify, and the pair is the point: a +// structural check is blind to data blocks that were relocated wrongly but left +// self-consistent, so the interesting result is fsck reporting a clean filesystem +// while the content verify finds files quietly full of zeroes. Recording only one +// of the two would lose exactly the comparison this test exists to make. +// +// It must run before anything mounts the volume — a mount can replay the journal +// and repair the very damage being measured — and with -n so the check itself +// changes nothing. Exit status 0 means fsck saw a clean filesystem; 4 means it +// found errors it was not allowed to fix. +func fsckDataVolume(device *evetest.EdgeDevice, appUUID uuid.UUID, + auth evetest.AuthMethod, dev string) (int, string) { + script := fmt.Sprintf("e2fsck -fn %s 2>&1; echo FSCK_RC=$?", dev) + out, _, _ := device.RunShellScriptInsideApp(appUUID, auth, script, 20*time.Minute, 0) + rc := -1 + if i := strings.Index(out, "FSCK_RC="); i >= 0 { + fmt.Sscanf(out[i:], "FSCK_RC=%d", &rc) + } + evetest.Logger().Errorf("[fsck] %s exit=%d\n%s", dev, rc, strings.TrimSpace(out)) + return rc, strings.TrimSpace(out) +} + +// fsckFoundStructuralDamage reports whether an e2fsck transcript contains findings +// that mean blocks or inodes were actually lost, crossed or orphaned — as opposed +// to the accounting an unclean capture always produces. +// +// Repairing stale free-block and free-inode counters, clearing the orphan-file +// feature flag and narrowing extent trees all happen on a volume that was simply +// snapshotted mid-write; treating those as damage would mark every iteration of a +// soak as a hit and bury the real signal. The patterns below are the ones that +// imply data actually went missing. +func fsckFoundStructuralDamage(out string) bool { + damage := []string{ + "Unattached inode", + "Unattached zero-length inode", + "multiply-claimed", + "Multiply-claimed", + "illegal block", + "Illegal block", + "illegal indirect block", + "lost+found", + "Inode bitmap differences", + "Block bitmap differences", + "Directory inode", + "has an incorrect filesize", + "Entry '", + "deleted/unused inode", + "root inode is not a directory", + "Corrupt", + "corrupted", + } + for _, d := range damage { + if strings.Contains(out, d) { + return true + } + } + return false +} + +// fsckDataVolumeAfterVerify unmounts the volume and checks it again, this time +// letting e2fsck replay the journal and repair what it finds. +// +// The read-only check taken before the mount cannot distinguish real damage from +// bookkeeping: the volume was never cleanly unmounted, so its journal is unreplayed +// and the superblock's free counts necessarily disagree with the disk. That check +// therefore reports errors on every run and says nothing on its own. Replaying +// first removes that noise, so whatever is still wrong here is genuine — at the +// cost of modifying the filesystem, which is why it runs only after the content +// verify has already been recorded. +// +// Exit 0 means clean once the journal was applied; 1 means e2fsck found and fixed +// real structural damage. +func fsckDataVolumeAfterVerify(device *evetest.EdgeDevice, appUUID uuid.UUID, + auth evetest.AuthMethod, dev, mountDir string) (int, string) { + script := fmt.Sprintf( + "umount %s 2>/dev/null; e2fsck -fy %s 2>&1; echo FSCK_RC=$?", mountDir, dev) + out, _, _ := device.RunShellScriptInsideApp(appUUID, auth, script, 20*time.Minute, 0) + rc := -1 + if i := strings.Index(out, "FSCK_RC="); i >= 0 { + fmt.Sscanf(out[i:], "FSCK_RC=%d", &rc) + } + evetest.Logger().Errorf("[fsck-replayed] %s exit=%d\n%s", dev, rc, strings.TrimSpace(out)) + return rc, strings.TrimSpace(out) +} + +// findDataVolumeDevice returns the guest block device holding the app's data +// volume, identified by size rather than by mounting anything, so the caller can +// check the filesystem before it is touched. The app's own root disk differs in +// size by orders of magnitude, so a size match is unambiguous here. +func findDataVolumeDevice(t Gomega, device *evetest.EdgeDevice, appUUID uuid.UUID, + auth evetest.AuthMethod, dataVolMiB uint32) string { + script := fmt.Sprintf(`set -u +WANT=%d +for d in /dev/vd[b-z] /dev/sd[b-z]; do + [ -b "$d" ] || continue + sz=$(blockdev --getsize64 "$d" 2>/dev/null) || continue + mib=$(( sz / 1048576 )) + echo "CAND $d ${mib}MiB" + diff=$(( mib - WANT )); [ "$diff" -lt 0 ] && diff=$(( -diff )) + [ "$diff" -le 128 ] && { echo "DEV=$d"; break; } +done`, dataVolMiB) + var dev string + t.Eventually(func(g Gomega) { + out, _, err := device.RunShellScriptInsideApp(appUUID, auth, script, 2*time.Minute, 0) + g.Expect(err).NotTo(HaveOccurred(), "listing guest block devices failed:\n%s", out) + for _, tok := range strings.Fields(out) { + if d, ok := strings.CutPrefix(tok, "DEV="); ok { + dev = d + } + } + g.Expect(dev).NotTo(BeEmpty(), + "no guest block device close to %d MiB — the data volume is not attached:\n%s", dataVolMiB, out) + }, 5*time.Minute, 15*time.Second).Should(Succeed()) + evetest.Logger().Infof("data volume device: %s", dev) + return dev +} + +// mountDataVolumeRO finds the app's data volume among the guest's block devices +// and mounts it at mountDir, returning which of the states above it is in. EVE-k +// does not auto-mount a container app's data volume at its MountDir +// (lf-edge/eve#6145), so after the conversion the verify has to do it; volverify's +// committed-index directory is what identifies the volume. +// +// The mount is read-only and skips the ext4 journal: the verify only reads, and +// replaying the journal would heal exactly the torn state this test is measuring. +// +// A volume that holds no pattern is reported as BLANK rather than as a failure — +// that is what the caller sees when the post-resize manifest check removed a torn +// volume and EVE recreated it empty. Only finding no data volume at all is an +// error, so the block-device inventory is dumped either way. +func mountDataVolumeRO(t Gomega, device *evetest.EdgeDevice, appUUID uuid.UUID, + auth evetest.AuthMethod, mountDir string) string { + script := fmt.Sprintf(`set -u +mkdir -p %[1]s +[ -d %[1]s/%[2]s ] && { echo STATE=%[3]s already-mounted; exit 0; } +mountpoint -q %[1]s && umount %[1]s +found= +for d in /dev/vd[b-z] /dev/sd[b-z]; do + [ -b "$d" ] || continue + found="$found $d" + mount -o ro,noload -t ext4 "$d" %[1]s 2>/dev/null || continue + if [ -d %[1]s/%[2]s ]; then echo "STATE=%[3]s $d"; exit 0; fi + umount %[1]s +done +echo "candidates:$found" +lsblk 2>/dev/null; blkid 2>/dev/null; cat /proc/mounts +[ -n "$found" ] && { echo STATE=%[4]s; exit 0; } +echo STATE=NO-DATA-DEVICE +exit 1`, mountDir, volverifyCommitDir, volumeStatePattern, volumeStateBlank) + state := "" + t.Eventually(func(t Gomega) { + stdout, stderr, err := device.RunShellScriptInsideApp(appUUID, auth, script, 2*time.Minute, 0) + t.Expect(err).NotTo(HaveOccurred(), + "no data volume among the guest block devices after the conversion:\n%s%s", stdout, stderr) + for _, tok := range strings.Fields(stdout) { + if s, ok := strings.CutPrefix(tok, "STATE="); ok { + state = s + } + } + t.Expect(state).To(Or(Equal(volumeStatePattern), Equal(volumeStateBlank)), + "could not classify the data volume:\n%s", stdout) + evetest.Logger().Infof("data volume state %s:\n%s", state, strings.TrimSpace(stdout)) + }, 5*time.Minute, 15*time.Second).Should(Succeed()) + return state +} + +// runOnEVEScript runs a shell script on EVE inside the pillar container, passing +// it through base64 so nothing has to survive the ssh → `eve exec pillar sh -c` +// quoting layers (the same trick the eden scripts use). args are appended as $1… +func runOnEVEScript(device *evetest.EdgeDevice, script string, + timeout time.Duration, args ...string) (string, error) { + b64 := base64.StdEncoding.EncodeToString([]byte(script)) + cmd := fmt.Sprintf( + `eve exec pillar sh -c 'echo %s | base64 -d > /tmp/evetest-frag.sh; sh /tmp/evetest-frag.sh %s; rm -f /tmp/evetest-frag.sh'`, + b64, strings.Join(args, " ")) + out, errOut, err := device.RunShellScript(cmd, timeout, 0) + if err != nil { + return out + errOut, err + } + return out, nil +} + +// fillPersistToPct fills /persist with incompressible files until it is pct% full, +// so that the app's data volume — created afterwards, while the filesystem is at +// its peak — is allocated in the high block groups that the shrink must evacuate. +// +// The bytes have to be incompressible. Zeroes would let the qcow2 backing file +// store the blocks sparsely, and the relocation would then read and write nothing, +// leaving the shrink as fast as it is on an empty filesystem — which is the whole +// problem this is here to fix. +// reserveMiB is the size of the data volume written next: the fill stops that much +// short of pct, so the peak INCLUDING the volume is the requested percentage. Filling +// all the way to pct and then writing the volume into what little is left collapses the +// allowance volumemgr grants apps — the filler counts as dom0 usage, which is subtracted +// from it — and at RemainingSpace 0 the device enters +// MAINTENANCE_MODE_REASON_LOW_DISK_SPACE and reboots. Observed 3/3 at 8 GiB (peak 95%, +// 3.0 GiB free) and 0/3 at 2 GiB, where the volume still fit in the slack. +func fillPersistToPct(t Gomega, device *evetest.EdgeDevice, pct int, reserveMiB uint32) { + // The filler lives under /persist/log, NOT /persist/tmp: onboot.sh removes + // /persist/tmp unconditionally on every boot (no threshold, no df), so a reboot + // anywhere between the fill and the trim silently voids the placement this step + // exists to create. /persist/log survives a boot and is still the first entry in + // onboot.sh's PERSIST_CLEANUPS list, so a device that genuinely runs out of space + // reclaims it instead of wedging. + const script = `set -u +PCT=$1 +RESERVE_KB=$(( $2 * 1024 )) +DIR=/persist/log/stressfill +rm -rf "$DIR"; mkdir -p "$DIR" +cap=$(df -k /persist | tail -1 | awk '{print $2}') +want=$(( cap * PCT / 100 - RESERVE_KB )) +[ "$want" -lt 0 ] && want=0 +n=0 +while [ "$(df -k /persist | tail -1 | awk '{print $3}')" -lt "$want" ]; do + f=$(printf "%s/%06d" "$DIR" "$n") + dd if=/dev/urandom of="$f" bs=1M count=256 2>/dev/null || break + n=$((n + 1)) +done +sync +echo "FILLED files=$n $(df -h /persist | tail -1)"` + out, err := runOnEVEScript(device, script, 40*time.Minute, + fmt.Sprintf("%d", pct), fmt.Sprintf("%d", reserveMiB)) + t.Expect(err).NotTo(HaveOccurred(), "filling /persist failed:\n%s", out) + t.Expect(out).To(ContainSubstring("FILLED"), "fill did not report completion:\n%s", out) + evetest.Logger().Infof("filled /persist to ~%d%%: %s", pct, strings.TrimSpace(out)) +} + +// trimPersistToGiB deletes the LOWEST-numbered fill files until /persist usage is +// down to keepGiB, which leaves the survivors — and the data volume created at peak +// — concentrated in the high blocks above the future shrink boundary. +// +// Trimming is what makes the shrink possible at all: the resizer refuses when the +// filesystem cannot fit in the target size, and EVE-k needs room afterwards for +// its own images. Deleting from the bottom is what keeps the relocation work high. +func trimPersistToGiB(t Gomega, device *evetest.EdgeDevice, keepGiB int) { + const script = `set -u +KEEP_KB=$(( $1 * 1024 * 1024 )) +DIR=/persist/log/stressfill +# A missing filler means the volume was not placed in the evacuation zone, so the +# iteration cannot say anything about shrink safety. Fail rather than continue: the +# silent version of this produced three clean-looking rows that had measured nothing. +[ -d "$DIR" ] || { echo "TRIM-FAILED no-fill-dir"; exit 1; } +for f in $(ls -1 "$DIR" 2>/dev/null | sort); do + [ "$(df -k /persist | tail -1 | awk '{print $3}')" -le "$KEEP_KB" ] && break + rm -f "$DIR/$f" +done +sync +echo "TRIMMED remaining=$(ls -1 "$DIR" 2>/dev/null | wc -l) $(df -h /persist | tail -1)"` + out, err := runOnEVEScript(device, script, 10*time.Minute, fmt.Sprintf("%d", keepGiB)) + t.Expect(err).NotTo(HaveOccurred(), "trimming /persist failed:\n%s", out) + t.Expect(out).To(ContainSubstring("TRIMMED"), "trim did not report completion:\n%s", out) + evetest.Logger().Infof("trimmed /persist to ~%d GiB: %s", keepGiB, strings.TrimSpace(out)) +} + +// assertVolumeAboveShrinkBoundary fails the run unless the app's data volume has +// blocks above the size the shrink will cut /persist down to — i.e. unless the +// shrink will actually have to relocate it. +// +// Without this the test can pass for the wrong reason: a volume that sits entirely +// below the boundary is untouched by the shrink, so a clean verify says nothing +// about whether an interrupted relocation corrupts data. The boundary comes from +// the resizer's own check (TargetBytes) rather than an assumption, and the block +// placement from filefrag, which reports physical extents relative to the +// filesystem the file lives on. +func assertVolumeAboveShrinkBoundary(t Gomega, device *evetest.EdgeDevice) { + disk, err := bootDiskPath(device) + t.Expect(err).NotTo(HaveOccurred()) + out, err := runEVE(device, "eve exec pillar /usr/bin/storage-resizer check --disk "+disk+" --json") + t.Expect(err).NotTo(HaveOccurred()) + // targetBytes is the post-shrink filesystem size (ShrinkResult.TargetBytes). + var target int64 + if i := strings.Index(out, `"targetBytes"`); i >= 0 { + fmt.Sscanf(out[i:], `"targetBytes": %d`, &target) + } + t.Expect(target).To(BeNumerically(">", 0), + "could not read targetBytes from the resizer check:\n%s", out) + + // The physical range is the third column once any space inside "start.. end" + // is closed up, which keeps the parse independent of filefrag's alignment; the + // second half of that range is the file's highest block. Verified against real + // filefrag output for both spacings and for a file with no extents. + const script = `set -u +sync +FF=/usr/sbin/filefrag +[ -x "$FF" ] || FF=$(command -v filefrag 2>/dev/null || echo filefrag) +max=0 +for d in /persist/vault/volumes /persist/clear/volumes /persist/vault/volumes-kvm /persist/clear/volumes-kvm; do + [ -d "$d" ] || continue + for f in "$d"/*; do + [ -f "$f" ] || continue + e=$("$FF" -b4096 -v "$f" 2>/dev/null | awk ' + { line=$0; gsub(/\.\.[ \t]+/, "..", line); $0=line } + $1 ~ /^[0-9]+:$/ { split($3, r, /\.\./); x=r[2]; sub(/:$/, "", x); if (x+0 > m) m=x+0 } + END { print m+0 }') + [ -n "$e" ] || continue + echo "VOL $f top4k=$e" + [ "$e" -gt "$max" ] && max=$e + done +done +echo "MAXTOP4K $max"` + fragOut, err := runOnEVEScript(device, script, 5*time.Minute) + t.Expect(err).NotTo(HaveOccurred(), "reading volume block placement failed:\n%s", fragOut) + var top4k int64 + if i := strings.Index(fragOut, "MAXTOP4K "); i >= 0 { + fmt.Sscanf(fragOut[i:], "MAXTOP4K %d", &top4k) + } + topByte := top4k * 4096 + evetest.Logger().Infof("volume top block %d (%.1f GiB) vs shrink target %.1f GiB\n%s", + top4k, float64(topByte)/(1<<30), float64(target)/(1<<30), strings.TrimSpace(fragOut)) + t.Expect(topByte).To(BeNumerically(">", target), + "the data volume lies entirely below the shrink boundary (top %.1f GiB vs target %.1f GiB), "+ + "so the shrink would not relocate it and a clean verify would prove nothing", + float64(topByte)/(1<<30), float64(target)/(1<<30)) +} + +// logPartitionState records what EVE currently reports for each base image, so a +// run that did not commit its target (or reverted) is visible after the fact. +// Best-effort; never fails the test. +func logPartitionState(device *evetest.EdgeDevice) { + info := device.GetDeviceInfo() + if info == nil { + evetest.Logger().Errorf("[partition-state] no device info") + return + } + for _, sw := range info.GetSwList() { + evetest.Logger().Errorf("[partition-state] %s partition=%s status=%s %s", + sw.GetShortVersion(), sw.GetPartitionState(), sw.GetUserStatus(), sw.GetSubStatusStr()) + } +} + +// waitDeviceOnTarget waits until the device is RUNNING the target flavor, which +// is a weaker and much earlier condition than the framework's upgrade wait: that +// one blocks until EVE commits the partition (state "active"), whereas this +// returns as soon as the target is the booted partition ("inprogress" counts). +// +// It fails fast if EVE flags any base image FAILED, so a rejected conversion still +// surfaces immediately rather than burning the whole budget. +func waitDeviceOnTarget(t Gomega, device *evetest.EdgeDevice, + hv evetest.Hypervisor, timeout time.Duration) { + // The target's short version carries a flavor suffix; for the conversion the + // only thing that distinguishes it from the kvm hop is that suffix. + suffix := "-kvm-" + if hv == evetest.HypervisorKubevirt { + suffix = "-k-" + } + t.Eventually(func(g Gomega) { + info := device.GetDeviceInfo() + g.Expect(info).NotTo(BeNil()) + var seen []string + for _, sw := range info.GetSwList() { + ver, state := sw.GetShortVersion(), sw.GetPartitionState() + seen = append(seen, fmt.Sprintf("%s[%s/%s]", ver, state, sw.GetUserStatus())) + if !strings.Contains(ver, suffix) { + continue + } + g.Expect(sw.GetUserStatus()).NotTo(Equal(eveinfo.BaseOsStatus_FAILED), + "EVE flagged %s FAILED: %s", ver, sw.GetSubStatusStr()) + if state == "inprogress" || state == "active" { + evetest.Logger().Infof("device is running %s (partition %s)", ver, state) + return + } + } + g.Expect(false).To(BeTrue(), "device not running a %s image yet: %v", suffix, seen) + }, timeout, 15*time.Second).Should(Succeed()) +} + +// Rounds of "wait, then try the documented PVC recovery" allowed before the app +// is declared stuck, and how long each round waits. +const ( + pvcRecoveryRounds = 3 + pvcRecoveryWait = 12 * time.Minute +) + +// waitAppRunningWithPVCRecovery waits for the app to reach RUNNING on EVE-k, +// applying the documented recovery for the known app-PVC wedge between attempts. +// +// The wedge is a Longhorn CSI create/verify race that leaves a PVC Pending +// forever: it is "stuck, not slow", so simply waiting longer never rescues it. +// Deleting the PVC lets the provisioner re-drive it cleanly, which is what makes +// data volumes above ~256 MiB usable at all — below that the race is rare, above +// it the wedge is the norm. +func waitAppRunningWithPVCRecovery(t Gomega, device *evetest.EdgeDevice, + appUUID uuid.UUID, dataVolMiB uint32) { + log := evetest.Logger() + for round := 1; round <= pvcRecoveryRounds; round++ { + if waitAppRunningQuietly(device, appUUID, pvcRecoveryWait) { + return + } + log.Errorf("app not RUNNING after %s (round %d/%d) — trying the PVC-wedge recovery", + pvcRecoveryWait, round, pvcRecoveryRounds) + if !recoverWedgedAppPVCs(device, dataVolMiB) { + log.Errorf("nothing safe to recover this round") + } + } + t.Expect(appIsRunning(device, appUUID)).To(BeTrue(), + "app never reached RUNNING after %d PVC-wedge recovery attempts", pvcRecoveryRounds) +} + +// appIsRunning reports the app's current state without asserting. +func appIsRunning(device *evetest.EdgeDevice, appUUID uuid.UUID) bool { + info := device.GetAppInfo(appUUID) + return info != nil && info.GetState() == eveinfo.ZSwState_RUNNING +} + +// waitAppRunningQuietly polls until the app is RUNNING or the timeout expires, +// returning whether it got there. Unlike the framework's waiter it does not fail +// the test on timeout, so the caller can intervene and keep waiting. +func waitAppRunningQuietly(device *evetest.EdgeDevice, appUUID uuid.UUID, timeout time.Duration) bool { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if appIsRunning(device, appUUID) { + return true + } + time.Sleep(15 * time.Second) + } + return appIsRunning(device, appUUID) +} + +// recoverWedgedAppPVCs deletes Pending PVCs in the app namespace so the +// provisioner re-drives them, and reports whether it deleted any. +// +// It must never delete the data volume's PVC or that volume's CDI scratch: the +// data volume holds the pattern this test verifies, and EVE recreates a deleted +// volume BLANK — the verify would then find an empty volume and the run would +// report a clean pass, a false negative wearing the clothes of a result. So the +// size check is deliberately biased towards protecting: anything within reach of +// the data volume's size is left alone, and if that is the wedged PVC then no +// recovery happens and the run is allowed to fail honestly. +// +// The discriminator is size because the harness does not know the volume UUIDs +// EVE assigns. That is sound while the data volume is comfortably larger than the +// app's image PVC (a few hundred MiB) — which is exactly the case where recovery +// is needed. At data-volume sizes near the image size the app's own PVC gets +// protected too and recovery no-ops; the classification is logged so that is +// visible rather than silent. +// +// Sizes arrive in whatever form kubectl prints, which for these PVCs is a plain +// byte count rather than the Gi/Mi suffixes one might expect, so every form is +// handled and anything unrecognised is protected rather than deleted. Getting this +// wrong is not a missed recovery but a destroyed volume followed by a run that +// passes because the volume came back empty. +func recoverWedgedAppPVCs(device *evetest.EdgeDevice, dataVolMiB uint32) bool { + log := evetest.Logger() + script := fmt.Sprintf(`set -u +DV=%d +LIST=$(eve exec kube kubectl -n eve-kube-app get pvc \ + -o custom-columns=N:.metadata.name,P:.status.phase,R:.spec.resources.requests.storage \ + --no-headers 2>/dev/null) +[ -z "$LIST" ] && { echo NO-PVCS; exit 0; } +echo "$LIST" | sed 's/^/PVC: /' +VICTIMS=$(echo "$LIST" | awk -v dv="$DV" ' + function mib(s) { + if (s ~ /^[0-9]+$/) { return s / 1048576 } + if (s ~ /^[0-9]+Ki$/) { sub(/Ki$/, "", s); return s / 1024 } + if (s ~ /^[0-9]+Mi$/) { sub(/Mi$/, "", s); return s + 0 } + if (s ~ /^[0-9]+Gi$/) { sub(/Gi$/, "", s); return s * 1024 } + if (s ~ /^[0-9]+Ti$/) { sub(/Ti$/, "", s); return s * 1048576 } + return -1 + } + $2 == "Pending" { + v = mib($3) + if (v < 0 || v >= dv - 32) { printf "PROTECTED %%s (%%s)\n", $1, $3; next } + printf "VICTIM %%s (%%s)\n", $1, $3 + }') +echo "$VICTIMS" +NAMES=$(echo "$VICTIMS" | awk '$1=="VICTIM" {print $2}') +[ -z "$NAMES" ] && { echo NOTHING-TO-RECOVER; exit 0; } +for v in $NAMES; do + echo "RECOVERING $v" + eve exec kube kubectl -n eve-kube-app delete pvc "$v" --wait=false 2>&1 | sed 's/^/ /' +done`, dataVolMiB) + out, errOut, err := device.RunShellScript(script, 3*time.Minute, 0) + log.Errorf("[pvc-recovery]\n%s%s(err=%v)", strings.TrimSpace(out), errOut, err) + return strings.Contains(out, "RECOVERING ") +} + +// assertWatchdogDriverBound fails the run if no watchdog driver is bound in the +// guest. It checks sysfs rather than the device node: a /dev/watchdog character +// node can exist with nothing behind it, so its presence alone proves nothing, +// whereas an entry under /sys/class/watchdog means a driver registered. It does +// not try to open the node — once EVE's watchdog service is up it holds it, and +// EBUSY here would be a false alarm. +// +// This checks the QEMU setup, not EVE. The stress build's resizer arms +// /dev/watchdog and then deliberately stops feeding it, which is how this test +// interrupts the offline resize — but if the resizer cannot open the device it +// exits quietly and nothing is interrupted. The conversion then completes and the +// volume verifies perfectly, which reads exactly like evidence that an interrupted +// shrink preserves the data. +// +// Necessary but not sufficient: this runs with the system fully up, whereas the +// resizer runs from an onboot container much earlier. Only the resizer's own +// console output confirms it armed the watchdog on that path. +func assertWatchdogDriverBound(t Gomega, device *evetest.EdgeDevice) { + t.Eventually(func(g Gomega) { + out, err := runEVE(device, + `ls /sys/class/watchdog/ 2>/dev/null | grep -q watchdog && echo WATCHDOG-DRIVER-BOUND || echo WATCHDOG-DRIVER-MISSING; `+ + `ls /sys/class/watchdog/ 2>&1; cat /sys/class/watchdog/watchdog0/identity 2>/dev/null`) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(out).To(ContainSubstring("WATCHDOG-DRIVER-BOUND"), + "no watchdog driver bound in the guest, so the stress resizer cannot "+ + "interrupt the resize and a clean result here would mean nothing; check "+ + "that QEMU exposes a watchdog and the chipset may reset:\n%s", out) + evetest.Logger().Infof("watchdog driver:\n%s", strings.TrimSpace(out)) + }, 2*time.Minute, 10*time.Second).Should(Succeed()) +} + +// captureResizeEvidence records whether the offline resize was actually +// interrupted, which is the whole premise of this test and is otherwise +// invisible: a clean conversion and a fault-injected one that happened to +// converge look identical from the harness side. +// +// Note the resize attempt counter on the CONFIG partition is NOT usable here: +// storage-resize.sh deletes it on the success path, so by the time the conversion +// has finished it always reads empty regardless of how many attempts it took. The +// durable evidence is what EVE recorded about why it rebooted — a watchdog reset +// is reported as its own boot reason — plus whatever the resizer left in the logs. +func captureResizeEvidence(device *evetest.EdgeDevice) { + log := evetest.Logger() + log.Errorf("=== offline-resize fault evidence ===") + probes := []struct{ what, script string }{ + {"boot / reboot reasons", `for f in /persist/boot-reason /persist/reboot-reason /persist/status/boot-reason /persist/status/reboot-reason /persist/log/reboot-reason.log; do [ -f "$f" ] && { echo "--- $f"; cat "$f"; }; done 2>/dev/null || echo NONE`}, + {"watchdog boot reason in logs", newlogProbe(`grep -ahoE "BootReason[A-Za-z]+" | sort | uniq -c | sort -rn | head`)}, + {"resize-failed.json", `cat /config/resize-failed.json 2>/dev/null || echo NONE`}, + {"watchdog device", `[ -c /dev/watchdog ] && echo PRESENT || echo MISSING; wdctl /dev/watchdog 2>&1 | head -8`}, + {"resizer/watchdog log lines", newlogProbe(`grep -ahiE "run-watchdog|storage-resizer|resize did not converge|watchdog" | tail -30`)}, + } + for _, p := range probes { + out, errOut, err := device.RunShellScript(p.script, 60*time.Second, 0) + log.Errorf("[resize:%s]\n%s%s(err=%v)", p.what, strings.TrimSpace(out), errOut, err) + } +} + +// captureVolManifest reports whether the post-resize volume-manifest check ran and +// what it concluded — whether it found the pre-shrink hashes, judged any volume +// torn, and removed it. This is what separates "the shrink left the volume alone" +// from "the detector cleaned up after it". Best-effort; never fails the test. +func captureVolManifest(device *evetest.EdgeDevice) { + log := evetest.Logger() + log.Errorf("=== volume-manifest (post-resize detect+recreate) ===") + probes := []struct{ what, script string }{ + {"manifest files on /persist", `eve exec pillar sh -c 'ls -l /persist/vault/volumes/.sha256 /persist/clear/volumes/.sha256 2>&1'`}, + {"volmanifest / recreate signatures (newlog)", newlogProbe(`grep -ahiE "volmanifest|recreateCorruptVolumes|verifyVolumes|torn by the resize" | tail -40`)}, + {"app volume files", `eve exec pillar sh -c 'ls -l /persist/vault/volumes /persist/clear/volumes 2>&1'`}, + } + for _, p := range probes { + out, errOut, err := device.RunShellScript(p.script, 60*time.Second, 0) + log.Errorf("[volmanifest:%s]\n%s%s(err=%v)", p.what, strings.TrimSpace(out), errOut, err) + } +} + +// verifyVolverifyPattern replays the pattern against the volume and returns the +// report. volverify exits non-zero on a dirty report, so the exit status is not an +// assertion failure here — the caller classifies the report instead. +func verifyVolverifyPattern(t Gomega, device *evetest.EdgeDevice, appUUID uuid.UUID, + auth evetest.AuthMethod, args string, expectCommitted int) string { + cmd := fmt.Sprintf("volverify verify %s --expect-committed %d", args, expectCommitted) + var report string + t.Eventually(func(t Gomega) { + stdout, _, _ := device.RunShellScriptInsideApp(appUUID, auth, cmd, 30*time.Minute, 0) + t.Expect(stdout).NotTo(BeEmpty(), "volverify verify produced no output") + report = strings.TrimSpace(stdout) + }, 32*time.Minute, 10*time.Second).Should(Succeed()) + return report +} + +// reportField extracts an integer "key=N" field from a volverify summary line, +// returning -1 when the field is absent or unparsable. +func reportField(report, key string) int { + for _, tok := range strings.Fields(report) { + if !strings.HasPrefix(tok, key+"=") { + continue + } + var n int + if _, err := fmt.Sscanf(strings.TrimPrefix(tok, key+"="), "%d", &n); err != nil { + return -1 + } + return n + } + return -1 +} diff --git a/evetest/tests/resize/kvmtok_resize_test.go b/evetest/tests/resize/kvmtok_resize_test.go new file mode 100644 index 00000000000..0890d4d1524 --- /dev/null +++ b/evetest/tests/resize/kvmtok_resize_test.go @@ -0,0 +1,791 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Package resize_test reproduces, in evetest, the eden EVE-kvm→EVE-k in-field +// boot-disk shrink+grow conversion — the proven sequence in eden's +// update_eve_image_kvm_to_k.txt (EXPECT_DECISION=shrink), one step at a time. +// +// This is the "basic resize" stage (no application volume, no fault injection): +// it proves evetest can drive the whole small→large conversion with the shrink +// and keep a container app alive across it. The ordering and the waits mirror the +// eden escript deliberately — they are load-bearing (vault must settle to a local +// TPM unlock before the app; the offline resize reboots several times; on EVE-k +// the volumemgr → longhorn → app → SSH readiness gates must be waited on in order). +// +// The one deliberate divergence from the eden escript: it DELETES the app before +// the k update (the old cross-flavor gate refused any volume). With the gate +// lifted (baseosmgr allow-shrink-with-volumes), this test KEEPS the app across the +// conversion. +package resize_test + +import ( + "fmt" + "net" + "strings" + "testing" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + uuid "github.com/satori/go.uuid" + + eveconfig "github.com/lf-edge/eve-api/go/config" + "github.com/lf-edge/eve-api/go/evecommon" + eveinfo "github.com/lf-edge/eve-api/go/info" + "github.com/lf-edge/eve/evetest" + "github.com/lf-edge/eve/evetest/constants" + "github.com/lf-edge/eve/evetest/matchers" + "github.com/lf-edge/eve/evetest/netmodels" +) + +const ( + initialEVEVersionParamKey = "INITIAL_EVE_VERSION" + initialHypervisorParamKey = "INITIAL_HYPERVISOR" + ramSizeMiBParamKey = "RAM_SIZE_MB" + cpusParamKey = "CPUS" + + // minDeviceRAMInMiB and minDeviceCPUs are the resource floors for the kvm→k + // conversion. EVE-k (kubevirt + Longhorn) needs well above the 8 GiB / 4 vCPU + // framework defaults; the eden runs that pass 7/7 use 16 GiB and 8 vCPUs, so + // require at least that here. + minDeviceRAMInMiB = 16384 + minDeviceCPUs = 8 + + appSSHUser = "root" + appSSHPassword = "testpassword" + appSSHFwdPort = 2222 + sshTimeout = 20 * time.Second + + // Stage B: a blank data volume attached to the app, with a marker written into + // it on EVE-kvm that must survive the shrink conversion onto EVE-k. + dataMountDir = "/mnt/data" + dataVolMiBParamKey = "DATAVOL_MB" // app data-volume size sweep: 2 GiB wedges the CSI create/verify race, 100 MiB passes + volMarker = "STAGE-B-VOL-MARKER-9f3a1c7e-survives-kvm-to-k" +) + +// TestKvmToKResize drives 16.6.0-kvm (small) → current EVE-kvm (same geometry, +// lands the conversion code) → current EVE-k (cross-flavor ⇒ offline shrink+grow), +// keeping a container app across the conversion. The device is provisioned from +// the LIVE image, the way eden brings one up. +// +// Parameters: +// - INITIAL_EVE_VERSION (required, e.g. "16.6.0") / INITIAL_HYPERVISOR (kvm) — +// the SMALL-geometry base; it must lack the large EVE-k geometry or the +// conversion is a no-op and proves nothing (asserted). +// - EVE_VERSION / HYPERVISOR — the conversion-capable current build; the kvm +// variant is the intermediate hop, the kubevirt variant is the k target. +// - DISK_SIZE_MB — keep the boot disk full (no free tail) so the conversion +// SHRINKS P3 rather than growing into free space (asserted: decision==shrink). +func TestKvmToKResize(test *testing.T) { + runKvmToKResize(test, evetest.CreateFromScratchWithLiveImage) +} + +// TestKvmToKResizeFromInstaller runs the same conversion against a boot disk the +// EVE installer laid out, instead of a pre-built live image written to the disk +// whole. That distinction is what the variant is for: an installer-written ESP +// carries a zero-length boot/.boot_repository, and the offline grow relocates the +// ESP by copying its FAT32 contents, which go-diskfs can only read with +// diskfs/go-diskfs#419 ("invalid start cluster: 0" without it). A live-image ESP +// has no such file, so only this variant covers that path. Run it against an EVE +// image built with the fix. +// +// Same parameters as TestKvmToKResize. Expect a longer setup: the installer VM +// boots and writes the disk before the test's own device is up. +func TestKvmToKResizeFromInstaller(test *testing.T) { + runKvmToKResize(test, evetest.CreateFromScratchWithInstaller) +} + +// runKvmToKResize is the body shared by both variants. provisionPolicy selects +// how the device's boot disk comes into existence; everything after Setup is +// identical, since the conversion sequence does not depend on it. +func runKvmToKResize(test *testing.T, provisionPolicy evetest.ExistingEdgeDeviceReusePolicy) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.EVEVersionParameter(), + evetest.HypervisorParameter(), + evetest.TPMParameter(), + evetest.DiskSizeMiBParameter(), + evetest.TestParameterDefinition{ + Key: ramSizeMiBParamKey, + DefaultValue: uint32(minDeviceRAMInMiB), + Description: evetest.TestParameterDescription{ + Summary: "Device RAM in MiB (EVE-k + Longhorn need >= 16 GiB)", + Default: "16384 (16 GiB)", + }, + }, + evetest.TestParameterDefinition{ + Key: cpusParamKey, + DefaultValue: uint8(minDeviceCPUs), + Description: evetest.TestParameterDescription{ + Summary: "Device vCPUs (EVE-k + Longhorn need >= 8)", + Default: "8", + }, + }, + evetest.TestParameterDefinition{ + Key: dataVolMiBParamKey, + DefaultValue: uint32(2048), + Description: evetest.TestParameterDescription{ + Summary: "App data-volume size in MiB (size->CSI-race sweep)", + Default: "2048 (2 GiB)", + }, + }, + evetest.TestParameterDefinition{ + Key: initialEVEVersionParamKey, + DefaultValue: "16.6.0", + Description: evetest.TestParameterDescription{ + Summary: "SMALL-geometry EVE-kvm base to start on (pre-large-geometry)", + Default: "16.6.0", + }, + }, + evetest.TestParameterDefinition{ + Key: initialHypervisorParamKey, + DefaultValue: evetest.HypervisorKVM, + Description: evetest.TestParameterDescription{ + Summary: "Hypervisor of the initial (small) base", + Default: "kvm", + AllowedValues: "kvm", + }, + }, + ) + + withTPM := evetest.GetTPMParameterValue() + diskSizeMiB := evetest.GetDiskSizeMiBParameterValue() + initialVersion := evetest.GetTestParameter[string](initialEVEVersionParamKey) + if initialVersion == "" { + evetestT.Fatalf("%s%s is required", constants.EnvPrefix, initialEVEVersionParamKey) + } + initialHypervisor := evetest.GetTestParameter[evetest.Hypervisor](initialHypervisorParamKey) + convVersion := evetest.GetEVEVersionParameterValue() // the conversion-capable build (kvm hop + k target share this version) + targetHypervisor := evetest.GetHypervisorParameterValue() + + // The kvm→k conversion provisions Longhorn/kubevirt onto P3; a boot disk + // smaller than the framework default (64 GiB) starves that geometry and the + // conversion wedges instead of failing cleanly. Reject an undersized + // DISK_SIZE_MB up front rather than burning a full run to discover it. + effectiveDiskMiB := diskSizeMiB + if effectiveDiskMiB == 0 { + effectiveDiskMiB = constants.DefaultEVEDeviceDiskSizeInMiB + } + if effectiveDiskMiB < constants.DefaultEVEDeviceDiskSizeInMiB { + evetestT.Fatalf("boot disk %d MiB is too small for the kvm→k conversion; "+ + "need at least %d MiB (64 GiB) — set DISK_SIZE_MB accordingly", + effectiveDiskMiB, constants.DefaultEVEDeviceDiskSizeInMiB) + } + ramSizeMiB := evetest.GetTestParameter[uint32](ramSizeMiBParamKey) + effectiveRAMMiB := ramSizeMiB + if effectiveRAMMiB == 0 { + effectiveRAMMiB = constants.DefaultEVEDeviceRAMInMiB + } + if effectiveRAMMiB < minDeviceRAMInMiB { + evetestT.Fatalf("device RAM %d MiB is too small for the kvm→k conversion "+ + "(EVE-k + Longhorn); need at least %d MiB (16 GiB) — set RAM_SIZE_MB accordingly", + effectiveRAMMiB, minDeviceRAMInMiB) + } + cpus := evetest.GetTestParameter[uint8](cpusParamKey) + effectiveCPUs := cpus + if effectiveCPUs == 0 { + effectiveCPUs = constants.DefaultEVEDeviceCPUs + } + if effectiveCPUs < minDeviceCPUs { + evetestT.Fatalf("device vCPUs %d is too few for the kvm→k conversion "+ + "(EVE-k + Longhorn); need at least %d — set CPUS accordingly", + effectiveCPUs, minDeviceCPUs) + } + dataVolBytes := uint64(evetest.GetTestParameter[uint32](dataVolMiBParamKey)) * evetest.MiB + + const devName = "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithEVEVersion: initialVersion, + WithHypervisor: initialHypervisor, + WithTPM: withTPM, + MinDiskSizeInMiB: diskSizeMiB, + MinRAMInMiB: effectiveRAMMiB, + MinCPUs: effectiveCPUs, + DeviceReusePolicy: provisionPolicy, + }, + evetest.RequireNetworkModel{NetworkModel: netmodels.SingleEthWithDHCP}, + ) + device := evetest.GetEdgeDevice(devName) + log := evetest.Logger() + log.Infof("app data-volume size: %d MiB", dataVolBytes/evetest.MiB) + + // Management + app network. + devConfig := evetest.NewEdgeDeviceConfig(devName) + networkUUID := devConfig.AddNetwork(evetest.DHCPNetworkConfig{NetworkType: evecommon.NetworkType_V4}) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "eth0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: networkUUID, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + device.ApplyConfig(devConfig, false, false) + + // The conversion is several reboots plus an EVE-k bring-up and runs close to + // the framework's default upgrade budget, so give it room; without this a + // conversion that is merely slow is reported as a failed one. + device.SetUpgradeTimeout(45 * time.Minute) + + // Step 1: baseline — the boot disk must be SMALL, else the conversion is a no-op. + log.Infof("baseline: asserting SMALL boot-disk geometry") + assertSmallGeometry(t, device) + evetest.Checkpoint("baseline-small") + + // Step 2: kvm→kvm hop — lands the conversion code, geometry unchanged. + log.Infof("kvm→kvm hop: upgrading to the conversion-capable build %s (kvm)", convVersion) + device.UpgradeEVE(convVersion, evetest.HypervisorKVM, true, false) + log.Infof("post-kvm-hop: geometry must still be SMALL") + assertSmallGeometry(t, device) + log.Infof("post-kvm-hop: storage-resizer check must decide 'shrink'") + assertCheckDecision(t, device, "shrink") + evetest.Checkpoint("kvm-hop-done") + + // Step 3: settle the vault to a LOCAL TPM unlock (a new rootfs moves PCRs, so + // the first boot unlocks via the controller key; reboot until it seals locally). + log.Infof("settling vault to a local TPM unlock") + settleVaultLocal(t, device) + evetest.Checkpoint("vault-settled") + + // Step 4: deploy a container app + a blank DATA VOLUME on EVE-kvm; wait RUNNING + // + SSH-reachable, then write a marker into the volume. + // SWITCH (L2, bridged) NI: the app gets its own DHCP address on the eth0 segment + // and is reached directly at app-IP:22 (the port-map is kept as a 2nd candidate + // endpoint). Verified: the LOCAL NI does NOT reconverge onto the kubevirt VMI + // after the conversion (no app IP even after a 10-min wait), whereas the switch + // NI does (~3.5 min); the readiness gate (step 7) absorbs that delay. + niUUID := devConfig.AddNetworkInstance(evetest.SwitchNetworkInstanceConfig{ + DisplayName: "switch-ni", + Port: "eth0", + }) + appUUID := devConfig.AddApplication(evetest.ApplicationInstanceConfig{ + DisplayName: "resize-test-app", + Activate: true, + Image: evetest.DockerContainer{ImageName: "milan4zededa/evetest-ubuntu-ctr", Tag: "1.0"}, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 512 * evetest.MiB, + NetworkAdapters: []evetest.AppNetworkAdapter{ + evetest.VirtualNetworkAdapter{ + LogicalLabel: "vif0", + NetworkInstanceUUID: niUUID, + PortFwdRules: []evetest.PortFwdRule{ + {Protocol: evetest.NetworkProtocolTCP, EdgeNodePort: appSSHFwdPort, AppPort: 22}, + }, + ACLAllowRules: []evetest.ACLAllowRule{ + {Protocol: evetest.NetworkProtocolAny, RemoteSubnet: evetest.IPSubnet("0.0.0.0/0")}, + }, + }, + }, + DataVolumes: []evetest.DataVolumeConfig{ + {SizeBytes: dataVolBytes, MountDir: dataMountDir}, + }, + }) + device.ApplyConfig(devConfig, false, false) + device.WaitUntilAppIsRunning(appUUID, 5*time.Minute) + appAuth := evetest.UsernamePasswordAuth{Username: appSSHUser, Password: appSSHPassword} + log.Infof("pre-conversion: app must be SSH-reachable") + assertAppSSH(t, device, appUUID, appAuth) + // Baseline network snapshot on EVE-kvm where the port-map SSH works, to diff + // against the post-conversion EVE-k snapshot if that SSH fails. + captureAppNet(device, "pre-conversion EVE-kvm (SSH OK)") + log.Infof("pre-conversion: writing a marker into the app data volume at %s", dataMountDir) + writeVolumeMarker(t, device, appUUID, appAuth, dataMountDir, volMarker) + capturePersist(device, "pre-conversion EVE-kvm (app running)") + evetest.Checkpoint("pre-conversion-app-ready") + + // Step 5: kvm→k — the cross-flavor seam triggers the offline shrink+grow. The + // app is KEPT (gate lifted). UpgradeEVE drives the multi-reboot conversion. + log.Infof("kvm→k conversion: upgrading to %s (%s) — triggers the offline shrink+grow", convVersion, targetHypervisor) + // On conversion failure the device returns online on the unconverted kvm + // partition — still SSH-reachable — while EVE flags the -k baseos FAILED, and + // UpgradeEVE fatals. Capture the on-device reason first (the deferred probe runs + // before harness teardown; conversionOK stays false because the fatal skips it). + conversionOK := false + defer func() { + if !conversionOK { + dumpConversionFailure(device) + } + }() + device.UpgradeEVE(convVersion, targetHypervisor, true, false) + conversionOK = true + // The offline shrink+grow reboots once more than UpgradeEVE accounts for (its + // intermediate initrd-resize boot is invisible to the controller, but the + // post-resize kvm boot and the -k boot are two observed reboots vs the one + // UpgradeEVE expects). Declare that extra reboot so the teardown check balances. + device.ExpectAdditionalReboots(1) + evetest.Checkpoint("conversion-complete") + + // Step 6: assert the boot disk grew to LARGE and P3 shrank. + log.Infof("post-conversion: asserting boot disk grew (shrink path: ESP-B created, P3 shrank)") + assertGrownShrink(t, device) + evetest.Checkpoint("geometry-grown") + + // Sample /persist + Longhorn disk accounting through the EVE-k startup window + // (k3s → volumemgr → Longhorn SC → app), to see the "no available disk for + // replica" accounting as it evolves. Stopped once readiness resolves below. + stopPersistSampler := startPersistSampler(device, 45*time.Second) + defer stopPersistSampler() + + // Step 7: post-conversion readiness — in order, so a failure localizes. + if targetHypervisor == evetest.HypervisorKubevirt { + log.Infof("(a) waiting for k3s node ready") + if !isK3sReady(device.GetClusterInfo()) { + clusterUpdates, stop := device.WatchClusterInfo() + defer stop() + t.Eventually(clusterUpdates, 20*time.Minute).Should(Receive( + matchers.SatisfyPredicate("K3s node is ready", isK3sReady))) + } + log.Infof("(b) waiting for volumemgr Initialized") + waitVolumemgrReady(t, device) + log.Infof("(c) waiting for longhorn StorageClass ready") + waitLonghornSC(t, device) + } + log.Infof("(d) waiting for the app to reach RUNNING on the target") + device.WaitUntilAppIsRunning(appUUID, 65*time.Minute) + // If the app network doesn't come back on EVE-k, snapshot the app network state + // for comparison against the pre-conversion EVE-kvm baseline (the deferred probe + // runs before teardown; appSSHOK stays false on the fatal). + appSSHOK := false + defer func() { + if !appSSHOK { + captureAppNet(device, "post-conversion EVE-k (app net FAILED)") + captureAppNetFailure(device, appUUID) + } + }() + // Readiness gate: the carried app's network reconvergence on EVE-k is slow / + // non-deterministic, so wait (generously) for it to report a routable IPv4 + // before asserting SSH. This separates "not ready yet" from "unreachable": a + // slow reconvergence still passes; a genuinely stuck one fails here with the + // network snapshot, distinguishing it from an SSH/routing problem. + log.Infof("(e0) post-conversion: waiting up to 10m for the app to report a routable IPv4") + waitAppHasRoutableIPv4(t, device, appUUID, 10*time.Minute) + log.Infof("(e) post-conversion: app must be SSH-reachable") + assertAppSSH(t, device, appUUID, appAuth) + appSSHOK = true + // (f) the data volume — and its marker — must have survived the shrink onto + // EVE-k. EVE-k does not auto-mount the container data volume at MountDir + // (lf-edge/eve#6145), so read it off the raw block device from inside the app. + log.Infof("(f) post-conversion: the app data-volume marker must survive on EVE-k") + assertVolumeMarker(t, device, appUUID, appAuth, volMarker) + evetest.Checkpoint("post-conversion-app-ready") +} + +// ---- helpers (mirror the eden escript's assert/wait scripts) ---- + +const eveShellTimeout = 30 * time.Second + +// dumpConversionFailure captures why the kvm→k conversion was marked FAILED by +// EVE. At that point the device has returned online on the unconverted kvm +// partition and is SSH-reachable, so it can be queried directly. Best-effort: it +// logs whatever each probe returns and never fails the (already-failing) test. +func dumpConversionFailure(device *evetest.EdgeDevice) { + log := evetest.Logger() + log.Errorf("kvm→k conversion FAILED — capturing on-device diagnostics") + probes := []struct{ what, script string }{ + {"resize-failed.json", "cat /config/resize-failed.json 2>/dev/null || echo NONE"}, + {"resize-flags", "echo repartition-inprogress=$(cat /config/repartition-inprogress 2>/dev/null); echo resize-reboots=$(cat /config/resize-reboots 2>/dev/null)"}, + {"zboot-status", "zboot status 2>&1 || true"}, + {"BaseOsStatus", "eve exec pillar sh -c 'cat /run/baseosmgr/BaseOsStatus/*.json 2>/dev/null' || echo NONE"}, + } + for _, p := range probes { + out, errOut, err := device.RunShellScript(p.script, eveShellTimeout, 0) + log.Errorf("[capture:%s]\n%s%s(err=%v)", p.what, out, errOut, err) + } +} + +// captureAppNet snapshots the EVE-side app networking behind the port-map SSH: +// the NAT DNAT rules (node:2222→app:22), the app's assigned IP + applied ACLs, +// and the node bridges/IPs. Taken on EVE-kvm (working) and again on EVE-k +// (failing) so the two can be diffed. Best-effort; never fails the test. +func captureAppNet(device *evetest.EdgeDevice, label string) { + log := evetest.Logger() + log.Errorf("=== app-network capture: %s ===", label) + probes := []struct{ what, script string }{ + {"nat-portmap", "eve exec pillar iptables -t nat -S 2>/dev/null | grep -aiE '2222|DNAT|to-destination' || echo none"}, + {"AppNetworkStatus", "eve exec pillar sh -c 'cat /run/zedrouter/AppNetworkStatus/*.json 2>/dev/null' || echo none"}, + {"bridges+ips", "eve exec pillar ip -br addr 2>/dev/null || echo none"}, + } + for _, p := range probes { + out, errOut, err := device.RunShellScript(p.script, eveShellTimeout, 0) + log.Errorf("[net:%s] %s:\n%s%s(err=%v)", label, p.what, strings.TrimSpace(out), errOut, err) + } +} + +// captureAppNetFailure characterizes the known EVE-k app-pvc-not-ready / CDI +// upload-pod wedge that leaves the app stuck CREATING_VOLUME (no VMI → no net). +// It grabs the signals ~/notes/kvm-to-k-resize-soak-app-pvc-not-ready-stats.md +// prescribes for pinning the variant (V1–V5) and whether the image's recovery +// engaged: the scratch/data PVC state (Terminating? storageClass?), the cdi-upload +// pod phase/reason, Longhorn volume+attachment state, the CDI upload-controller +// reconcile errors, and the volumemgr / MOUNT-WEDGE-RECOVERY newlog signatures. +// newlogCat emits every device log record newlogd has kept, for a grep to consume. +// This is the retrieval form from the eve-device-logs skill (see also +// kvm-to-k-conversion-testing/scripts/assert-seal-v2.sh) — do not hand-roll another. +// Narrowing to collect/ finds almost nothing: newlogd moves a collect file into the +// gzipped queues once it passes 550000 bytes or a 300 s timer, so that directory is a +// ~5-minute window rather than a boot's worth of records. -exec … \; runs one zcat +// per file, which both keeps -f safe on the plaintext chunks and avoids an argument +// list that a device with tens of thousands of chunks overflows. +const newlogCat = `find /persist/newlog -name "dev.log.*" -exec zcat -f {} \; 2>/dev/null` + +// newlogProbe builds a diagnostic-table entry running pipeline over that history. +// pipeline must not contain single quotes. +func newlogProbe(pipeline string) string { + return `eve exec pillar sh -c '` + newlogCat + ` | ` + pipeline + `' || echo none` +} + +func captureAppNetFailure(device *evetest.EdgeDevice, appUUID uuid.UUID) { + log := evetest.Logger() + log.Errorf("=== EVE-k app-pvc-not-ready / CDI-wedge diagnostics (app %s) ===", appUUID) + probes := []struct{ what, script string }{ + {"eve-kube-app pvc+pods -o wide", `eve exec kube kubectl -n eve-kube-app get pvc,pods -o wide 2>/dev/null || echo none`}, + {"describe pvc (scratch Terminating/SC/finalizers/events)", `eve exec kube kubectl -n eve-kube-app describe pvc 2>/dev/null || echo none`}, + {"describe pods (cdi-upload phase/reason/events)", `eve exec kube kubectl -n eve-kube-app describe pods 2>/dev/null || echo none`}, + {"longhorn volumes", `eve exec kube kubectl get volumes.longhorn.io -A -o wide 2>/dev/null || echo none`}, + {"volumeattachments", `eve exec kube kubectl get volumeattachments 2>/dev/null || echo none`}, + {"longhorn-csi-provisioner log (CreateVolume 404/500 origin)", `eve exec kube kubectl -n longhorn-system logs deployment/csi-provisioner -c csi-provisioner --tail=250 2>/dev/null || echo none`}, + {"longhorn-manager log (create/verify race)", `eve exec kube kubectl -n longhorn-system logs -l app=longhorn-manager -c longhorn-manager --tail=250 --prefix 2>/dev/null || echo none`}, + {"longhorn StorageClass (replicas/config)", `eve exec kube kubectl get sc longhorn -o yaml 2>/dev/null || echo none`}, + {"longhorn version (manager ds image)", `eve exec kube kubectl -n longhorn-system get ds longhorn-manager -o wide 2>/dev/null || echo none`}, + {"CDI upload-controller log", `eve exec kube kubectl -n cdi logs deployment/cdi-deployment --tail=60 2>/dev/null || echo none`}, + // The upload SERVER, not the controller: this is the process that receives the + // transfer, and the one observed Ready and idle while the upload never + // completes. Its own logs were never captured, so every wedge so far has been + // diagnosed without looking at the component that stalls. + // --previous as well as the live log: on a 512 MiB reproduction the upload pod + // reported Succeeded and was gone by the time this ran, so the live fetch + // returned nothing and the component's own account of the transfer was lost. + {"CDI upload-server pod log (the stalling component)", `eve exec kube kubectl -n eve-kube-app logs -l cdi.kubevirt.io=cdi-upload-server --tail=120 --prefix 2>/dev/null; eve exec kube kubectl -n eve-kube-app logs -l cdi.kubevirt.io=cdi-upload-server --tail=120 --prefix --previous 2>/dev/null; echo "--- end upload-server logs"`}, + // Warning events outlive the pods that caused them, so they survive an upload + // pod that terminated. This is where an attach failure shows up — the observed + // end state is a Bound claim whose Longhorn volume is detached, which no + // pod-scoped probe explains. + {"eve-kube-app warning events (attach/mount failures)", `eve exec kube kubectl -n eve-kube-app get events --field-selector type=Warning --sort-by=.lastTimestamp 2>/dev/null | tail -40 || echo none`}, + {"longhorn-system warning events", `eve exec kube kubectl -n longhorn-system get events --field-selector type=Warning --sort-by=.lastTimestamp 2>/dev/null | tail -30 || echo none`}, + // How far the transfer got. Stuck at 0% means it never started (proxy/route), + // stuck partway means it began and stalled — different faults entirely. + {"CDI upload progress annotations", `eve exec kube kubectl -n eve-kube-app get pvc -o custom-columns=NAME:.metadata.name,PHASE:.status.phase,PROGRESS:'.metadata.annotations.cdi\.kubevirt\.io/storage\.pod\.progress',PODPHASE:'.metadata.annotations.cdi\.kubevirt\.io/storage\.pod\.phase',RUNNING:'.metadata.annotations.cdi\.kubevirt\.io/storage\.condition\.running' 2>/dev/null || echo none`}, + // Whether the target volume is actually attachable underneath: an upload can + // idle because Longhorn never brought the volume up on the uploader's node. + {"longhorn volume detail for the upload target", `eve exec kube kubectl -n longhorn-system get volumes.longhorn.io -o custom-columns=NAME:.metadata.name,STATE:.status.state,ROBUST:.status.robustness,NODE:.status.currentNodeID,SIZE:.spec.size 2>/dev/null || echo none`}, + // A wedged import re-drives this call for tens of minutes, so 30 lines is all + // retry noise: on every wedged run of 2026-07-31 the window came back exactly + // full, crowding out the one line that said whether the recovery below had + // run. An absent line then cannot be told from a truncated one. + {"volumemgr RolloutDiskToPVC / V5 signature (newlog)", newlogProbe(`grep -aiE "RolloutDiskToPVC|retryFailedClusterVolumeCreate|terminating:true|local-path" | tail -200`)}, + // Recovery actions get their own probe so retry spam cannot bury them. + {"PVC recovery actions taken (scratch/target delete)", newlogProbe(`grep -aiE "deleted wedged scratch PVC|could not delete wedged scratch|deleted wedged target PVC" | tail -40`)}, + {"MOUNT-WEDGE-RECOVERY (detector fired?)", newlogProbe(`grep -ai MOUNT-WEDGE-RECOVERY | tail -10`)}, + {"kubevirt vmi -A", `eve exec kube kubectl get vmi -A -o wide 2>/dev/null || echo none`}, + } + for _, p := range probes { + out, errOut, err := device.RunShellScript(p.script, 90*time.Second, 0) + log.Errorf("[wedge:%s]\n%s%s(err=%v)", p.what, strings.TrimSpace(out), errOut, err) + } +} + +// capturePersist snapshots everything about /persist sizing/usage plus Longhorn's +// own view of the node disk — storageMaximum/Available/Scheduled/Reserved and the +// scheduling settings — i.e. the accounting behind "no available disk for replica". +// The kube/longhorn probes no-op on EVE-kvm (pre-conversion) and light up on EVE-k. +// Best-effort; never fails the test. +func capturePersist(device *evetest.EdgeDevice, label string) { + log := evetest.Logger() + log.Errorf("=== /persist capture: %s ===", label) + probes := []struct{ what, script string }{ + {"df persist+submounts", `eve exec pillar sh -c 'df -B1 | awk "NR==1 || /persist/"' 2>/dev/null || echo none`}, + {"lsblk sizes+mounts", `eve exec pillar lsblk -b -o NAME,PARTLABEL,SIZE,FSTYPE,MOUNTPOINT 2>/dev/null || echo none`}, + {"du -d1 /persist", `eve exec pillar sh -c 'du -x -B1 -d1 /persist 2>/dev/null | sort -rn' || echo none`}, + {"longhorn node disk accounting", `eve exec kube kubectl -n longhorn-system get nodes.longhorn.io -o yaml 2>/dev/null | grep -aE "name:|path:|storageMaximum|storageAvailable|storageScheduled|storageReserved|allowScheduling|diskUUID|type:|reason:|message:" || echo none`}, + {"longhorn node -o wide", `eve exec kube kubectl -n longhorn-system get nodes.longhorn.io -o wide 2>/dev/null || echo none`}, + {"longhorn storage settings", `eve exec kube kubectl -n longhorn-system get settings.longhorn.io -o custom-columns=NAME:.metadata.name,VALUE:.value 2>/dev/null | grep -aiE "NAME|reserved|over-provisioning|minimal-available|soft-anti-affinity" || echo none`}, + {"eve-kube-app pvc (requested sizes)", `eve exec kube kubectl -n eve-kube-app get pvc 2>/dev/null || echo none`}, + } + // The same pre-flight baseosmgr runs to decide the conversion. When it declines, + // the controller carries only the one-line reason ("persist is too full to free + // the needed space"); the JSON carries the numbers behind it — needed/target/used + // bytes, the resize2fs floor estimate and the maxFullPercent policy — which is the + // difference between knowing that it declined and knowing why. + if disk, err := bootDiskPath(device); err == nil { + probes = append(probes, struct{ what, script string }{ + "storage-resizer check (conversion pre-flight)", + "eve exec pillar /usr/bin/storage-resizer check --disk " + disk + " --json 2>&1 || echo none", + }) + } else { + log.Errorf("[persist:%s] storage-resizer check: could not resolve boot disk: %v", label, err) + } + for _, p := range probes { + out, errOut, err := device.RunShellScript(p.script, 60*time.Second, 0) + log.Errorf("[persist:%s] %s:\n%s%s(err=%v)", label, p.what, strings.TrimSpace(out), errOut, err) + } +} + +// startPersistSampler runs capturePersist in the background every interval (bounded) +// so /persist + Longhorn disk accounting can be watched evolving while k3s/Longhorn/ +// CDI start after the conversion. Returns a stop func (call once, e.g. via defer). +func startPersistSampler(device *evetest.EdgeDevice, interval time.Duration) (stop func()) { + done := make(chan struct{}) + go func() { + for n := 0; n < 13; n++ { + capturePersist(device, fmt.Sprintf("post-conversion startup sample #%d", n)) + select { + case <-done: + return + case <-time.After(interval): + } + } + }() + return func() { close(done) } +} + +// partsLsblk enumerates partition sizes across ALL block devices (not a fixed +// /dev/sda — evetest's broker QEMU presents the boot disk as virtio /dev/vda). +// We identify partitions by PARTLABEL, so the parent disk name doesn't matter. +const partsLsblk = "eve exec pillar lsblk -b -P -o NAME,PARTLABEL,PARTUUID,SIZE" + +// runEVE runs a command on EVE and returns cleaned stdout (dropping empty and +// level=… lines) plus any error, WITHOUT asserting — so callers can retry, since +// EVE SSH/pillar can be briefly unavailable right after a reboot or the conversion. +func runEVE(device *evetest.EdgeDevice, script string) (string, error) { + stdout, _, err := device.RunShellScript(script, eveShellTimeout, 0) + if err != nil { + return "", err + } + var lines []string + for _, l := range strings.Split(stdout, "\n") { + if strings.TrimSpace(l) == "" || strings.Contains(l, "level=") { + continue + } + lines = append(lines, l) + } + return strings.Join(lines, "\n"), nil +} + +// assertSmallGeometry asserts ESP/IMGA/IMGB are small and no ESP-B exists yet +// (mirror capture-partitions.sh assert-small). Retried: it also runs right after +// the kvm→kvm reboot. +func assertSmallGeometry(t Gomega, device *evetest.EdgeDevice) { + t.Eventually(func(g Gomega) { + out, err := runEVE(device, partsLsblk) + g.Expect(err).NotTo(HaveOccurred()) + // ESP-B carries the fresh-install GUID ...30056; it exists only after the grow. + g.Expect(strings.ToLower(out)).NotTo(ContainSubstring("30056"), + "SMALL geometry must not have an ESP-B yet:\n%s", out) + imga := partSizeBytes(out, "IMGA") + imgb := partSizeBytes(out, "IMGB") + const oneGiB = int64(1) << 30 + g.Expect(imga).To(BeNumerically(">", 0), "could not read IMGA size:\n%s", out) + g.Expect(imga).To(BeNumerically("<", oneGiB), "IMGA is not SMALL (%d bytes):\n%s", imga, out) + g.Expect(imgb).To(BeNumerically("<", oneGiB), "IMGB is not SMALL (%d bytes):\n%s", imgb, out) + }, 3*time.Minute, 10*time.Second).Should(Succeed()) +} + +// assertGrownShrink asserts the LARGE layout after the conversion: the reserved +// ESP-B was created and IMGA/IMGB grew past 8 GiB (mirror capture-partitions.sh +// assert-grown). That the shrink path (not grow) was taken is confirmed earlier by +// assertCheckDecision(shrink). Retried: it runs right after the EVE-k boot. +func assertGrownShrink(t Gomega, device *evetest.EdgeDevice) { + t.Eventually(func(g Gomega) { + out, err := runEVE(device, partsLsblk) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(strings.ToLower(out)).To(ContainSubstring("30056"), + "grow must have created the reserved ESP-B (GUID ...30056):\n%s", out) + const eightGiB = int64(8) << 30 + g.Expect(partSizeBytes(out, "IMGA")).To(BeNumerically(">=", eightGiB), "IMGA did not grow to LARGE:\n%s", out) + g.Expect(partSizeBytes(out, "IMGB")).To(BeNumerically(">=", eightGiB), "IMGB did not grow to LARGE:\n%s", out) + }, 5*time.Minute, 10*time.Second).Should(Succeed()) +} + +// partSizeBytes extracts the SIZE of the partition with PARTLABEL==label from a +// `lsblk -b -P` dump. +func partSizeBytes(lsblk, label string) int64 { + for _, line := range strings.Split(lsblk, "\n") { + if !strings.Contains(line, fmt.Sprintf("PARTLABEL=%q", label)) { + continue + } + for _, field := range strings.Fields(line) { + if strings.HasPrefix(field, "SIZE=") { + var n int64 + fmt.Sscanf(strings.Trim(strings.TrimPrefix(field, "SIZE="), `"`), "%d", &n) + return n + } + } + } + return 0 +} + +// bootDiskPath resolves the boot disk (the parent of the IMGA partition), e.g. +// /dev/vda under evetest's virtio QEMU. +func bootDiskPath(device *evetest.EdgeDevice) (string, error) { + out, err := runEVE(device, `eve exec pillar sh -c 'lsblk -ndo pkname $(findfs PARTLABEL=IMGA)'`) + if err != nil { + return "", err + } + name := strings.TrimSpace(out) + if name == "" { + return "", fmt.Errorf("could not resolve boot disk (IMGA parent)") + } + return "/dev/" + name, nil +} + +// assertCheckDecision asserts storage-resizer's pre-flight check returns the +// expected decision on the live boot disk (mirror assert-check-decision.sh). +func assertCheckDecision(t Gomega, device *evetest.EdgeDevice, want string) { + t.Eventually(func(g Gomega) { + disk, err := bootDiskPath(device) + g.Expect(err).NotTo(HaveOccurred()) + out, err := runEVE(device, "eve exec pillar /usr/bin/storage-resizer check --disk "+disk+" --json") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(out).To(ContainSubstring(fmt.Sprintf("%q", want)), + "storage-resizer check did not decide %q:\n%s", want, out) + }, 3*time.Minute, 10*time.Second).Should(Succeed()) +} + +// settleVaultLocal reboots through the controller until the vault reports a local +// TPM unlock (UnlockMethod==1); a controller-key unlock (2) triggers a reboot, +// no-tpm (3) fails (mirror settle-vault-local.sh). +func settleVaultLocal(t Gomega, device *evetest.EdgeDevice) { + log := evetest.Logger() + readUnlock := func() string { + out, _, err := device.RunShellScript( + `eve exec pillar sh -c "cat /run/vaultmgr/VaultStatus/*.json 2>/dev/null"`, eveShellTimeout, 0) + if err != nil { + return "" + } + switch { + case strings.Contains(out, `"UnlockMethod":1`): + return "local" + case strings.Contains(out, `"UnlockMethod":2`): + return "controller" + case strings.Contains(out, `"UnlockMethod":3`): + return "no-tpm" + } + return "" + } + for attempt := 1; attempt <= 4; attempt++ { + var decided string + t.Eventually(func() string { decided = readUnlock(); return decided }, 12*time.Minute, 10*time.Second). + ShouldNot(BeEmpty(), "vault never reported an unlock method") + switch decided { + case "local": + log.Infof("vault settled: local TPM unlock") + return + case "no-tpm": + t.Expect(decided).NotTo(Equal("no-tpm"), "vault is no-tpm; this test requires TPM") + return + default: // controller-key: reboot to re-seal + log.Infof("vault unlock=controller-key (attempt %d); rebooting to re-seal", attempt) + device.RequestReboot(true) + } + } + t.Expect(false).To(BeTrue(), "vault did not settle to a local unlock after 4 reboots") +} + +// waitVolumemgrReady waits for volumemgr Initialized:true (mirror +// wait-for-volumemgr-ready.sh). The budget must exceed volumemgr's own pre-publish +// block on a slow EVE-k bring-up: up to 20m in WaitForKubernetes (node + KubeVirt + +// Longhorn) plus up to 20m more in storageWait before any VolumeMgrStatus is +// published, so 45m could expire right as volumemgr is released (eden 29346e15). +func waitVolumemgrReady(t Gomega, device *evetest.EdgeDevice) { + t.Eventually(func() string { + out, _, _ := device.RunShellScript( + "eve exec pillar cat /run/volumemgr/VolumeMgrStatus/volumemgr.json 2>/dev/null", eveShellTimeout, 0) + return out + }, 60*time.Minute, 15*time.Second).Should(ContainSubstring(`"Initialized":true`), + "volumemgr did not reach Initialized:true") +} + +// waitLonghornSC waits for the longhorn StorageClass (mirror wait-for-longhorn-sc.sh). +func waitLonghornSC(t Gomega, device *evetest.EdgeDevice) { + t.Eventually(func() string { + out, _, _ := device.RunShellScript("eve exec kube kubectl get sc 2>/dev/null", eveShellTimeout, 0) + return out + }, 50*time.Minute, 30*time.Second).Should(ContainSubstring("longhorn"), + "longhorn StorageClass not ready") +} + +// waitAppHasRoutableIPv4 blocks until the app reports at least one routable +// (non-link-local) IPv4 address in its ZInfoApp, or fatals after timeout. This is +// the app-network readiness signal: without a reported IPv4 the harness cannot +// build a direct app-IP SSH endpoint, so a still-converging app looks the same as +// an unreachable one. It logs the address it settles on. +func waitAppHasRoutableIPv4(t Gomega, device *evetest.EdgeDevice, appUUID uuid.UUID, timeout time.Duration) { + t.Eventually(func(t Gomega) { + info := device.GetAppInfo(appUUID) + t.Expect(info).NotTo(BeNil()) + var found string + for _, netInfo := range info.GetNetwork() { + for _, ipStr := range netInfo.GetIPAddrs() { + ip := net.ParseIP(ipStr) + if ip != nil && ip.To4() != nil && !ip.IsLinkLocalUnicast() { + found = ipStr + } + } + } + t.Expect(found).NotTo(BeEmpty(), "app reports no routable IPv4 yet") + evetest.Logger().Infof("app reports routable IPv4 %s", found) + }, timeout, 5*time.Second).Should(Succeed()) +} + +// assertAppSSH asserts the app is reachable over SSH (returns non-empty hostname). +func assertAppSSH(t Gomega, device *evetest.EdgeDevice, appUUID uuid.UUID, auth evetest.AuthMethod) { + t.Eventually(func(t Gomega) { + out, _, err := device.RunShellScriptInsideApp(appUUID, auth, "hostname", sshTimeout, 0) + t.Expect(err).NotTo(HaveOccurred()) + t.Expect(strings.TrimSpace(out)).NotTo(BeEmpty()) + }, 3*time.Minute, 5*time.Second).Should(Succeed()) +} + +// writeVolumeMarker writes a unique marker into the app's mounted data volume and +// fsyncs it, failing if the volume isn't mounted at mountDir. Run on EVE-kvm, +// where the runx shim formats and mounts the blank volume at MountDir. +func writeVolumeMarker(t Gomega, device *evetest.EdgeDevice, appUUID uuid.UUID, auth evetest.AuthMethod, mountDir, marker string) { + script := fmt.Sprintf( + "grep -q ' %s ' /proc/mounts || { echo NOT-MOUNTED; cat /proc/mounts; exit 1; }; "+ + "echo %q > %s/marker && sync && echo WROTE-OK", + mountDir, marker, mountDir) + t.Eventually(func(t Gomega) { + out, _, err := device.RunShellScriptInsideApp(appUUID, auth, script, 60*time.Second, 0) + t.Expect(err).NotTo(HaveOccurred()) + t.Expect(out).To(ContainSubstring("WROTE-OK")) + }, 2*time.Minute, 5*time.Second).Should(Succeed()) +} + +// assertVolumeMarker verifies the marker survived the conversion by reading it off +// the data volume's RAW block device from inside the app — EVE-k does not +// auto-mount the container data volume at MountDir (lf-edge/eve#6145). The data +// volume is a virtio disk after the rootfs (/dev/vd[b-z]); grep it for the marker. +func assertVolumeMarker(t Gomega, device *evetest.EdgeDevice, appUUID uuid.UUID, auth evetest.AuthMethod, marker string) { + script := fmt.Sprintf( + "for d in /dev/vd[b-z] /dev/sd[b-z]; do [ -b \"$d\" ] || continue; "+ + "grep -aq %q \"$d\" 2>/dev/null && { echo FOUND-ON $d; exit 0; }; done; "+ + "echo NOT-FOUND; ls -l /dev/vd* /dev/sd* 2>/dev/null; lsblk 2>/dev/null; blkid 2>/dev/null; exit 1", + marker) + t.Eventually(func(t Gomega) { + out, _, err := device.RunShellScriptInsideApp(appUUID, auth, script, 90*time.Second, 0) + t.Expect(err).NotTo(HaveOccurred()) + t.Expect(out).To(ContainSubstring("FOUND-ON")) + }, 3*time.Minute, 10*time.Second).Should(Succeed()) +} + +// isK3sReady reports whether the single k3s node is ready with healthy storage. +func isK3sReady(info *eveinfo.ZInfoKubeCluster) bool { + if info == nil || len(info.Nodes) != 1 { + return false + } + if info.Storage.Health != eveinfo.ServiceStatus_SERVICE_STATUS_HEALTHY { + return false + } + for _, cond := range info.Nodes[0].GetConditions() { + if cond.GetType() == eveinfo.KubeNodeConditionType_KUBE_NODE_CONDITION_TYPE_READY { + return cond.GetSet() + } + } + return false +}