From 7cb9a804e8e88cebfc100ff376e33fa6409b5b73 Mon Sep 17 00:00:00 2001 From: eriknordmark Date: Thu, 23 Jul 2026 22:38:57 -0700 Subject: [PATCH 01/29] evetest: add volverify data-volume test app Adds a test application that writes a deterministic, self-verifying fill/delete pattern to an application volume and later checks it, to detect corruption caused by a watchdog-interrupted EVE-kvm to EVE-k offline filesystem shrink. It is the volume-content ground truth the resize soak pairs with the on-device fsck marker. Each block's content is a pure function of its logical identity (file id and block index within the file, not physical placement), so a successful shrink relocation verifies clean while a torn extent tree surfaces as a misplaced, zeroed, or torn block. A seeded op stream drives create/delete churn with a crash-safe two-slot committed index, so the writer resumes across reboots and the verifier classifies each file as ok, present-corrupt, orphaned (in lost+found), lost, or resurrected. verify accepts an off-volume committed-index floor so loss of the last work is still detected when fsck clears both the data and the on-volume bookkeeping. Ships as an ubuntu image driven over SSH, with unit tests and a root-gated loopback-ext4 fidelity script. Signed-off-by: eriknordmark Co-Authored-By: Claude Opus 4.8 (1M context) --- evetest/testapps/volverify/Dockerfile | 38 +++ evetest/testapps/volverify/Makefile | 16 + evetest/testapps/volverify/README.md | 49 +++ .../testapps/volverify/cmd/volverify/main.go | 116 +++++++ evetest/testapps/volverify/go.mod | 3 + evetest/testapps/volverify/init.sh | 10 + .../volverify/internal/verify/block.go | 188 ++++++++++++ .../volverify/internal/verify/block_test.go | 68 +++++ .../volverify/internal/verify/commit.go | 129 ++++++++ .../volverify/internal/verify/commit_test.go | 46 +++ .../volverify/internal/verify/config.go | 54 ++++ .../volverify/internal/verify/engine.go | 287 ++++++++++++++++++ .../volverify/internal/verify/engine_test.go | 233 ++++++++++++++ .../volverify/internal/verify/model.go | 103 +++++++ .../volverify/internal/verify/opstream.go | 81 +++++ .../internal/verify/opstream_test.go | 75 +++++ .../volverify/internal/verify/report.go | 84 +++++ .../testapps/volverify/internal/verify/rng.go | 28 ++ .../volverify/internal/verify/util.go | 25 ++ .../volverify/scripts/loopback-ext4-test.sh | 91 ++++++ 20 files changed, 1724 insertions(+) create mode 100644 evetest/testapps/volverify/Dockerfile create mode 100644 evetest/testapps/volverify/Makefile create mode 100644 evetest/testapps/volverify/README.md create mode 100644 evetest/testapps/volverify/cmd/volverify/main.go create mode 100644 evetest/testapps/volverify/go.mod create mode 100755 evetest/testapps/volverify/init.sh create mode 100644 evetest/testapps/volverify/internal/verify/block.go create mode 100644 evetest/testapps/volverify/internal/verify/block_test.go create mode 100644 evetest/testapps/volverify/internal/verify/commit.go create mode 100644 evetest/testapps/volverify/internal/verify/commit_test.go create mode 100644 evetest/testapps/volverify/internal/verify/config.go create mode 100644 evetest/testapps/volverify/internal/verify/engine.go create mode 100644 evetest/testapps/volverify/internal/verify/engine_test.go create mode 100644 evetest/testapps/volverify/internal/verify/model.go create mode 100644 evetest/testapps/volverify/internal/verify/opstream.go create mode 100644 evetest/testapps/volverify/internal/verify/opstream_test.go create mode 100644 evetest/testapps/volverify/internal/verify/report.go create mode 100644 evetest/testapps/volverify/internal/verify/rng.go create mode 100644 evetest/testapps/volverify/internal/verify/util.go create mode 100755 evetest/testapps/volverify/scripts/loopback-ext4-test.sh diff --git a/evetest/testapps/volverify/Dockerfile b/evetest/testapps/volverify/Dockerfile new file mode 100644 index 00000000000..957f55ddc00 --- /dev/null +++ b/evetest/testapps/volverify/Dockerfile @@ -0,0 +1,38 @@ +# Copyright (c) 2026 Zededa, Inc. +# SPDX-License-Identifier: Apache-2.0 + +ARG GOLANG_VERSION=1.25 + +# hadolint ignore=DL3029 +FROM --platform=$BUILDPLATFORM golang:${GOLANG_VERSION} AS build + +ARG TARGETOS +ARG TARGETARCH + +WORKDIR /app + +COPY go.mod ./ +RUN go mod download + +COPY . . +RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -o /volverify ./cmd/volverify + +FROM ubuntu:24.04 + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# hadolint ignore=DL3008 +RUN apt-get update && \ + apt-get install -y --no-install-recommends openssh-server e2fsprogs && \ + rm -rf /var/lib/apt/lists/* && \ + mkdir /run/sshd + +RUN echo 'root:testpassword' | chpasswd +RUN sed -i 's/#PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config && \ + sed -i 's/#PasswordAuthentication.*/PasswordAuthentication yes/' /etc/ssh/sshd_config +EXPOSE 22 + +COPY --from=build /volverify /usr/local/bin/volverify +COPY init.sh / + +CMD ["/bin/bash", "/init.sh"] diff --git a/evetest/testapps/volverify/Makefile b/evetest/testapps/volverify/Makefile new file mode 100644 index 00000000000..eacb99e9226 --- /dev/null +++ b/evetest/testapps/volverify/Makefile @@ -0,0 +1,16 @@ +# Copyright (c) 2026 Zededa, Inc. +# SPDX-License-Identifier: Apache-2.0 + +EVETEST_ORG ?= lfedge +IMAGE = $(EVETEST_ORG)/evetest-volverify +# Update VERSION whenever there is a change made to this app. +VERSION ?= 1.0 + +DOCKER_TARGET ?= load +DOCKER_PLATFORM ?= $(shell uname -s | tr '[A-Z]' '[a-z]')/$(subst aarch64,arm64,$(subst x86_64,amd64,$(shell uname -m))) + +build: + docker buildx build \ + --$(DOCKER_TARGET) \ + --platform $(DOCKER_PLATFORM) \ + -t $(IMAGE):$(VERSION) . diff --git a/evetest/testapps/volverify/README.md b/evetest/testapps/volverify/README.md new file mode 100644 index 00000000000..bb72c40acf8 --- /dev/null +++ b/evetest/testapps/volverify/README.md @@ -0,0 +1,49 @@ +# volverify — application-volume corruption verifier + +A test-app tool that writes a deterministic, self-verifying fill/delete pattern to +an application volume and later checks it, to detect corruption caused by a +watchdog-interrupted EVE-kvm→EVE-k offline filesystem shrink. It is the ground +truth the soak harness pairs with the resize fsck marker. + +Design: `~/notes/kvm-to-k-appvol-shrink-soak-design.md` (§4). Deployed inside the +evetest ubuntu app and driven over SSH via `RunShellScriptInsideApp`. + +## What it does + +- **Layer 1** — every 4 KiB block is `AES-CTR(key=derive(fileID), iv=blockIndex)` + plus a header carrying the *logical* identity `(fileID, blockIndex)` and CRCs. + The identity is logical (file offset ÷ block size), never physical disk + placement — placement changes by design when the shrink relocates the P3 tail, + and the verifier checks that each logical read still yields the identity's + bytes. Reproducible, incompressible, non-zero, so a zeroed/torn/misplaced block + is unambiguous. +- **Layer 2** — a `masterSeed`-seeded PRNG drives a deterministic + create/delete/mkdir/rmdir op stream; the writer fsyncs and advances a 2-slot + ping-pong committed-index every `--commit-every` ops. The verifier replays the + stream to the committed index and classifies each expected file: + `ok / present-corrupt / orphaned (in lost+found) / lost / resurrected`. + +## Usage + +```sh +volverify write --dir /mnt/data --seed 42 --ops 100000 # crash-safe, resumable +volverify verify --dir /mnt/data --seed 42 --ops 100000 # exits non-zero on any anomaly +``` + +Both invocations must use the same `--seed` and size flags. `write` is idempotent +across reboots (it resumes from the committed index). + +`verify --expect-committed ` supplies an off-volume floor on the committed op +index. The on-volume commit slots live on the same volume being shrunk, so fsck can +clear them along with the last files' data — which would make the verifier expect +nothing and mask the loss. Since the soak harness runs `write` to completion before +the shrink, it knows the true high-water mark and passes it here, so the last work is +still expected (and its loss flagged). + +## Build + +```sh +make build # docker image lfedge/evetest-volverify:1.0 +GOWORK=off go test ./... # unit tests (fault-injection classification) +sudo ./scripts/loopback-ext4-test.sh # on-fs fidelity check (real ext4 + e2fsck) +``` diff --git a/evetest/testapps/volverify/cmd/volverify/main.go b/evetest/testapps/volverify/cmd/volverify/main.go new file mode 100644 index 00000000000..0abeca762c7 --- /dev/null +++ b/evetest/testapps/volverify/cmd/volverify/main.go @@ -0,0 +1,116 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Command volverify writes and later verifies a deterministic, self-describing +// fill/delete pattern on an application volume, to detect corruption caused by a +// watchdog-interrupted EVE-kvm→EVE-k offline filesystem shrink. +// +// It is deployed inside the evetest test app and driven over SSH: +// +// volverify write --dir /mnt/data --seed 42 --ops 100000 +// volverify verify --dir /mnt/data --seed 42 --ops 100000 +// +// write is crash-safe and resumable: run it repeatedly across reboots. verify +// exits non-zero when it finds any anomaly and prints a machine-readable summary. +package main + +import ( + "flag" + "fmt" + "os" + + "github.com/lf-edge/eve/evetest/testapps/volverify/internal/verify" +) + +func main() { + if len(os.Args) < 2 { + usage() + os.Exit(2) + } + cmd := os.Args[1] + fs := flag.NewFlagSet(cmd, flag.ExitOnError) + dir := fs.String("dir", "", "volume mount point to operate on (required)") + def := verify.DefaultConfig() + seed := fs.Uint64("seed", def.Seed, "master seed for the op stream") + ops := fs.Uint64("ops", def.Ops, "number of ops to apply / expect") + commitEvery := fs.Uint64("commit-every", def.CommitEvery, "fsync + commit cadence in ops") + blockSize := fs.Int("block-size", def.BlockSize, "on-disk block size in bytes") + dirFanout := fs.Int("dir-fanout", def.DirFanout, "per-level file-tree fan-out") + smallBlocks := fs.Int("small-blocks", def.SmallBlocks, "max blocks for a small file") + medBlocks := fs.Int("med-blocks", def.MedBlocks, "max blocks for a medium file") + maxBlocks := fs.Int("max-blocks", def.MaxBlocks, "max blocks for a large file") + expectCommitted := fs.Int64("expect-committed", def.ExpectCommitted, + "verify: floor on the committed op index (harness high-water mark); -1 = trust on-volume commit only") + _ = fs.Parse(os.Args[2:]) + + if *dir == "" { + fmt.Fprintln(os.Stderr, "error: --dir is required") + os.Exit(2) + } + cfg := verify.Config{ + Seed: *seed, + BlockSize: *blockSize, + Ops: *ops, + CommitEvery: *commitEvery, + DirFanout: *dirFanout, + SmallBlocks: *smallBlocks, + MedBlocks: *medBlocks, + MaxBlocks: *maxBlocks, + ExpectCommitted: *expectCommitted, + } + + switch cmd { + case "write": + w, err := verify.NewWriter(*dir, cfg) + if err != nil { + fatal(err) + } + if err := w.Run(); err != nil { + fatal(err) + } + fmt.Println("write: complete") + case "verify": + rep, err := verify.Verify(*dir, cfg) + if err != nil { + fatal(err) + } + fmt.Println(rep.String()) + for _, a := range rep.Anomalies { + fmt.Printf(" ANOMALY file=%d verdict=%s path=%s expBlocks=%d sizeMismatch=%v blocks=%v\n", + a.FileID, a.Verdict, a.Path, a.ExpectBlocks, a.SizeMismatch, blockCountsString(a.BlockCounts)) + } + for _, id := range rep.Resurrected { + fmt.Printf(" ANOMALY resurrected file=%d\n", id) + } + if !rep.Clean() { + os.Exit(1) + } + fmt.Println("verify: clean") + default: + usage() + os.Exit(2) + } +} + +func blockCountsString(m map[verify.BlockStatus]int) string { + out := "" + for s, n := range m { + if s == verify.BlockOK { + continue + } + out += fmt.Sprintf("%s=%d ", s, n) + } + if out == "" { + return "-" + } + return out +} + +func fatal(err error) { + fmt.Fprintln(os.Stderr, "error:", err) + os.Exit(1) +} + +func usage() { + fmt.Fprintln(os.Stderr, "usage: volverify --dir [--seed N --ops N ...]") +} diff --git a/evetest/testapps/volverify/go.mod b/evetest/testapps/volverify/go.mod new file mode 100644 index 00000000000..efbed6df6cd --- /dev/null +++ b/evetest/testapps/volverify/go.mod @@ -0,0 +1,3 @@ +module github.com/lf-edge/eve/evetest/testapps/volverify + +go 1.25 diff --git a/evetest/testapps/volverify/init.sh b/evetest/testapps/volverify/init.sh new file mode 100755 index 00000000000..9ed55971073 --- /dev/null +++ b/evetest/testapps/volverify/init.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# Copyright (c) 2026 Zededa, Inc. +# SPDX-License-Identifier: Apache-2.0 + +# The volverify binary is not a daemon: the test drives it on demand over SSH +# (RunShellScriptInsideApp). Start sshd and keep the container alive. + +/usr/sbin/sshd +exec sleep infinity diff --git a/evetest/testapps/volverify/internal/verify/block.go b/evetest/testapps/volverify/internal/verify/block.go new file mode 100644 index 00000000000..641071f2c39 --- /dev/null +++ b/evetest/testapps/volverify/internal/verify/block.go @@ -0,0 +1,188 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Package verify implements the two-layer self-verifying fill/delete pattern used +// to detect application-volume corruption caused by a watchdog-interrupted +// EVE-kvm→EVE-k offline filesystem shrink (design: kvm-to-k-appvol-shrink-soak). +// +// Layer 1 (this file) gives every on-disk block content that is a pure function +// of the file's *logical* identity — (fileID, logical block index within the +// file), never the physical disk placement. Physical placement changes by design +// when resize2fs relocates the P3 tail, so the verifier reads a file back through +// the filesystem and checks that each logical block still yields the bytes its +// identity implies. A block that reads back as another file's or another index's +// content (an extent tree torn during relocation) is caught by the identity in +// its header. +package verify + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/sha256" + "encoding/binary" + "hash/crc32" +) + +const ( + // blockMagic marks a well-formed block header ("VVB1" little-endian). + blockMagic = 0x31425656 + // headerLen is the fixed prefix each block reserves for its header. + headerLen = 32 +) + +var castagnoli = crc32.MakeTable(crc32.Castagnoli) + +// deriveKey returns the 16-byte AES key for a file's block content. It is a pure +// function of fileID, so a block's expected bytes depend only on file identity +// and logical index — never on stored state, so no content manifest is needed. +func deriveKey(fileID uint64) []byte { + var b [8]byte + binary.LittleEndian.PutUint64(b[:], fileID) + sum := sha256.Sum256(append([]byte("volverify-key-v1"), b[:]...)) + return sum[:16] +} + +// blockBody fills dst with the deterministic, incompressible, non-zero body for +// (fileID, blockIndex): an AES-CTR keystream over zeros. blockIndex occupies the +// high half of the IV so per-block counter ranges never overlap. +func blockBody(fileID, blockIndex uint64, dst []byte) { + blk, err := aes.NewCipher(deriveKey(fileID)) + if err != nil { + panic(err) // deriveKey always yields a valid 16-byte key + } + var iv [aes.BlockSize]byte + binary.BigEndian.PutUint64(iv[0:8], blockIndex) + ctr := cipher.NewCTR(blk, iv[:]) + for i := range dst { + dst[i] = 0 + } + ctr.XORKeyStream(dst, dst) +} + +// BuildBlock returns a blockSize-byte block for the logical (fileID, blockIndex). +func BuildBlock(fileID, blockIndex uint64, blockSize int) []byte { + buf := make([]byte, blockSize) + body := buf[headerLen:] + blockBody(fileID, blockIndex, body) + binary.LittleEndian.PutUint32(buf[0:4], blockMagic) + binary.LittleEndian.PutUint64(buf[4:12], fileID) + binary.LittleEndian.PutUint64(buf[12:20], blockIndex) + binary.LittleEndian.PutUint32(buf[20:24], uint32(len(body))) + binary.LittleEndian.PutUint32(buf[24:28], crc32.Checksum(body, castagnoli)) + binary.LittleEndian.PutUint32(buf[28:32], crc32.Checksum(buf[0:28], castagnoli)) + return buf +} + +// BlockStatus classifies one on-disk block against its expected logical identity. +type BlockStatus int + +const ( + // BlockOK means the block content matches the expected keystream. + BlockOK BlockStatus = iota + // BlockZeroed means the block is all zero — fsck cleared it or a torn/short write. + BlockZeroed + // BlockGarbage means there is no valid header (bad magic/header-CRC/length). + BlockGarbage + // BlockMisplaced means a valid header carrying another file's or index's identity. + BlockMisplaced + // BlockTorn means the header is intact but the body CRC disagrees with the body. + BlockTorn + // BlockAltered means the block is self-consistent but its body != expected keystream. + BlockAltered +) + +func (s BlockStatus) String() string { + switch s { + case BlockOK: + return "ok" + case BlockZeroed: + return "zeroed" + case BlockGarbage: + return "garbage" + case BlockMisplaced: + return "misplaced" + case BlockTorn: + return "torn" + case BlockAltered: + return "altered" + } + return "unknown" +} + +// BlockResult is a classified block plus, for a misplaced block, the logical +// identity its header actually carried (so cross-file smearing can be traced). +type BlockResult struct { + Status BlockStatus + FoundFile uint64 + FoundIndex uint64 +} + +// parseHeader extracts a block's declared logical identity. ok is false when the +// block has no usable header (bad magic, header-CRC mismatch, or wrong body length). +func parseHeader(buf []byte, blockSize int) (fileID, blockIndex uint64, ok bool) { + if len(buf) < headerLen { + return 0, 0, false + } + if binary.LittleEndian.Uint32(buf[0:4]) != blockMagic { + return 0, 0, false + } + if binary.LittleEndian.Uint32(buf[28:32]) != crc32.Checksum(buf[0:28], castagnoli) { + return 0, 0, false + } + if int(binary.LittleEndian.Uint32(buf[20:24])) != blockSize-headerLen { + return 0, 0, false + } + return binary.LittleEndian.Uint64(buf[4:12]), binary.LittleEndian.Uint64(buf[12:20]), true +} + +// VerifyBlock classifies buf against the block expected at logical (expFile, expIndex). +func VerifyBlock(buf []byte, expFile, expIndex uint64, blockSize int) BlockResult { + if isZero(buf) { + return BlockResult{Status: BlockZeroed} + } + fid, bidx, ok := parseHeader(buf, blockSize) + if !ok { + return BlockResult{Status: BlockGarbage} + } + if fid != expFile || bidx != expIndex { + return BlockResult{Status: BlockMisplaced, FoundFile: fid, FoundIndex: bidx} + } + body := buf[headerLen:] + if binary.LittleEndian.Uint32(buf[24:28]) != crc32.Checksum(body, castagnoli) { + return BlockResult{Status: BlockTorn} + } + exp := make([]byte, len(body)) + blockBody(expFile, expIndex, exp) + if !bytesEqual(exp, body) { + return BlockResult{Status: BlockAltered} + } + return BlockResult{Status: BlockOK} +} + +// headerFileID reports the file identity a block claims, for the lost+found scan +// (design §4.3): an orphaned file is identified by reading its first block header. +func headerFileID(buf []byte, blockSize int) (uint64, bool) { + fid, _, ok := parseHeader(buf, blockSize) + return fid, ok +} + +func isZero(b []byte) bool { + for _, x := range b { + if x != 0 { + return false + } + } + return true +} + +func bytesEqual(a, b []byte) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} diff --git a/evetest/testapps/volverify/internal/verify/block_test.go b/evetest/testapps/volverify/internal/verify/block_test.go new file mode 100644 index 00000000000..722570be064 --- /dev/null +++ b/evetest/testapps/volverify/internal/verify/block_test.go @@ -0,0 +1,68 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package verify + +import "testing" + +const testBlockSize = 256 + +func TestBlockRoundTrip(t *testing.T) { + b := BuildBlock(7, 3, testBlockSize) + if res := VerifyBlock(b, 7, 3, testBlockSize); res.Status != BlockOK { + t.Fatalf("roundtrip: got %v, want ok", res.Status) + } + // The body must be non-zero and reproducible. + b2 := BuildBlock(7, 3, testBlockSize) + if !bytesEqual(b, b2) { + t.Fatal("block content not reproducible for same identity") + } + if isZero(b[headerLen:]) { + t.Fatal("block body is all-zero") + } +} + +func TestBlockZeroed(t *testing.T) { + b := make([]byte, testBlockSize) + if res := VerifyBlock(b, 1, 0, testBlockSize); res.Status != BlockZeroed { + t.Fatalf("got %v, want zeroed", res.Status) + } +} + +func TestBlockGarbage(t *testing.T) { + b := BuildBlock(1, 0, testBlockSize) + b[1] ^= 0xff // corrupt the magic + if res := VerifyBlock(b, 1, 0, testBlockSize); res.Status != BlockGarbage { + t.Fatalf("got %v, want garbage", res.Status) + } +} + +func TestBlockMisplaced(t *testing.T) { + // A block written for file 9 read where file 1 is expected. + b := BuildBlock(9, 0, testBlockSize) + res := VerifyBlock(b, 1, 0, testBlockSize) + if res.Status != BlockMisplaced || res.FoundFile != 9 { + t.Fatalf("got %v foundFile=%d, want misplaced foundFile=9", res.Status, res.FoundFile) + } + // Same file, wrong index is also misplaced. + b2 := BuildBlock(1, 5, testBlockSize) + if res := VerifyBlock(b2, 1, 0, testBlockSize); res.Status != BlockMisplaced || res.FoundIndex != 5 { + t.Fatalf("got %v foundIndex=%d, want misplaced foundIndex=5", res.Status, res.FoundIndex) + } +} + +func TestBlockTorn(t *testing.T) { + b := BuildBlock(1, 0, testBlockSize) + // Flip a body byte without updating the body CRC: the header still parses. + b[headerLen+10] ^= 0xff + if res := VerifyBlock(b, 1, 0, testBlockSize); res.Status != BlockTorn { + t.Fatalf("got %v, want torn", res.Status) + } +} + +func TestBlockHeaderFileID(t *testing.T) { + b := BuildBlock(42, 0, testBlockSize) + if id, ok := headerFileID(b, testBlockSize); !ok || id != 42 { + t.Fatalf("got id=%d ok=%v, want 42 true", id, ok) + } +} diff --git a/evetest/testapps/volverify/internal/verify/commit.go b/evetest/testapps/volverify/internal/verify/commit.go new file mode 100644 index 00000000000..39d29c81fd1 --- /dev/null +++ b/evetest/testapps/volverify/internal/verify/commit.go @@ -0,0 +1,129 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package verify + +import ( + "encoding/binary" + "errors" + "fmt" + "hash/crc32" + "os" + "path/filepath" + "syscall" +) + +// commitDirName holds the two ping-pong committed-index slots on the volume. +const commitDirName = ".vv-commit" + +const ( + commitMagic = 0x56564349 // "VVCI" little-endian + commitRecLen = 24 // magic(4) + generation(8) + index(8) + crc(4) +) + +// commitRecord is the durable "everything through op Index is fsynced" marker. +// generation strictly increases so recovery can pick the newest valid slot even +// when a crash tore the most recent write (design §4.2). +type commitRecord struct { + generation uint64 + index int64 +} + +func slotPath(volDir string, slot int) string { + return filepath.Join(volDir, commitDirName, fmt.Sprintf("commit.%d", slot)) +} + +// writeCommit atomically publishes rec into the ping-pong slot chosen by its +// generation parity, fsyncing the file and its directory before returning. +func writeCommit(volDir string, rec commitRecord) error { + dir := filepath.Join(volDir, commitDirName) + if err := os.MkdirAll(dir, 0o755); err != nil { + return err + } + var buf [commitRecLen]byte + binary.LittleEndian.PutUint32(buf[0:4], commitMagic) + binary.LittleEndian.PutUint64(buf[4:12], rec.generation) + binary.LittleEndian.PutUint64(buf[12:20], uint64(rec.index)) + binary.LittleEndian.PutUint32(buf[20:24], crc32.Checksum(buf[0:20], castagnoli)) + + path := slotPath(volDir, int(rec.generation%2)) + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) + if err != nil { + return err + } + if _, err := f.Write(buf[:]); err != nil { + f.Close() + return err + } + if err := f.Sync(); err != nil { + f.Close() + return err + } + if err := f.Close(); err != nil { + return err + } + return fsyncDir(dir) +} + +// readCommit returns the highest-generation valid slot's index, or -1 when no +// valid slot exists (fresh volume, or both slots torn). +func readCommit(volDir string) int64 { + best := int64(-1) + bestGen := int64(-1) + for slot := 0; slot < 2; slot++ { + rec, ok := readSlot(slotPath(volDir, slot)) + if !ok { + continue + } + if int64(rec.generation) > bestGen { + bestGen = int64(rec.generation) + best = rec.index + } + } + return best +} + +func readSlot(path string) (commitRecord, bool) { + data, err := os.ReadFile(path) + if err != nil || len(data) != commitRecLen { + return commitRecord{}, false + } + if binary.LittleEndian.Uint32(data[0:4]) != commitMagic { + return commitRecord{}, false + } + if binary.LittleEndian.Uint32(data[20:24]) != crc32.Checksum(data[0:20], castagnoli) { + return commitRecord{}, false + } + return commitRecord{ + generation: binary.LittleEndian.Uint64(data[4:12]), + index: int64(binary.LittleEndian.Uint64(data[12:20])), + }, true +} + +// nextGeneration returns the generation to use for the next commit write. +func nextGeneration(volDir string) uint64 { + var maxGen int64 = -1 + for slot := 0; slot < 2; slot++ { + if rec, ok := readSlot(slotPath(volDir, slot)); ok { + if int64(rec.generation) > maxGen { + maxGen = int64(rec.generation) + } + } + } + return uint64(maxGen + 1) +} + +// fsyncDir flushes a directory entry so a rename/create is durable. +func fsyncDir(dir string) error { + d, err := os.Open(dir) + if err != nil { + return err + } + defer d.Close() + err = d.Sync() + // Some filesystems reject fsync on a directory; that is not fatal here. + if err != nil && (errors.Is(err, os.ErrInvalid) || errors.Is(err, syscall.EINVAL)) { + return nil + } + return err +} diff --git a/evetest/testapps/volverify/internal/verify/commit_test.go b/evetest/testapps/volverify/internal/verify/commit_test.go new file mode 100644 index 00000000000..2e2f22e68f7 --- /dev/null +++ b/evetest/testapps/volverify/internal/verify/commit_test.go @@ -0,0 +1,46 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package verify + +import ( + "os" + "testing" +) + +func TestCommitPingPong(t *testing.T) { + dir := t.TempDir() + if got := readCommit(dir); got != -1 { + t.Fatalf("fresh volume: got committed=%d, want -1", got) + } + for i := 0; i < 5; i++ { + gen := nextGeneration(dir) + if err := writeCommit(dir, commitRecord{generation: gen, index: int64(i * 10)}); err != nil { + t.Fatal(err) + } + } + if got := readCommit(dir); got != 40 { + t.Fatalf("got committed=%d, want 40", got) + } +} + +func TestCommitTornNewestSlotFallsBack(t *testing.T) { + dir := t.TempDir() + // generation 0 -> slot 0 (index 100), generation 1 -> slot 1 (index 200). + if err := writeCommit(dir, commitRecord{generation: 0, index: 100}); err != nil { + t.Fatal(err) + } + if err := writeCommit(dir, commitRecord{generation: 1, index: 200}); err != nil { + t.Fatal(err) + } + if got := readCommit(dir); got != 200 { + t.Fatalf("got %d, want 200", got) + } + // Tear the newest slot (slot 1); recovery must fall back to slot 0. + if err := os.WriteFile(slotPath(dir, 1), []byte("torn"), 0o644); err != nil { + t.Fatal(err) + } + if got := readCommit(dir); got != 100 { + t.Fatalf("after tearing newest slot: got %d, want 100", got) + } +} diff --git a/evetest/testapps/volverify/internal/verify/config.go b/evetest/testapps/volverify/internal/verify/config.go new file mode 100644 index 00000000000..e2544373c38 --- /dev/null +++ b/evetest/testapps/volverify/internal/verify/config.go @@ -0,0 +1,54 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package verify + +// Config parameterizes an op stream. The same Config (and Seed) must be supplied +// to the writer and to the later verifier, since both replay the identical stream. +type Config struct { + Seed uint64 // master seed for the deterministic op stream + BlockSize int // on-disk block size in bytes (must be > headerLen) + Ops uint64 // total number of ops the writer applies + CommitEvery uint64 // fsync + advance the committed index every this many ops + DirFanout int // per-level fan-out of the file placement tree + SmallBlocks int // upper bound (inclusive) on a "small" file, in blocks + MedBlocks int // upper bound on a "medium" file, in blocks + MaxBlocks int // upper bound on a "large" file, in blocks + + // ExpectCommitted is an off-volume floor on the committed op index for the + // verifier: the effective committed index is max(on-volume commit, + // ExpectCommitted). It closes the blind spot where fsck clears both the last + // files' data and the on-volume commit slots — the harness, which ran the + // writer to completion, supplies the true high-water mark so the verifier + // still expects (and thus flags loss of) the last work. -1 means "trust the + // on-volume commit only". Authoritative only when the caller knows the writer + // finished; a value above reality would over-expect. Ignored by the writer. + ExpectCommitted int64 +} + +// DefaultConfig returns a Config sized for on-device churn against a large blank +// volume: 4 KiB blocks and a heavy-tailed size mix (mostly KB, occasional +// MB/hundreds-MB) so allocated blocks scatter through the shrink evacuation zone. +func DefaultConfig() Config { + return Config{ + Seed: 1, + BlockSize: 4096, + Ops: 100000, + CommitEvery: 64, + DirFanout: 16, + SmallBlocks: 4, // <= 16 KiB + MedBlocks: 256, // <= 1 MiB + MaxBlocks: 65536, // <= 256 MiB + ExpectCommitted: -1, + } +} + +// valid reports whether the Config is self-consistent enough to run. +func (c Config) valid() bool { + return c.BlockSize > headerLen && + c.CommitEvery > 0 && + c.DirFanout > 0 && + c.SmallBlocks >= 1 && + c.MedBlocks >= c.SmallBlocks && + c.MaxBlocks >= c.MedBlocks +} diff --git a/evetest/testapps/volverify/internal/verify/engine.go b/evetest/testapps/volverify/internal/verify/engine.go new file mode 100644 index 00000000000..16b06db1672 --- /dev/null +++ b/evetest/testapps/volverify/internal/verify/engine.go @@ -0,0 +1,287 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package verify + +import ( + "fmt" + "io" + "os" + "path/filepath" +) + +// Writer applies the deterministic op stream to a volume, fsyncing and advancing +// the committed index every Config.CommitEvery ops so that, after an abrupt power +// loss, everything through the last committed index is durable (design §4.2). +type Writer struct { + volDir string + cfg Config +} + +// NewWriter returns a Writer that operates on the volume mounted at volDir. +func NewWriter(volDir string, cfg Config) (*Writer, error) { + if !cfg.valid() { + return nil, fmt.Errorf("invalid config: %+v", cfg) + } + return &Writer{volDir: volDir, cfg: cfg}, nil +} + +// Run applies ops until Config.Ops, resuming after a crash from the committed +// index. It is safe to call repeatedly across reboots on the same volume. +func (w *Writer) Run() error { + gen := newGenerator(w.cfg) + m := newModel(w.cfg) + + // Rebuild in-memory state up to the committed index without touching disk; + // this also advances the generator to the first uncommitted op. + committed := readCommit(w.volDir) + for n := uint64(0); int64(n) <= committed; n++ { + m.apply(gen.next(m, n)) + } + gen64 := nextGeneration(w.volDir) + + touchedFiles := make(map[string]bool) + touchedDirs := make(map[string]bool) + + commit := func(index uint64) error { + for p := range touchedFiles { + if err := fsyncFile(p); err != nil { + return err + } + } + for d := range touchedDirs { + if err := fsyncDir(d); err != nil { + return err + } + } + if err := writeCommit(w.volDir, commitRecord{generation: gen64, index: int64(index)}); err != nil { + return err + } + gen64++ + touchedFiles = make(map[string]bool) + touchedDirs = make(map[string]bool) + return nil + } + + start := uint64(committed + 1) + for n := start; n < w.cfg.Ops; n++ { + o := gen.next(m, n) + if err := w.applyOp(o, touchedFiles, touchedDirs); err != nil { + return fmt.Errorf("op %d (%v): %w", n, o.typ, err) + } + m.apply(o) + if (n+1)%w.cfg.CommitEvery == 0 { + if err := commit(n); err != nil { + return err + } + } + } + if w.cfg.Ops > start { + return commit(w.cfg.Ops - 1) + } + return nil +} + +// applyOp performs one op against the volume and records what it touched so the +// next commit can fsync it. +func (w *Writer) applyOp(o op, touchedFiles, touchedDirs map[string]bool) error { + switch o.typ { + case opCreate: + rel := filePathFor(w.cfg, o.fileID) + path := filepath.Join(w.volDir, rel) + parent := filepath.Dir(path) + if err := os.MkdirAll(parent, 0o755); err != nil { + return err + } + if err := w.writeFile(path, o.fileID, o.nblocks); err != nil { + return err + } + touchedFiles[path] = true + touchedDirs[parent] = true + case opDelete: + rel := filePathFor(w.cfg, o.fileID) + path := filepath.Join(w.volDir, rel) + if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + return err + } + touchedDirs[filepath.Dir(path)] = true + case opMkdir: + path := filepath.Join(w.volDir, o.dir) + if err := os.MkdirAll(path, 0o755); err != nil { + return err + } + touchedDirs[filepath.Dir(path)] = true + case opRmdir: + path := filepath.Join(w.volDir, o.dir) + if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + // A non-empty scratch dir cannot be removed; leave it in place. + if !isNotEmpty(err) { + return err + } + } + touchedDirs[filepath.Dir(path)] = true + } + return nil +} + +// writeFile writes nblocks Layer-1 blocks for fileID, replacing any prior content. +func (w *Writer) writeFile(path string, fileID uint64, nblocks int) error { + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) + if err != nil { + return err + } + defer f.Close() + for i := 0; i < nblocks; i++ { + if _, err := f.Write(BuildBlock(fileID, uint64(i), w.cfg.BlockSize)); err != nil { + return err + } + } + return nil +} + +// Verify reconstructs the expected committed state and checks the on-disk tree +// against it, classifying every expected file (design §4.2, §4.3). +func Verify(volDir string, cfg Config) (Report, error) { + if !cfg.valid() { + return Report{}, fmt.Errorf("invalid config: %+v", cfg) + } + gen := newGenerator(cfg) + m := newModel(cfg) + committed := readCommit(volDir) + if cfg.ExpectCommitted > committed { + // Trust the harness-supplied high-water mark over on-volume bookkeeping + // that fsck may have cleared alongside the data (see Config.ExpectCommitted). + committed = cfg.ExpectCommitted + } + for n := uint64(0); int64(n) <= committed; n++ { + m.apply(gen.next(m, n)) + } + + orphans := scanLostFound(volDir, cfg.BlockSize) + + rep := Report{ + CommittedIndex: committed, + LiveExpected: len(m.liveIDs), + DeletedExpected: len(m.deleted), + } + + for _, id := range m.liveIDs { + meta := m.files[id] + rel := m.filePath(id) + path := filepath.Join(volDir, rel) + anom, ok := checkFile(path, id, meta.nblocks, cfg.BlockSize) + if ok { + rep.FilesOK++ + continue + } + if anom.Verdict == FileLost && orphans[id] { + anom.Verdict = FileOrphaned + } + anom.FileID = id + anom.Path = rel + anom.ExpectBlocks = meta.nblocks + rep.Anomalies = append(rep.Anomalies, anom) + } + + for id := range m.deleted { + path := filepath.Join(volDir, m.filePath(id)) + if _, err := os.Stat(path); err == nil { + rep.Resurrected = append(rep.Resurrected, id) + } + } + return rep, nil +} + +// checkFile classifies one expected-live file. ok is true only for a fully-clean +// file (present, correct size, every block OK). +func checkFile(path string, fileID uint64, nblocks, blockSize int) (FileAnomaly, bool) { + f, err := os.Open(path) + if err != nil { + if os.IsNotExist(err) { + return FileAnomaly{Verdict: FileLost}, false + } + return FileAnomaly{Verdict: FileLost}, false + } + defer f.Close() + + anom := FileAnomaly{Verdict: FilePresentCorrupt, BlockCounts: make(map[BlockStatus]int)} + if fi, err := f.Stat(); err == nil { + if fi.Size() != int64(nblocks)*int64(blockSize) { + anom.SizeMismatch = true + } + } + + buf := make([]byte, blockSize) + allOK := !anom.SizeMismatch + for i := 0; i < nblocks; i++ { + n, err := io.ReadFull(f, buf) + if err == io.EOF || err == io.ErrUnexpectedEOF { + // Fewer blocks than expected: treat the missing tail as zeroed. + for ; n < blockSize; n++ { + buf[n] = 0 + } + res := VerifyBlock(buf, fileID, uint64(i), blockSize) + anom.BlockCounts[res.Status]++ + allOK = false + continue + } + if err != nil { + anom.BlockCounts[BlockGarbage]++ + allOK = false + continue + } + res := VerifyBlock(buf, fileID, uint64(i), blockSize) + anom.BlockCounts[res.Status]++ + if res.Status != BlockOK { + allOK = false + } + } + if allOK { + return FileAnomaly{}, true + } + return anom, false +} + +// scanLostFound returns the set of fileIDs recoverable from a lost+found +// directory at the volume root, identified by reading each entry's first block +// header (design §4.3). It is best-effort: a missing or unreadable lost+found +// yields an empty set. +func scanLostFound(volDir string, blockSize int) map[uint64]bool { + out := make(map[uint64]bool) + lf := filepath.Join(volDir, "lost+found") + entries, err := os.ReadDir(lf) + if err != nil { + return out + } + buf := make([]byte, blockSize) + for _, e := range entries { + if e.IsDir() { + continue + } + f, err := os.Open(filepath.Join(lf, e.Name())) + if err != nil { + continue + } + n, _ := io.ReadFull(f, buf) + f.Close() + if n < blockSize { + continue + } + if id, ok := headerFileID(buf, blockSize); ok { + out[id] = true + } + } + return out +} + +func fsyncFile(path string) error { + f, err := os.Open(path) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + defer f.Close() + return f.Sync() +} diff --git a/evetest/testapps/volverify/internal/verify/engine_test.go b/evetest/testapps/volverify/internal/verify/engine_test.go new file mode 100644 index 00000000000..7192fc4e4e1 --- /dev/null +++ b/evetest/testapps/volverify/internal/verify/engine_test.go @@ -0,0 +1,233 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package verify + +import ( + "os" + "path/filepath" + "testing" +) + +// committedModel rebuilds the expected state up to the committed index, mirroring +// what Verify does, so a test can locate specific live/deleted files to corrupt. +func committedModel(dir string, cfg Config) *model { + gen := newGenerator(cfg) + m := newModel(cfg) + c := readCommit(dir) + for n := uint64(0); int64(n) <= c; n++ { + m.apply(gen.next(m, n)) + } + return m +} + +func writeVolume(t *testing.T, cfg Config) (string, *model) { + t.Helper() + dir := t.TempDir() + w, err := NewWriter(dir, cfg) + if err != nil { + t.Fatal(err) + } + if err := w.Run(); err != nil { + t.Fatal(err) + } + return dir, committedModel(dir, cfg) +} + +func aLiveFile(t *testing.T, m *model) (uint64, int) { + t.Helper() + if len(m.liveIDs) == 0 { + t.Fatal("no live files after write") + } + id := m.liveIDs[0] + return id, m.files[id].nblocks +} + +func TestWriteVerifyClean(t *testing.T) { + cfg := testConfig() + dir, m := writeVolume(t, cfg) + rep, err := Verify(dir, cfg) + if err != nil { + t.Fatal(err) + } + if !rep.Clean() { + t.Fatalf("expected clean, got %s\n%+v", rep, rep.Anomalies) + } + if rep.FilesOK != len(m.liveIDs) { + t.Fatalf("FilesOK=%d, want %d", rep.FilesOK, len(m.liveIDs)) + } +} + +func TestTruncateDetected(t *testing.T) { + cfg := testConfig() + dir, m := writeVolume(t, cfg) + id, _ := aLiveFile(t, m) + if err := os.Truncate(filepath.Join(dir, filePathFor(cfg, id)), 0); err != nil { + t.Fatal(err) + } + rep, _ := Verify(dir, cfg) + a := findAnomaly(rep, id) + if a == nil || a.Verdict != FilePresentCorrupt || !a.SizeMismatch { + t.Fatalf("truncate not detected as present-corrupt: %+v", a) + } +} + +func TestZeroBlockDetected(t *testing.T) { + cfg := testConfig() + dir, m := writeVolume(t, cfg) + id, _ := aLiveFile(t, m) + f, err := os.OpenFile(filepath.Join(dir, filePathFor(cfg, id)), os.O_WRONLY, 0) + if err != nil { + t.Fatal(err) + } + if _, err := f.WriteAt(make([]byte, cfg.BlockSize), 0); err != nil { + t.Fatal(err) + } + f.Close() + rep, _ := Verify(dir, cfg) + a := findAnomaly(rep, id) + if a == nil || a.Verdict != FilePresentCorrupt || a.BlockCounts[BlockZeroed] == 0 { + t.Fatalf("zeroed block not detected: %+v", a) + } +} + +func TestMisplacedDetected(t *testing.T) { + cfg := testConfig() + dir, m := writeVolume(t, cfg) + id, nblocks := aLiveFile(t, m) + other := id + 100000 // an identity distinct from every live/deleted file + f, err := os.OpenFile(filepath.Join(dir, filePathFor(cfg, id)), os.O_WRONLY|os.O_TRUNC, 0o644) + if err != nil { + t.Fatal(err) + } + for i := 0; i < nblocks; i++ { + if _, err := f.Write(BuildBlock(other, uint64(i), cfg.BlockSize)); err != nil { + t.Fatal(err) + } + } + f.Close() + rep, _ := Verify(dir, cfg) + a := findAnomaly(rep, id) + if a == nil || a.BlockCounts[BlockMisplaced] == 0 { + t.Fatalf("misplaced data not detected: %+v", a) + } +} + +func TestLostThenOrphaned(t *testing.T) { + cfg := testConfig() + dir, m := writeVolume(t, cfg) + id, _ := aLiveFile(t, m) + if err := os.Remove(filepath.Join(dir, filePathFor(cfg, id))); err != nil { + t.Fatal(err) + } + rep, _ := Verify(dir, cfg) + if a := findAnomaly(rep, id); a == nil || a.Verdict != FileLost { + t.Fatalf("removed file not reported lost: %+v", a) + } + // Now place the file in lost+found (identified by its first block header). + lf := filepath.Join(dir, "lost+found") + if err := os.MkdirAll(lf, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(lf, "#131074"), BuildBlock(id, 0, cfg.BlockSize), 0o644); err != nil { + t.Fatal(err) + } + rep, _ = Verify(dir, cfg) + if a := findAnomaly(rep, id); a == nil || a.Verdict != FileOrphaned { + t.Fatalf("lost+found file not reported orphaned: %+v", a) + } +} + +func TestResurrectedDetected(t *testing.T) { + cfg := testConfig() + dir, m := writeVolume(t, cfg) + var delID uint64 + found := false + for id := range m.deleted { + delID = id + found = true + break + } + if !found { + t.Skip("no deleted files in this op stream") + } + path := filepath.Join(dir, filePathFor(cfg, delID)) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, BuildBlock(delID, 0, cfg.BlockSize), 0o644); err != nil { + t.Fatal(err) + } + rep, _ := Verify(dir, cfg) + if len(rep.Resurrected) == 0 { + t.Fatal("resurrected deleted file not detected") + } +} + +func TestCrashResumeIdempotent(t *testing.T) { + cfg := testConfig() + dir, _ := writeVolume(t, cfg) + // Running the writer again must resume from the commit and leave the volume + // clean (idempotent re-application of already-committed ops). + w, _ := NewWriter(dir, cfg) + if err := w.Run(); err != nil { + t.Fatal(err) + } + rep, _ := Verify(dir, cfg) + if !rep.Clean() { + t.Fatalf("resume left volume dirty: %s", rep) + } +} + +func TestInFlightExtraFileTolerated(t *testing.T) { + cfg := testConfig() + dir, _ := writeVolume(t, cfg) + // A file whose id was never part of the committed op stream stands in for an + // in-flight (op > committed index) create. It is outside the expected set, so + // the verifier must tolerate it rather than flag it (design §4.2). + extra := cfg.Ops + 5 + path := filepath.Join(dir, filePathFor(cfg, extra)) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, BuildBlock(extra, 0, cfg.BlockSize), 0o644); err != nil { + t.Fatal(err) + } + rep, _ := Verify(dir, cfg) + if !rep.Clean() { + t.Fatalf("in-flight extra file should be tolerated, got %s\n%+v", rep, rep.Anomalies) + } +} + +func TestExpectCommittedSurvivesMetadataLoss(t *testing.T) { + cfg := testConfig() + dir, m := writeVolume(t, cfg) + id, _ := aLiveFile(t, m) + // Simulate fsck clearing both the on-volume commit metadata and a last file. + if err := os.RemoveAll(filepath.Join(dir, commitDirName)); err != nil { + t.Fatal(err) + } + if err := os.Remove(filepath.Join(dir, filePathFor(cfg, id))); err != nil { + t.Fatal(err) + } + // Without the floor the verifier reads no commit, expects nothing, and the + // loss is masked. + if rep, _ := Verify(dir, cfg); !rep.Clean() { + t.Fatalf("baseline: expected masked-clean without floor, got %s", rep) + } + // With the harness high-water mark as a floor, the loss is caught. + cfg.ExpectCommitted = int64(cfg.Ops - 1) + rep, _ := Verify(dir, cfg) + if a := findAnomaly(rep, id); a == nil || a.Verdict != FileLost { + t.Fatalf("floor: loss after metadata clearing not caught: %+v", a) + } +} + +func findAnomaly(rep Report, id uint64) *FileAnomaly { + for i := range rep.Anomalies { + if rep.Anomalies[i].FileID == id { + return &rep.Anomalies[i] + } + } + return nil +} diff --git a/evetest/testapps/volverify/internal/verify/model.go b/evetest/testapps/volverify/internal/verify/model.go new file mode 100644 index 00000000000..38132b2cf6f --- /dev/null +++ b/evetest/testapps/volverify/internal/verify/model.go @@ -0,0 +1,103 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package verify + +import ( + "sort" +) + +// fileMeta records a live file's placement and logical length (in blocks). +type fileMeta struct { + nblocks int +} + +// model is the expected filesystem state reconstructed by replaying the op +// stream. The writer advances it as it applies ops; the verifier rebuilds it up +// to the committed index and then checks the on-disk tree against it. +// +// liveIDs and dirList are kept sorted so that op selection (which live file to +// delete, which scratch dir to remove) is a pure function of the PRNG draw and +// therefore identical in the writer and the verifier — Go map iteration order is +// randomized and must never drive a selection. +type model struct { + cfg Config + files map[uint64]fileMeta + liveIDs []uint64 + deleted map[uint64]bool + dirs map[string]bool + dirList []string +} + +func newModel(cfg Config) *model { + return &model{ + cfg: cfg, + files: make(map[uint64]fileMeta), + deleted: make(map[uint64]bool), + dirs: make(map[string]bool), + } +} + +// filePath returns the volume-relative path of a file (see filePathFor). +func (m *model) filePath(fileID uint64) string { + return filePathFor(m.cfg, fileID) +} + +func (m *model) addFile(fileID uint64, nblocks int) { + if _, ok := m.files[fileID]; !ok { + i := sort.Search(len(m.liveIDs), func(i int) bool { return m.liveIDs[i] >= fileID }) + m.liveIDs = append(m.liveIDs, 0) + copy(m.liveIDs[i+1:], m.liveIDs[i:]) + m.liveIDs[i] = fileID + } + m.files[fileID] = fileMeta{nblocks: nblocks} + delete(m.deleted, fileID) +} + +func (m *model) removeFile(fileID uint64) { + if _, ok := m.files[fileID]; !ok { + return + } + delete(m.files, fileID) + i := sort.Search(len(m.liveIDs), func(i int) bool { return m.liveIDs[i] >= fileID }) + if i < len(m.liveIDs) && m.liveIDs[i] == fileID { + m.liveIDs = append(m.liveIDs[:i], m.liveIDs[i+1:]...) + } + m.deleted[fileID] = true +} + +func (m *model) addDir(dir string) { + if m.dirs[dir] { + return + } + m.dirs[dir] = true + i := sort.SearchStrings(m.dirList, dir) + m.dirList = append(m.dirList, "") + copy(m.dirList[i+1:], m.dirList[i:]) + m.dirList[i] = dir +} + +func (m *model) removeDir(dir string) { + if !m.dirs[dir] { + return + } + delete(m.dirs, dir) + i := sort.SearchStrings(m.dirList, dir) + if i < len(m.dirList) && m.dirList[i] == dir { + m.dirList = append(m.dirList[:i], m.dirList[i+1:]...) + } +} + +// apply advances the model by one op. +func (m *model) apply(o op) { + switch o.typ { + case opCreate: + m.addFile(o.fileID, o.nblocks) + case opDelete: + m.removeFile(o.fileID) + case opMkdir: + m.addDir(o.dir) + case opRmdir: + m.removeDir(o.dir) + } +} diff --git a/evetest/testapps/volverify/internal/verify/opstream.go b/evetest/testapps/volverify/internal/verify/opstream.go new file mode 100644 index 00000000000..951e944276c --- /dev/null +++ b/evetest/testapps/volverify/internal/verify/opstream.go @@ -0,0 +1,81 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package verify + +import "fmt" + +// opType is the kind of filesystem mutation an op performs. +type opType int + +const ( + opCreate opType = iota + opDelete + opMkdir + opRmdir +) + +// op is one entry in the deterministic op stream (design §4.2). fileID equals the +// op index for a create. +type op struct { + n uint64 + typ opType + fileID uint64 + nblocks int + dir string +} + +// generator produces the op stream. Given the same seed it yields the same ops, +// provided it is driven against a model advanced by those same ops — the two +// selection ops (delete/rmdir) consult the model, so the writer and verifier must +// both replay from op 0 to stay in lockstep. +type generator struct { + rng *prng + cfg Config +} + +func newGenerator(cfg Config) *generator { + return &generator{rng: newPRNG(cfg.Seed), cfg: cfg} +} + +// scratchDir names one of a bounded set of churn directories, kept disjoint from +// the file-placement tree so mkdir/rmdir churn never removes a live file's parent. +func scratchDir(k int) string { + return fmt.Sprintf("scratch/s%03d", k) +} + +// pickNBlocks draws a file length from a heavy-tailed distribution: mostly small, +// occasionally medium, rarely large. +func (g *generator) pickNBlocks() int { + r := g.rng.next() % 1000 + switch { + case r < 850: + return 1 + g.rng.intn(g.cfg.SmallBlocks) + case r < 990: + return 1 + g.rng.intn(g.cfg.MedBlocks) + default: + return 1 + g.rng.intn(g.cfg.MaxBlocks) + } +} + +// next returns op n for the current model state and advances the generator. +func (g *generator) next(m *model, n uint64) op { + choice := g.rng.next() % 100 + switch { + case choice < 65: + return op{n: n, typ: opCreate, fileID: n, nblocks: g.pickNBlocks()} + case choice < 85: + if len(m.liveIDs) == 0 { + return op{n: n, typ: opCreate, fileID: n, nblocks: g.pickNBlocks()} + } + id := m.liveIDs[g.rng.intn(len(m.liveIDs))] + return op{n: n, typ: opDelete, fileID: id} + case choice < 93: + return op{n: n, typ: opMkdir, dir: scratchDir(g.rng.intn(g.cfg.DirFanout * g.cfg.DirFanout))} + default: + if len(m.dirList) == 0 { + return op{n: n, typ: opCreate, fileID: n, nblocks: g.pickNBlocks()} + } + return op{n: n, typ: opRmdir, dir: m.dirList[g.rng.intn(len(m.dirList))]} + } +} diff --git a/evetest/testapps/volverify/internal/verify/opstream_test.go b/evetest/testapps/volverify/internal/verify/opstream_test.go new file mode 100644 index 00000000000..1690ebca13d --- /dev/null +++ b/evetest/testapps/volverify/internal/verify/opstream_test.go @@ -0,0 +1,75 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package verify + +import "testing" + +func testConfig() Config { + return Config{ + Seed: 1234, + BlockSize: testBlockSize, + Ops: 300, + CommitEvery: 8, + DirFanout: 4, + SmallBlocks: 2, + MedBlocks: 4, + MaxBlocks: 8, + ExpectCommitted: -1, + } +} + +// replayOps drives the generator against a fresh model, returning the full stream. +func replayOps(cfg Config, n uint64) []op { + gen := newGenerator(cfg) + m := newModel(cfg) + ops := make([]op, 0, n) + for i := uint64(0); i < n; i++ { + o := gen.next(m, i) + ops = append(ops, o) + m.apply(o) + } + return ops +} + +func TestOpStreamDeterministic(t *testing.T) { + cfg := testConfig() + a := replayOps(cfg, cfg.Ops) + b := replayOps(cfg, cfg.Ops) + if len(a) != len(b) { + t.Fatalf("length mismatch %d vs %d", len(a), len(b)) + } + for i := range a { + if a[i] != b[i] { + t.Fatalf("op %d differs: %+v vs %+v", i, a[i], b[i]) + } + } +} + +func TestOpStreamSeedSensitive(t *testing.T) { + cfg := testConfig() + a := replayOps(cfg, cfg.Ops) + cfg.Seed++ + b := replayOps(cfg, cfg.Ops) + same := true + for i := range a { + if a[i] != b[i] { + same = false + break + } + } + if same { + t.Fatal("different seeds produced identical op streams") + } +} + +func TestOpStreamHasMix(t *testing.T) { + cfg := testConfig() + var counts [4]int + for _, o := range replayOps(cfg, cfg.Ops) { + counts[o.typ]++ + } + if counts[opCreate] == 0 || counts[opDelete] == 0 { + t.Fatalf("expected creates and deletes, got %+v", counts) + } +} diff --git a/evetest/testapps/volverify/internal/verify/report.go b/evetest/testapps/volverify/internal/verify/report.go new file mode 100644 index 00000000000..bb476dab855 --- /dev/null +++ b/evetest/testapps/volverify/internal/verify/report.go @@ -0,0 +1,84 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package verify + +import "fmt" + +// FileVerdict is the outcome for one expected-live file. +type FileVerdict int + +const ( + // FileOK means the file is present, correctly sized, and every block verified. + FileOK FileVerdict = iota + // FilePresentCorrupt means the file is present but has bad/misplaced/short + // blocks — the dangerous case EVE serves as-is (design §2.4, §4.3). + FilePresentCorrupt + // FileOrphaned means the file is missing from its path but was recovered in + // lost+found — self-heals to a blank/content-tree recreate (safe). + FileOrphaned + // FileLost means the file is missing and not in lost+found — data loss. + FileLost +) + +func (v FileVerdict) String() string { + switch v { + case FileOK: + return "ok" + case FilePresentCorrupt: + return "present-corrupt" + case FileOrphaned: + return "orphaned" + case FileLost: + return "lost" + } + return "unknown" +} + +// FileAnomaly describes one non-OK expected file. +type FileAnomaly struct { + FileID uint64 + Path string + Verdict FileVerdict + ExpectBlocks int + // BlockCounts tallies each block status seen in a present-corrupt file. + BlockCounts map[BlockStatus]int + // SizeMismatch is set when the on-disk size differs from expected. + SizeMismatch bool +} + +// Report summarizes a verification pass. It is the volume-content ground truth +// the soak harness pairs with the resize fsck marker. +type Report struct { + CommittedIndex int64 + LiveExpected int + DeletedExpected int + FilesOK int + Anomalies []FileAnomaly + // Resurrected lists committed-deleted files that reappeared (e.g. from + // lost+found) — an anomaly in its own right. + Resurrected []uint64 +} + +// Clean reports whether the pass found no anomalies of any kind. +func (r Report) Clean() bool { + return len(r.Anomalies) == 0 && len(r.Resurrected) == 0 +} + +// Counts returns the number of anomalies of each verdict. +func (r Report) Counts() map[FileVerdict]int { + c := make(map[FileVerdict]int) + for _, a := range r.Anomalies { + c[a.Verdict]++ + } + return c +} + +// String renders a compact human-readable summary. +func (r Report) String() string { + c := r.Counts() + return fmt.Sprintf( + "committed=%d live=%d deleted=%d ok=%d present-corrupt=%d orphaned=%d lost=%d resurrected=%d", + r.CommittedIndex, r.LiveExpected, r.DeletedExpected, r.FilesOK, + c[FilePresentCorrupt], c[FileOrphaned], c[FileLost], len(r.Resurrected)) +} diff --git a/evetest/testapps/volverify/internal/verify/rng.go b/evetest/testapps/volverify/internal/verify/rng.go new file mode 100644 index 00000000000..68fdaf9d004 --- /dev/null +++ b/evetest/testapps/volverify/internal/verify/rng.go @@ -0,0 +1,28 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package verify + +// prng is a splitmix64 stream. It is deterministic given its seed, so the writer +// and verifier reproduce the identical op stream by seeding from the same +// masterSeed and drawing in the same order (design §4.2). +type prng struct { + state uint64 +} + +func newPRNG(seed uint64) *prng { + return &prng{state: seed} +} + +func (p *prng) next() uint64 { + p.state += 0x9E3779B97F4A7C15 + z := p.state + z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9 + z = (z ^ (z >> 27)) * 0x94D049BB133111EB + return z ^ (z >> 31) +} + +// intn returns a value in [0,n). n must be positive. +func (p *prng) intn(n int) int { + return int(p.next() % uint64(n)) +} diff --git a/evetest/testapps/volverify/internal/verify/util.go b/evetest/testapps/volverify/internal/verify/util.go new file mode 100644 index 00000000000..aac62b08c2f --- /dev/null +++ b/evetest/testapps/volverify/internal/verify/util.go @@ -0,0 +1,25 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package verify + +import ( + "errors" + "fmt" + "syscall" +) + +// filePathFor returns the volume-relative path of a file, scattering files across +// a bounded two-level tree derived from the fileID alone. Writer and verifier +// both use it so a file always lands at the same place. +func filePathFor(cfg Config, fileID uint64) string { + fan := uint64(cfg.DirFanout) + top := fileID % fan + sub := (fileID / fan) % fan + return fmt.Sprintf("d%02d/d%02d/f%d.blk", top, sub, fileID) +} + +// isNotEmpty reports whether err is a "directory not empty" error. +func isNotEmpty(err error) bool { + return errors.Is(err, syscall.ENOTEMPTY) || errors.Is(err, syscall.EEXIST) +} diff --git a/evetest/testapps/volverify/scripts/loopback-ext4-test.sh b/evetest/testapps/volverify/scripts/loopback-ext4-test.sh new file mode 100755 index 00000000000..1ed4a34170d --- /dev/null +++ b/evetest/testapps/volverify/scripts/loopback-ext4-test.sh @@ -0,0 +1,91 @@ +#!/bin/bash +# Copyright (c) 2026 Zededa, Inc. +# SPDX-License-Identifier: Apache-2.0 +# +# On-filesystem fidelity check for volverify: round-trips the write/verify pattern +# on a real loopback ext4 (not tmpfs), then injects truncation and block-zeroing +# and confirms the verifier detects them. A final stage corrupts filesystem data +# and runs e2fsck so the lost+found / orphaned-vs-present-corrupt path exercises +# the real fsck behavior the soak depends on. +# +# Requires root (losetup/mkfs/mount). Run: sudo ./scripts/loopback-ext4-test.sh + +set -euo pipefail + +if [ "$(id -u)" -ne 0 ]; then + echo "must run as root (losetup/mount); re-run with sudo" >&2 + exit 1 +fi + +HERE="$(cd "$(dirname "$0")/.." && pwd)" +WORK="$(mktemp -d)" +IMG="$WORK/vol.img" +MNT="$WORK/mnt" +BIN="$WORK/volverify" +LOOP="" + +cleanup() { + mountpoint -q "$MNT" && umount "$MNT" || true + [ -n "$LOOP" ] && losetup -d "$LOOP" 2>/dev/null || true + rm -rf "$WORK" +} +trap cleanup EXIT + +# Deterministic op stream small enough to run quickly but large enough to spill +# files across many block groups. +SEED=20260723 +OPS=4000 +COMMON="--seed $SEED --ops $OPS --block-size 4096 --small-blocks 8 --med-blocks 512 --max-blocks 4096" + +echo "== build volverify ==" +( cd "$HERE" && GOWORK=off go build -o "$BIN" ./cmd/volverify ) + +echo "== create + mount loopback ext4 (1 GiB) ==" +mkdir -p "$MNT" +truncate -s 1G "$IMG" +mkfs.ext4 -q -F "$IMG" +LOOP="$(losetup --find --show "$IMG")" +mount "$LOOP" "$MNT" + +echo "== write pattern ==" +# shellcheck disable=SC2086 +"$BIN" write --dir "$MNT" $COMMON + +echo "== verify (expect clean on a real ext4) ==" +# shellcheck disable=SC2086 +"$BIN" verify --dir "$MNT" $COMMON +echo "PASS: clean round-trip on ext4" + +echo "== inject truncation + block-zeroing on two files ==" +# Collect matches with mapfile rather than `find | head` — under pipefail the +# early pipe close makes find exit 141 (SIGPIPE) and set -e then aborts. +mapfile -t FILES < <(find "$MNT" -name 'f*.blk') +if [ "${#FILES[@]}" -lt 2 ]; then + echo "FAIL: need >=2 data files to corrupt, found ${#FILES[@]}" >&2 + exit 1 +fi +VICTIM="${FILES[0]}" +VICTIM2="${FILES[1]}" +truncate -s 0 "$VICTIM" +dd if=/dev/zero of="$VICTIM2" bs=4096 count=1 conv=notrunc status=none + +echo "== verify (expect anomalies) ==" +# shellcheck disable=SC2086 +if "$BIN" verify --dir "$MNT" $COMMON; then + echo "FAIL: verifier reported clean after injected corruption" >&2 + exit 1 +fi +echo "PASS: verifier detected injected truncation/zeroing" + +echo "== fsck path: corrupt fs data then e2fsck, re-verify (informational) ==" +umount "$MNT" +# Zero a 4 MiB span in the data area to force e2fsck into real repairs. +dd if=/dev/zero of="$IMG" bs=1M count=4 seek=200 conv=notrunc status=none +e2fsck -fy "$IMG" || true +mount "$LOOP" "$MNT" +echo "-- lost+found after e2fsck:"; ls -1 "$MNT/lost+found" 2>/dev/null || true +echo "-- verifier report after e2fsck:" +# shellcheck disable=SC2086 +"$BIN" verify --dir "$MNT" $COMMON || true + +echo "ALL DONE" From 1c5bd5f2a59a5d5cb7f1e373b8636ac0399931e6 Mon Sep 17 00:00:00 2001 From: eriknordmark Date: Thu, 23 Jul 2026 22:38:57 -0700 Subject: [PATCH 02/29] evetest: support blank data volumes on apps Lets a test attach one or more empty data volumes to an application beyond its image. Each is created as a blank disk of a given size (VCOT_BLANK, no content tree) and mounted at a chosen path. This is needed to exercise a data volume through the EVE-kvm to EVE-k offline shrink: the volume must be a large file in /persist so the resize relocates its blocks. The existing app config only carried the image volume and its size. Signed-off-by: eriknordmark Co-Authored-By: Claude Opus 4.8 (1M context) --- evetest/devconfig.go | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/evetest/devconfig.go b/evetest/devconfig.go index 5a2a752c43d..0a9490ab1d1 100644 --- a/evetest/devconfig.go +++ b/evetest/devconfig.go @@ -886,9 +886,20 @@ type ApplicationInstanceConfig struct { UserData string NetworkAdapters []AppNetworkAdapter EnforceNetIntfOrder bool + // DataVolumes are additional blank (empty) volumes attached to the app beyond + // its image. Each has no content tree — EVE creates an empty disk of the given + // size (VCOT_BLANK) and mounts it at MountDir. + DataVolumes []DataVolumeConfig // Many more parameters can be configured; they will be added later as needed. } +// DataVolumeConfig describes one blank data volume attached to an application. +type DataVolumeConfig struct { + SizeBytes uint64 // volume size in bytes (Maxsizebytes) + MountDir string // mount point inside the app (default: /mnt/vol) + ReadOnly bool +} + func (config ApplicationInstanceConfig) toProto(th *TestHarness, devName string, appUUID, volumeUUID uuid.UUID) *eveconfig.AppInstanceConfig { vmConfig := &eveconfig.VmConfig{ @@ -2272,6 +2283,32 @@ func (dc *EdgeDeviceConfig) addApplicationWithUUIDs( contentTreeUUID, datastoreUUID, config.DisplayName) dc.ContentInfo = append(dc.ContentInfo, contentTree) dc.Datastores = append(dc.Datastores, dsConfig) + + // Attach any blank data volumes: a VCOT_BLANK Volume (no content tree or + // datastore) plus a VolumeRef mounting it into the app. appInstConfig is a + // pointer already stored in dc.Apps, so extending its VolumeRefList here is + // reflected in the stored config. + for i, dv := range config.DataVolumes { + dvUUID := dc.th.newUUID("application data volume") + mountDir := dv.MountDir + if mountDir == "" { + mountDir = fmt.Sprintf("/mnt/vol%d", i) + } + appInstConfig.VolumeRefList = append(appInstConfig.VolumeRefList, + &eveconfig.VolumeRef{ + Uuid: dvUUID.String(), + MountDir: mountDir, + }) + dc.Volumes = append(dc.Volumes, &eveconfig.Volume{ + Uuid: dvUUID.String(), + Origin: &eveconfig.VolumeContentOrigin{ + Type: eveconfig.VolumeContentOriginType_VCOT_BLANK, + }, + Maxsizebytes: int64(dv.SizeBytes), + Readonly: dv.ReadOnly, + DisplayName: fmt.Sprintf("%s-data%d", config.DisplayName, i), + }) + } } // UpdateApplication updates an existing application instance identified From 1c38ba82baae7ad2fe9a66d080952e5e86a50e14 Mon Sep 17 00:00:00 2001 From: eriknordmark Date: Thu, 23 Jul 2026 22:38:57 -0700 Subject: [PATCH 03/29] evetest: add app-volume shrink corruption test Adds a test that checks whether a watchdog-interrupted EVE-kvm to EVE-k offline shrink corrupts an application data volume. It deploys the volverify app with a large blank data volume, fills it before the upgrade, drives the kvm to kubevirt conversion (which repartitions and shrinks /persist, relocating the volume's blocks), and re-verifies the pattern afterward. The watchdog fault is expected to come from the target EVE image rather than the harness, so the test simply upgrades to that build. It fails when the volume comes back present but corrupt (the case EVE would silently serve to the app) and surfaces the recoverable orphaned/lost counts for the soak to tally. Signed-off-by: eriknordmark Co-Authored-By: Claude Opus 4.8 (1M context) --- .../tests/appvolshrink/appvolshrink_test.go | 302 ++++++++++++++++++ 1 file changed, 302 insertions(+) create mode 100644 evetest/tests/appvolshrink/appvolshrink_test.go diff --git a/evetest/tests/appvolshrink/appvolshrink_test.go b/evetest/tests/appvolshrink/appvolshrink_test.go new file mode 100644 index 00000000000..3eb080c16e9 --- /dev/null +++ b/evetest/tests/appvolshrink/appvolshrink_test.go @@ -0,0 +1,302 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package appvolshrink_test + +import ( + "fmt" + "strings" + "testing" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + eveconfig "github.com/lf-edge/eve-api/go/config" + "github.com/lf-edge/eve-api/go/evecommon" + eveinfo "github.com/lf-edge/eve-api/go/info" + "github.com/lf-edge/eve/evetest" + "github.com/lf-edge/eve/evetest/constants" + "github.com/lf-edge/eve/evetest/matchers" + "github.com/lf-edge/eve/evetest/netmodels" + "github.com/lf-edge/eve/pkg/pillar/types" +) + +const ( + initialEVEVersionParamKey = "INITIAL_EVE_VERSION" + initialHypervisorParamKey = "INITIAL_HYPERVISOR" + seedParamKey = "VOLVERIFY_SEED" + opsParamKey = "VOLVERIFY_OPS" + volSizeMiBParamKey = "DATA_VOLUME_MB" + volverifyImageParamKey = "VOLVERIFY_IMAGE" + + // defaultVolverifyImage is the canonical published testapp image; override + // VOLVERIFY_IMAGE to run against a personal registry copy. + defaultVolverifyImage = "lfedge/evetest-volverify" + + appSSHUser = "root" + appSSHPassword = "testpassword" + appSSHFwdPort = 2222 + dataMountDir = "/mnt/data" + blockSize = 4096 +) + +// TestAppVolumeShrinkCorruption checks whether a watchdog-interrupted EVE-kvm→EVE-k +// offline filesystem shrink corrupts an application data volume, and if so whether +// the corruption is detectable. +// +// It deploys the volverify app (github.com/lf-edge/eve/evetest/testapps/volverify) +// with a large BLANK data volume, fills it with a deterministic self-verifying +// pattern before the upgrade, drives the kvm→kubevirt upgrade (which repartitions +// and shrinks /persist, relocating the volume's blocks), and then re-verifies the +// pattern. The watchdog fault is baked into the target EVE image (the fork#7 no-pet +// stress build fires the HW watchdog inside the offline resizer), so this test just +// upgrades TO that build; the soak loops the test externally (design §5, §7). +// +// The target build must also relax the shrink+volumes gate (§3.1) — the shippable +// EVE refuses a cross-flavor shrink while a volume is present. +// +// Parameters: +// - EVE_VERSION / HYPERVISOR: target build + hypervisor (default hypervisor kubevirt here). +// - INITIAL_EVE_VERSION (required, e.g. "16.6.0") / INITIAL_HYPERVISOR (default kvm). +// - DISK_SIZE_MB: device disk size (default 131072 = 128 GiB). +// - DATA_VOLUME_MB: blank data-volume size (default 40960 = 40 GiB). +// - VOLVERIFY_SEED / VOLVERIFY_OPS: pattern seed and op count. +func TestAppVolumeShrinkCorruption(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.EVEVersionParameter(), + evetest.HypervisorParameter(), + evetest.TPMParameter(), + evetest.DiskSizeMiBParameter(), + evetest.TestParameterDefinition{ + Key: initialEVEVersionParamKey, + DefaultValue: "16.6.0", + Description: evetest.TestParameterDescription{ + Summary: "EVE-kvm version to start on (before the kvm→k conversion)", + Default: "16.6.0", + }, + }, + evetest.TestParameterDefinition{ + Key: initialHypervisorParamKey, + DefaultValue: evetest.HypervisorKVM, + Description: evetest.TestParameterDescription{ + Summary: "Hypervisor of the initial (pre-upgrade) EVE version", + Default: "kvm", + AllowedValues: "kvm|kubevirt", + }, + }, + evetest.TestParameterDefinition{ + Key: volSizeMiBParamKey, + DefaultValue: uint32(40960), + Description: evetest.TestParameterDescription{ + Summary: "Blank data-volume size in MiB (should exceed the shrink evacuation zone)", + Default: "40960", + }, + }, + evetest.TestParameterDefinition{ + Key: seedParamKey, + DefaultValue: uint64(20260723), + Description: evetest.TestParameterDescription{ + Summary: "volverify master seed for the fill/delete pattern", + Default: "20260723", + }, + }, + evetest.TestParameterDefinition{ + Key: opsParamKey, + DefaultValue: uint64(200000), + Description: evetest.TestParameterDescription{ + Summary: "volverify op count (creates/deletes) to fill the volume", + Default: "200000", + }, + }, + evetest.TestParameterDefinition{ + Key: volverifyImageParamKey, + DefaultValue: defaultVolverifyImage, + Description: evetest.TestParameterDescription{ + Summary: "Docker Hub repo of the volverify test app (override for a personal registry copy)", + Default: defaultVolverifyImage, + }, + }, + ) + + withTPM := evetest.GetTPMParameterValue() + diskSizeMiB := evetest.GetDiskSizeMiBParameterValue() + if diskSizeMiB == 0 { + diskSizeMiB = 131072 // 128 GiB — enough headroom for a real shrink + } + initialVersion := evetest.GetTestParameter[string](initialEVEVersionParamKey) + if initialVersion == "" { + evetestT.Fatalf("%s%s is required", constants.EnvPrefix, initialEVEVersionParamKey) + } + initialHypervisor := evetest.GetTestParameter[evetest.Hypervisor](initialHypervisorParamKey) + targetVersion := evetest.GetEVEVersionParameterValue() + targetHypervisor := evetest.GetHypervisorParameterValue() + volSizeMiB := evetest.GetTestParameter[uint32](volSizeMiBParamKey) + seed := evetest.GetTestParameter[uint64](seedParamKey) + ops := evetest.GetTestParameter[uint64](opsParamKey) + volverifyImage := evetest.GetTestParameter[string](volverifyImageParamKey) + + const devName = "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithEVEVersion: initialVersion, + WithHypervisor: initialHypervisor, + WithTPM: withTPM, + MinDiskSizeInMiB: diskSizeMiB, + DeviceReusePolicy: evetest.CreateFromScratchWithInstaller, + }, + evetest.RequireNetworkModel{NetworkModel: netmodels.SingleEthWithDHCP}, + ) + device := evetest.GetEdgeDevice(devName) + log := evetest.Logger() + + devConfig := evetest.NewEdgeDeviceConfig(devName) + networkUUID := devConfig.AddNetwork(evetest.DHCPNetworkConfig{ + NetworkType: evecommon.NetworkType_V4, + }) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "eth0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: networkUUID, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + + const nodeReadyCond = eveinfo.KubeNodeConditionType_KUBE_NODE_CONDITION_TYPE_READY + isK3sReady := func(info *eveinfo.ZInfoKubeCluster) bool { + if info == nil || len(info.Nodes) != 1 { + return false + } + if info.Storage.Health != eveinfo.ServiceStatus_SERVICE_STATUS_HEALTHY { + return false + } + for _, cond := range info.Nodes[0].GetConditions() { + if cond.GetType() == nodeReadyCond { + return cond.GetSet() + } + } + return false + } + + if initialHypervisor == evetest.HypervisorKubevirt { + clusterUpdates, stopClusterWatch := device.WatchClusterInfo() + defer stopClusterWatch() + device.ApplyConfig(devConfig, true, true) + t.Eventually(clusterUpdates, 20*time.Minute).Should(Receive( + matchers.SatisfyPredicate("K3s node is ready", isK3sReady))) + evetest.Checkpoint("k3s-ready") + } + + niUUID := devConfig.AddNetworkInstance(evetest.LocalNetworkInstanceConfig{ + DisplayName: "local-ni", + Port: "eth0", + Subnet: evetest.IPSubnet("10.11.12.0/24"), + DHCPRange: types.IPRange{ + Start: evetest.IPAddress("10.11.12.2"), + End: evetest.IPAddress("10.11.12.254"), + }, + Gateway: evetest.IPAddress("10.11.12.1"), + MTU: 1500, + }) + appUUID := devConfig.AddApplication(evetest.ApplicationInstanceConfig{ + DisplayName: "volverify-app", + Activate: true, + Image: evetest.DockerContainer{ + ImageName: volverifyImage, + Tag: "1.0", + }, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 512 * evetest.MiB, + DataVolumes: []evetest.DataVolumeConfig{ + {SizeBytes: uint64(volSizeMiB) * evetest.MiB, MountDir: dataMountDir}, + }, + NetworkAdapters: []evetest.AppNetworkAdapter{ + evetest.VirtualNetworkAdapter{ + LogicalLabel: "vif0", + NetworkInstanceUUID: niUUID, + PortFwdRules: []evetest.PortFwdRule{ + {Protocol: evetest.NetworkProtocolTCP, EdgeNodePort: appSSHFwdPort, AppPort: 22}, + }, + ACLAllowRules: []evetest.ACLAllowRule{ + {Protocol: evetest.NetworkProtocolAny, RemoteSubnet: evetest.IPSubnet("0.0.0.0/0")}, + }, + }, + }, + }) + device.ApplyConfig(devConfig, false, false) + device.WaitUntilAppIsRunning(appUUID, 5*time.Minute) + + appAuth := evetest.UsernamePasswordAuth{Username: appSSHUser, Password: appSSHPassword} + + // Fill the volume with the deterministic pattern before the shrink. The write + // of a multi-GiB volume can take a while, so allow a generous timeout. + writeCmd := fmt.Sprintf("volverify write --dir %s --seed %d --ops %d --block-size %d", + dataMountDir, seed, ops, blockSize) + log.Infof("Filling data volume: %s", writeCmd) + t.Eventually(func(t Gomega) { + _, stderr, err := device.RunShellScriptInsideApp(appUUID, appAuth, writeCmd, 60*time.Minute, 0) + t.Expect(err).NotTo(HaveOccurred(), "write stderr: %s", stderr) + }, 65*time.Minute, 10*time.Second).Should(Succeed()) + evetest.Checkpoint("volume-filled") + + // Drive the watchdog-interrupted kvm→k conversion (shrink relocates the volume). + device.UpgradeEVE(targetVersion, targetHypervisor, true, false) + evetest.Checkpoint("upgrade-complete") + + if targetHypervisor == evetest.HypervisorKubevirt { + if !isK3sReady(device.GetClusterInfo()) { + clusterUpdates, stopClusterWatch := device.WatchClusterInfo() + defer stopClusterWatch() + t.Eventually(clusterUpdates, 20*time.Minute).Should(Receive( + matchers.SatisfyPredicate("K3s node is ready", isK3sReady))) + } + evetest.Checkpoint("k3s-ready-post-upgrade") + } + device.WaitUntilAppIsRunning(appUUID, 5*time.Minute) + + // Re-verify. The on-volume commit lives on the shrunk filesystem and may be + // cleared by fsck along with data, so pass the harness high-water mark + // (ops-1) as an off-volume floor (design §4.2). + verifyCmd := fmt.Sprintf("volverify verify --dir %s --seed %d --ops %d --block-size %d --expect-committed %d", + dataMountDir, seed, ops, blockSize, ops-1) + log.Infof("Verifying data volume: %s", verifyCmd) + var report string + t.Eventually(func(t Gomega) { + stdout, _, _ := device.RunShellScriptInsideApp(appUUID, appAuth, verifyCmd, 30*time.Minute, 0) + t.Expect(stdout).NotTo(BeEmpty()) + report = strings.TrimSpace(stdout) + }, 32*time.Minute, 10*time.Second).Should(Succeed()) + log.Infof("volverify report:\n%s", report) + + // present-corrupt is the dangerous silent case (§2.4): a torn-but-present + // volume EVE would serve to the app as-is. orphaned/lost are the recoverable + // modes (recreate path). Fail on present-corrupt; surface the rest for the + // soak to tally. Phase 3's detect→recreate should drive present-corrupt to 0. + presentCorrupt := reportField(report, "present-corrupt") + t.Expect(presentCorrupt).To(BeNumerically(">=", 0), + "could not parse present-corrupt count from report:\n%s", report) + t.Expect(presentCorrupt).To(Equal(0), + "data volume served corrupt-but-present after the interrupted shrink:\n%s", report) + evetest.Checkpoint("volume-verified") +} + +// reportField extracts an integer "key=N" field from a volverify summary line. +func reportField(report, key string) int { + for _, tok := range strings.Fields(report) { + if strings.HasPrefix(tok, key+"=") { + var n int + _, err := fmt.Sscanf(strings.TrimPrefix(tok, key+"="), "%d", &n) + if err != nil { + return -1 + } + return n + } + } + return -1 +} From 9e898999809042728e72e1f1694eed17eca83f45 Mon Sep 17 00:00:00 2001 From: eriknordmark Date: Mon, 27 Jul 2026 17:41:18 -0700 Subject: [PATCH 04/29] =?UTF-8?q?evetest:=20add=20kvm=E2=86=92k=20resize?= =?UTF-8?q?=20app-volume=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add TestKvmToKResize: it drives EVE-kvm→EVE-k across the offline boot-disk shrink+grow while keeping a container app and its data volume, then asserts the volume's marker survived the conversion. Includes ordered post-conversion readiness gates (volumemgr Initialized, Longhorn StorageClass, app RUNNING, app SSH) for failure localization, a /persist + Longhorn node-disk accounting capture (pre-conversion baseline plus a sampler through the EVE-k startup window) for diagnosing the post-conversion app-provisioning wedge, and DISK_SIZE_MB / RAM_SIZE_MB / CPUS / DATAVOL_MB parameters with floors matching the eden kvm→k bringup (64 GiB disk, 16 GiB RAM, 8 vCPU). Also add EdgeDevice.ExpectAdditionalReboots so a test can declare the extra device-initiated reboots an offline shrink+grow performs, keeping the harness teardown reboot-count check accurate. Signed-off-by: eriknordmark Co-Authored-By: Claude Opus 4.8 --- evetest/edgedevice.go | 12 + evetest/tests/resize/kvmtok_resize_test.go | 712 +++++++++++++++++++++ 2 files changed, 724 insertions(+) create mode 100644 evetest/tests/resize/kvmtok_resize_test.go diff --git a/evetest/edgedevice.go b/evetest/edgedevice.go index 45ac71dffb7..21c3b9a067b 100644 --- a/evetest/edgedevice.go +++ b/evetest/edgedevice.go @@ -626,6 +626,18 @@ func (d *EdgeDevice) HardReboot(waitUntilRebooted bool) { }) } +// ExpectAdditionalReboots tells the harness to expect n more device-initiated +// reboots beyond those it already accounts for (UpgradeEVE counts one reboot per +// upgrade). A cross-flavor boot-disk conversion that runs an offline shrink/grow +// reboots additional times; the number is known only to the test driving it (a +// plain kvm<->k conversion with no resize reboots differently than a shrink+grow), +// so the test declares it here to keep the teardown reboot-count check accurate. +func (d *EdgeDevice) ExpectAdditionalReboots(n int) { + for i := 0; i < n; i++ { + d.th.incExpectedRebootCount(d.devName) + } +} + // rebootAndWait executes triggerFn to initiate a device reboot and, if // wait is true, blocks until the device confirms the reboot by reporting // a ZInfoDevice.lastRebootTime strictly after the moment triggerFn was called. diff --git a/evetest/tests/resize/kvmtok_resize_test.go b/evetest/tests/resize/kvmtok_resize_test.go new file mode 100644 index 00000000000..5be64375a2f --- /dev/null +++ b/evetest/tests/resize/kvmtok_resize_test.go @@ -0,0 +1,712 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Package resize_test reproduces, in evetest, the eden EVE-kvm→EVE-k in-field +// boot-disk shrink+grow conversion — the proven sequence in eden's +// update_eve_image_kvm_to_k.txt (EXPECT_DECISION=shrink), one step at a time. +// +// This is the "basic resize" stage (no application volume, no fault injection): +// it proves evetest can drive the whole small→large conversion with the shrink +// and keep a container app alive across it. The ordering and the waits mirror the +// eden escript deliberately — they are load-bearing (vault must settle to a local +// TPM unlock before the app; the offline resize reboots several times; on EVE-k +// the volumemgr → longhorn → app → SSH readiness gates must be waited on in order). +// +// The one deliberate divergence from the eden escript: it DELETES the app before +// the k update (the old cross-flavor gate refused any volume). With the gate +// lifted (baseosmgr allow-shrink-with-volumes), this test KEEPS the app across the +// conversion. +package resize_test + +import ( + "fmt" + "net" + "strings" + "testing" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + uuid "github.com/satori/go.uuid" + + eveconfig "github.com/lf-edge/eve-api/go/config" + "github.com/lf-edge/eve-api/go/evecommon" + eveinfo "github.com/lf-edge/eve-api/go/info" + "github.com/lf-edge/eve/evetest" + "github.com/lf-edge/eve/evetest/constants" + "github.com/lf-edge/eve/evetest/matchers" + "github.com/lf-edge/eve/evetest/netmodels" +) + +const ( + initialEVEVersionParamKey = "INITIAL_EVE_VERSION" + initialHypervisorParamKey = "INITIAL_HYPERVISOR" + ramSizeMiBParamKey = "RAM_SIZE_MB" + cpusParamKey = "CPUS" + + // minDeviceRAMInMiB and minDeviceCPUs are the resource floors for the kvm→k + // conversion. EVE-k (kubevirt + Longhorn) needs well above the 8 GiB / 4 vCPU + // framework defaults; the eden runs that pass 7/7 use 16 GiB and 8 vCPUs, so + // require at least that here. + minDeviceRAMInMiB = 16384 + minDeviceCPUs = 8 + + appSSHUser = "root" + appSSHPassword = "testpassword" + appSSHFwdPort = 2222 + sshTimeout = 20 * time.Second + + // Stage B: a blank data volume attached to the app, with a marker written into + // it on EVE-kvm that must survive the shrink conversion onto EVE-k. + dataMountDir = "/mnt/data" + dataVolMiBParamKey = "DATAVOL_MB" // app data-volume size sweep: 2 GiB wedges the CSI create/verify race, 100 MiB passes + volMarker = "STAGE-B-VOL-MARKER-9f3a1c7e-survives-kvm-to-k" +) + +// TestKvmToKResize drives 16.6.0-kvm (small) → current EVE-kvm (same geometry, +// lands the conversion code) → current EVE-k (cross-flavor ⇒ offline shrink+grow), +// keeping a container app across the conversion. +// +// Parameters: +// - INITIAL_EVE_VERSION (required, e.g. "16.6.0") / INITIAL_HYPERVISOR (kvm) — +// the SMALL-geometry base; it must lack the large EVE-k geometry or the +// conversion is a no-op and proves nothing (asserted). +// - EVE_VERSION / HYPERVISOR — the conversion-capable current build; the kvm +// variant is the intermediate hop, the kubevirt variant is the k target. +// - DISK_SIZE_MB — keep the boot disk full (no free tail) so the conversion +// SHRINKS P3 rather than growing into free space (asserted: decision==shrink). +func TestKvmToKResize(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.EVEVersionParameter(), + evetest.HypervisorParameter(), + evetest.TPMParameter(), + evetest.DiskSizeMiBParameter(), + evetest.TestParameterDefinition{ + Key: ramSizeMiBParamKey, + DefaultValue: uint32(minDeviceRAMInMiB), + Description: evetest.TestParameterDescription{ + Summary: "Device RAM in MiB (EVE-k + Longhorn need >= 16 GiB)", + Default: "16384 (16 GiB)", + }, + }, + evetest.TestParameterDefinition{ + Key: cpusParamKey, + DefaultValue: uint8(minDeviceCPUs), + Description: evetest.TestParameterDescription{ + Summary: "Device vCPUs (EVE-k + Longhorn need >= 8)", + Default: "8", + }, + }, + evetest.TestParameterDefinition{ + Key: dataVolMiBParamKey, + DefaultValue: uint32(2048), + Description: evetest.TestParameterDescription{ + Summary: "App data-volume size in MiB (size->CSI-race sweep)", + Default: "2048 (2 GiB)", + }, + }, + evetest.TestParameterDefinition{ + Key: initialEVEVersionParamKey, + DefaultValue: "16.6.0", + Description: evetest.TestParameterDescription{ + Summary: "SMALL-geometry EVE-kvm base to start on (pre-large-geometry)", + Default: "16.6.0", + }, + }, + evetest.TestParameterDefinition{ + Key: initialHypervisorParamKey, + DefaultValue: evetest.HypervisorKVM, + Description: evetest.TestParameterDescription{ + Summary: "Hypervisor of the initial (small) base", + Default: "kvm", + AllowedValues: "kvm", + }, + }, + ) + + withTPM := evetest.GetTPMParameterValue() + diskSizeMiB := evetest.GetDiskSizeMiBParameterValue() + initialVersion := evetest.GetTestParameter[string](initialEVEVersionParamKey) + if initialVersion == "" { + evetestT.Fatalf("%s%s is required", constants.EnvPrefix, initialEVEVersionParamKey) + } + initialHypervisor := evetest.GetTestParameter[evetest.Hypervisor](initialHypervisorParamKey) + convVersion := evetest.GetEVEVersionParameterValue() // the conversion-capable build (kvm hop + k target share this version) + targetHypervisor := evetest.GetHypervisorParameterValue() + + // The kvm→k conversion provisions Longhorn/kubevirt onto P3; a boot disk + // smaller than the framework default (64 GiB) starves that geometry and the + // conversion wedges instead of failing cleanly. Reject an undersized + // DISK_SIZE_MB up front rather than burning a full run to discover it. + effectiveDiskMiB := diskSizeMiB + if effectiveDiskMiB == 0 { + effectiveDiskMiB = constants.DefaultEVEDeviceDiskSizeInMiB + } + if effectiveDiskMiB < constants.DefaultEVEDeviceDiskSizeInMiB { + evetestT.Fatalf("boot disk %d MiB is too small for the kvm→k conversion; "+ + "need at least %d MiB (64 GiB) — set DISK_SIZE_MB accordingly", + effectiveDiskMiB, constants.DefaultEVEDeviceDiskSizeInMiB) + } + ramSizeMiB := evetest.GetTestParameter[uint32](ramSizeMiBParamKey) + effectiveRAMMiB := ramSizeMiB + if effectiveRAMMiB == 0 { + effectiveRAMMiB = constants.DefaultEVEDeviceRAMInMiB + } + if effectiveRAMMiB < minDeviceRAMInMiB { + evetestT.Fatalf("device RAM %d MiB is too small for the kvm→k conversion "+ + "(EVE-k + Longhorn); need at least %d MiB (16 GiB) — set RAM_SIZE_MB accordingly", + effectiveRAMMiB, minDeviceRAMInMiB) + } + cpus := evetest.GetTestParameter[uint8](cpusParamKey) + effectiveCPUs := cpus + if effectiveCPUs == 0 { + effectiveCPUs = constants.DefaultEVEDeviceCPUs + } + if effectiveCPUs < minDeviceCPUs { + evetestT.Fatalf("device vCPUs %d is too few for the kvm→k conversion "+ + "(EVE-k + Longhorn); need at least %d — set CPUS accordingly", + effectiveCPUs, minDeviceCPUs) + } + dataVolBytes := uint64(evetest.GetTestParameter[uint32](dataVolMiBParamKey)) * evetest.MiB + + const devName = "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithEVEVersion: initialVersion, + WithHypervisor: initialHypervisor, + WithTPM: withTPM, + MinDiskSizeInMiB: diskSizeMiB, + MinRAMInMiB: effectiveRAMMiB, + MinCPUs: effectiveCPUs, + // Provision from the LIVE image (as eden does), not the installer. The + // installer ESP carries a 0-byte marker (boot/.boot_repository) that the + // offline grow's FAT32 copy (go-diskfs) rejects with "invalid start + // cluster: 0"; the live ESP has no such file. See the go-diskfs issue. + DeviceReusePolicy: evetest.CreateFromScratchWithLiveImage, + }, + evetest.RequireNetworkModel{NetworkModel: netmodels.SingleEthWithDHCP}, + ) + device := evetest.GetEdgeDevice(devName) + log := evetest.Logger() + log.Infof("app data-volume size: %d MiB", dataVolBytes/evetest.MiB) + + // Management + app network. + devConfig := evetest.NewEdgeDeviceConfig(devName) + networkUUID := devConfig.AddNetwork(evetest.DHCPNetworkConfig{NetworkType: evecommon.NetworkType_V4}) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "eth0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: networkUUID, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + device.ApplyConfig(devConfig, false, false) + + // Step 1: baseline — the boot disk must be SMALL, else the conversion is a no-op. + log.Infof("baseline: asserting SMALL boot-disk geometry") + assertSmallGeometry(t, device) + evetest.Checkpoint("baseline-small") + + // Step 2: kvm→kvm hop — lands the conversion code, geometry unchanged. + log.Infof("kvm→kvm hop: upgrading to the conversion-capable build %s (kvm)", convVersion) + device.UpgradeEVE(convVersion, evetest.HypervisorKVM, true, false) + log.Infof("post-kvm-hop: geometry must still be SMALL") + assertSmallGeometry(t, device) + log.Infof("post-kvm-hop: storage-resizer check must decide 'shrink'") + assertCheckDecision(t, device, "shrink") + evetest.Checkpoint("kvm-hop-done") + + // Step 3: settle the vault to a LOCAL TPM unlock (a new rootfs moves PCRs, so + // the first boot unlocks via the controller key; reboot until it seals locally). + log.Infof("settling vault to a local TPM unlock") + settleVaultLocal(t, device) + evetest.Checkpoint("vault-settled") + + // Step 4: deploy a container app + a blank DATA VOLUME on EVE-kvm; wait RUNNING + // + SSH-reachable, then write a marker into the volume. + // SWITCH (L2, bridged) NI: the app gets its own DHCP address on the eth0 segment + // and is reached directly at app-IP:22 (the port-map is kept as a 2nd candidate + // endpoint). Verified: the LOCAL NI does NOT reconverge onto the kubevirt VMI + // after the conversion (no app IP even after a 10-min wait), whereas the switch + // NI does (~3.5 min); the readiness gate (step 7) absorbs that delay. + niUUID := devConfig.AddNetworkInstance(evetest.SwitchNetworkInstanceConfig{ + DisplayName: "switch-ni", + Port: "eth0", + }) + appUUID := devConfig.AddApplication(evetest.ApplicationInstanceConfig{ + DisplayName: "resize-test-app", + Activate: true, + Image: evetest.DockerContainer{ImageName: "milan4zededa/evetest-ubuntu-ctr", Tag: "1.0"}, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 512 * evetest.MiB, + NetworkAdapters: []evetest.AppNetworkAdapter{ + evetest.VirtualNetworkAdapter{ + LogicalLabel: "vif0", + NetworkInstanceUUID: niUUID, + PortFwdRules: []evetest.PortFwdRule{ + {Protocol: evetest.NetworkProtocolTCP, EdgeNodePort: appSSHFwdPort, AppPort: 22}, + }, + ACLAllowRules: []evetest.ACLAllowRule{ + {Protocol: evetest.NetworkProtocolAny, RemoteSubnet: evetest.IPSubnet("0.0.0.0/0")}, + }, + }, + }, + DataVolumes: []evetest.DataVolumeConfig{ + {SizeBytes: dataVolBytes, MountDir: dataMountDir}, + }, + }) + device.ApplyConfig(devConfig, false, false) + device.WaitUntilAppIsRunning(appUUID, 5*time.Minute) + appAuth := evetest.UsernamePasswordAuth{Username: appSSHUser, Password: appSSHPassword} + log.Infof("pre-conversion: app must be SSH-reachable") + assertAppSSH(t, device, appUUID, appAuth) + // Baseline network snapshot on EVE-kvm where the port-map SSH works, to diff + // against the post-conversion EVE-k snapshot if that SSH fails. + captureAppNet(device, "pre-conversion EVE-kvm (SSH OK)") + log.Infof("pre-conversion: writing a marker into the app data volume at %s", dataMountDir) + writeVolumeMarker(t, device, appUUID, appAuth, dataMountDir, volMarker) + capturePersist(device, "pre-conversion EVE-kvm (app running)") + evetest.Checkpoint("pre-conversion-app-ready") + + // Step 5: kvm→k — the cross-flavor seam triggers the offline shrink+grow. The + // app is KEPT (gate lifted). UpgradeEVE drives the multi-reboot conversion. + log.Infof("kvm→k conversion: upgrading to %s (%s) — triggers the offline shrink+grow", convVersion, targetHypervisor) + // On conversion failure the device returns online on the unconverted kvm + // partition — still SSH-reachable — while EVE flags the -k baseos FAILED, and + // UpgradeEVE fatals. Capture the on-device reason first (the deferred probe runs + // before harness teardown; conversionOK stays false because the fatal skips it). + conversionOK := false + defer func() { + if !conversionOK { + dumpConversionFailure(device) + } + }() + device.UpgradeEVE(convVersion, targetHypervisor, true, false) + conversionOK = true + // The offline shrink+grow reboots once more than UpgradeEVE accounts for (its + // intermediate initrd-resize boot is invisible to the controller, but the + // post-resize kvm boot and the -k boot are two observed reboots vs the one + // UpgradeEVE expects). Declare that extra reboot so the teardown check balances. + device.ExpectAdditionalReboots(1) + evetest.Checkpoint("conversion-complete") + + // Step 6: assert the boot disk grew to LARGE and P3 shrank. + log.Infof("post-conversion: asserting boot disk grew (shrink path: ESP-B created, P3 shrank)") + assertGrownShrink(t, device) + evetest.Checkpoint("geometry-grown") + + // Sample /persist + Longhorn disk accounting through the EVE-k startup window + // (k3s → volumemgr → Longhorn SC → app), to see the "no available disk for + // replica" accounting as it evolves. Stopped once readiness resolves below. + stopPersistSampler := startPersistSampler(device, 45*time.Second) + defer stopPersistSampler() + + // Step 7: post-conversion readiness — in order, so a failure localizes. + if targetHypervisor == evetest.HypervisorKubevirt { + log.Infof("(a) waiting for k3s node ready") + if !isK3sReady(device.GetClusterInfo()) { + clusterUpdates, stop := device.WatchClusterInfo() + defer stop() + t.Eventually(clusterUpdates, 20*time.Minute).Should(Receive( + matchers.SatisfyPredicate("K3s node is ready", isK3sReady))) + } + log.Infof("(b) waiting for volumemgr Initialized") + waitVolumemgrReady(t, device) + log.Infof("(c) waiting for longhorn StorageClass ready") + waitLonghornSC(t, device) + } + log.Infof("(d) waiting for the app to reach RUNNING on the target") + device.WaitUntilAppIsRunning(appUUID, 65*time.Minute) + // If the app network doesn't come back on EVE-k, snapshot the app network state + // for comparison against the pre-conversion EVE-kvm baseline (the deferred probe + // runs before teardown; appSSHOK stays false on the fatal). + appSSHOK := false + defer func() { + if !appSSHOK { + captureAppNet(device, "post-conversion EVE-k (app net FAILED)") + captureAppNetFailure(device, appUUID) + } + }() + // Readiness gate: the carried app's network reconvergence on EVE-k is slow / + // non-deterministic, so wait (generously) for it to report a routable IPv4 + // before asserting SSH. This separates "not ready yet" from "unreachable": a + // slow reconvergence still passes; a genuinely stuck one fails here with the + // network snapshot, distinguishing it from an SSH/routing problem. + log.Infof("(e0) post-conversion: waiting up to 10m for the app to report a routable IPv4") + waitAppHasRoutableIPv4(t, device, appUUID, 10*time.Minute) + log.Infof("(e) post-conversion: app must be SSH-reachable") + assertAppSSH(t, device, appUUID, appAuth) + appSSHOK = true + // (f) the data volume — and its marker — must have survived the shrink onto + // EVE-k. EVE-k does not auto-mount the container data volume at MountDir + // (lf-edge/eve#6145), so read it off the raw block device from inside the app. + log.Infof("(f) post-conversion: the app data-volume marker must survive on EVE-k") + assertVolumeMarker(t, device, appUUID, appAuth, volMarker) + evetest.Checkpoint("post-conversion-app-ready") +} + +// ---- helpers (mirror the eden escript's assert/wait scripts) ---- + +const eveShellTimeout = 30 * time.Second + +// dumpConversionFailure captures why the kvm→k conversion was marked FAILED by +// EVE. At that point the device has returned online on the unconverted kvm +// partition and is SSH-reachable, so it can be queried directly. Best-effort: it +// logs whatever each probe returns and never fails the (already-failing) test. +func dumpConversionFailure(device *evetest.EdgeDevice) { + log := evetest.Logger() + log.Errorf("kvm→k conversion FAILED — capturing on-device diagnostics") + probes := []struct{ what, script string }{ + {"resize-failed.json", "cat /config/resize-failed.json 2>/dev/null || echo NONE"}, + {"resize-flags", "echo repartition-inprogress=$(cat /config/repartition-inprogress 2>/dev/null); echo resize-reboots=$(cat /config/resize-reboots 2>/dev/null)"}, + {"zboot-status", "zboot status 2>&1 || true"}, + {"BaseOsStatus", "eve exec pillar sh -c 'cat /run/baseosmgr/BaseOsStatus/*.json 2>/dev/null' || echo NONE"}, + } + for _, p := range probes { + out, errOut, err := device.RunShellScript(p.script, eveShellTimeout, 0) + log.Errorf("[capture:%s]\n%s%s(err=%v)", p.what, out, errOut, err) + } +} + +// captureAppNet snapshots the EVE-side app networking behind the port-map SSH: +// the NAT DNAT rules (node:2222→app:22), the app's assigned IP + applied ACLs, +// and the node bridges/IPs. Taken on EVE-kvm (working) and again on EVE-k +// (failing) so the two can be diffed. Best-effort; never fails the test. +func captureAppNet(device *evetest.EdgeDevice, label string) { + log := evetest.Logger() + log.Errorf("=== app-network capture: %s ===", label) + probes := []struct{ what, script string }{ + {"nat-portmap", "eve exec pillar iptables -t nat -S 2>/dev/null | grep -aiE '2222|DNAT|to-destination' || echo none"}, + {"AppNetworkStatus", "eve exec pillar sh -c 'cat /run/zedrouter/AppNetworkStatus/*.json 2>/dev/null' || echo none"}, + {"bridges+ips", "eve exec pillar ip -br addr 2>/dev/null || echo none"}, + } + for _, p := range probes { + out, errOut, err := device.RunShellScript(p.script, eveShellTimeout, 0) + log.Errorf("[net:%s] %s:\n%s%s(err=%v)", label, p.what, strings.TrimSpace(out), errOut, err) + } +} + +// captureAppNetFailure characterizes the known EVE-k app-pvc-not-ready / CDI +// upload-pod wedge that leaves the app stuck CREATING_VOLUME (no VMI → no net). +// It grabs the signals ~/notes/kvm-to-k-resize-soak-app-pvc-not-ready-stats.md +// prescribes for pinning the variant (V1–V5) and whether the image's recovery +// engaged: the scratch/data PVC state (Terminating? storageClass?), the cdi-upload +// pod phase/reason, Longhorn volume+attachment state, the CDI upload-controller +// reconcile errors, and the volumemgr / MOUNT-WEDGE-RECOVERY newlog signatures. +func captureAppNetFailure(device *evetest.EdgeDevice, appUUID uuid.UUID) { + log := evetest.Logger() + log.Errorf("=== EVE-k app-pvc-not-ready / CDI-wedge diagnostics (app %s) ===", appUUID) + probes := []struct{ what, script string }{ + {"eve-kube-app pvc+pods -o wide", `eve exec kube kubectl -n eve-kube-app get pvc,pods -o wide 2>/dev/null || echo none`}, + {"describe pvc (scratch Terminating/SC/finalizers/events)", `eve exec kube kubectl -n eve-kube-app describe pvc 2>/dev/null || echo none`}, + {"describe pods (cdi-upload phase/reason/events)", `eve exec kube kubectl -n eve-kube-app describe pods 2>/dev/null || echo none`}, + {"longhorn volumes", `eve exec kube kubectl get volumes.longhorn.io -A -o wide 2>/dev/null || echo none`}, + {"volumeattachments", `eve exec kube kubectl get volumeattachments 2>/dev/null || echo none`}, + {"longhorn-csi-provisioner log (CreateVolume 404/500 origin)", `eve exec kube kubectl -n longhorn-system logs deployment/csi-provisioner -c csi-provisioner --tail=250 2>/dev/null || echo none`}, + {"longhorn-manager log (create/verify race)", `eve exec kube kubectl -n longhorn-system logs -l app=longhorn-manager -c longhorn-manager --tail=250 --prefix 2>/dev/null || echo none`}, + {"longhorn StorageClass (replicas/config)", `eve exec kube kubectl get sc longhorn -o yaml 2>/dev/null || echo none`}, + {"longhorn version (manager ds image)", `eve exec kube kubectl -n longhorn-system get ds longhorn-manager -o wide 2>/dev/null || echo none`}, + {"CDI upload-controller log", `eve exec kube kubectl -n cdi logs deployment/cdi-deployment --tail=60 2>/dev/null || echo none`}, + {"volumemgr RolloutDiskToPVC / V5 signature (newlog)", `eve exec pillar sh -c 'grep -aiE "RolloutDiskToPVC|retryFailedClusterVolumeCreate|terminating:true|local-path" /persist/newlog/collect/*.log 2>/dev/null | tail -30' || echo none`}, + {"MOUNT-WEDGE-RECOVERY (detector fired?)", `eve exec pillar sh -c 'grep -ai MOUNT-WEDGE-RECOVERY /persist/newlog/collect/*.log 2>/dev/null | tail -10' || echo none`}, + {"kubevirt vmi -A", `eve exec kube kubectl get vmi -A -o wide 2>/dev/null || echo none`}, + } + for _, p := range probes { + out, errOut, err := device.RunShellScript(p.script, 90*time.Second, 0) + log.Errorf("[wedge:%s]\n%s%s(err=%v)", p.what, strings.TrimSpace(out), errOut, err) + } +} + +// capturePersist snapshots everything about /persist sizing/usage plus Longhorn's +// own view of the node disk — storageMaximum/Available/Scheduled/Reserved and the +// scheduling settings — i.e. the accounting behind "no available disk for replica". +// The kube/longhorn probes no-op on EVE-kvm (pre-conversion) and light up on EVE-k. +// Best-effort; never fails the test. +func capturePersist(device *evetest.EdgeDevice, label string) { + log := evetest.Logger() + log.Errorf("=== /persist capture: %s ===", label) + probes := []struct{ what, script string }{ + {"df persist+submounts", `eve exec pillar sh -c 'df -B1 | awk "NR==1 || /persist/"' 2>/dev/null || echo none`}, + {"lsblk sizes+mounts", `eve exec pillar lsblk -b -o NAME,PARTLABEL,SIZE,FSTYPE,MOUNTPOINT 2>/dev/null || echo none`}, + {"du -d1 /persist", `eve exec pillar sh -c 'du -x -B1 -d1 /persist 2>/dev/null | sort -rn' || echo none`}, + {"longhorn node disk accounting", `eve exec kube kubectl -n longhorn-system get nodes.longhorn.io -o yaml 2>/dev/null | grep -aE "name:|path:|storageMaximum|storageAvailable|storageScheduled|storageReserved|allowScheduling|diskUUID|type:|reason:|message:" || echo none`}, + {"longhorn node -o wide", `eve exec kube kubectl -n longhorn-system get nodes.longhorn.io -o wide 2>/dev/null || echo none`}, + {"longhorn storage settings", `eve exec kube kubectl -n longhorn-system get settings.longhorn.io -o custom-columns=NAME:.metadata.name,VALUE:.value 2>/dev/null | grep -aiE "NAME|reserved|over-provisioning|minimal-available|soft-anti-affinity" || echo none`}, + {"eve-kube-app pvc (requested sizes)", `eve exec kube kubectl -n eve-kube-app get pvc 2>/dev/null || echo none`}, + } + for _, p := range probes { + out, errOut, err := device.RunShellScript(p.script, 60*time.Second, 0) + log.Errorf("[persist:%s] %s:\n%s%s(err=%v)", label, p.what, strings.TrimSpace(out), errOut, err) + } +} + +// startPersistSampler runs capturePersist in the background every interval (bounded) +// so /persist + Longhorn disk accounting can be watched evolving while k3s/Longhorn/ +// CDI start after the conversion. Returns a stop func (call once, e.g. via defer). +func startPersistSampler(device *evetest.EdgeDevice, interval time.Duration) (stop func()) { + done := make(chan struct{}) + go func() { + for n := 0; n < 13; n++ { + capturePersist(device, fmt.Sprintf("post-conversion startup sample #%d", n)) + select { + case <-done: + return + case <-time.After(interval): + } + } + }() + return func() { close(done) } +} + +// partsLsblk enumerates partition sizes across ALL block devices (not a fixed +// /dev/sda — evetest's broker QEMU presents the boot disk as virtio /dev/vda). +// We identify partitions by PARTLABEL, so the parent disk name doesn't matter. +const partsLsblk = "eve exec pillar lsblk -b -P -o NAME,PARTLABEL,PARTUUID,SIZE" + +// runEVE runs a command on EVE and returns cleaned stdout (dropping empty and +// level=… lines) plus any error, WITHOUT asserting — so callers can retry, since +// EVE SSH/pillar can be briefly unavailable right after a reboot or the conversion. +func runEVE(device *evetest.EdgeDevice, script string) (string, error) { + stdout, _, err := device.RunShellScript(script, eveShellTimeout, 0) + if err != nil { + return "", err + } + var lines []string + for _, l := range strings.Split(stdout, "\n") { + if strings.TrimSpace(l) == "" || strings.Contains(l, "level=") { + continue + } + lines = append(lines, l) + } + return strings.Join(lines, "\n"), nil +} + +// assertSmallGeometry asserts ESP/IMGA/IMGB are small and no ESP-B exists yet +// (mirror capture-partitions.sh assert-small). Retried: it also runs right after +// the kvm→kvm reboot. +func assertSmallGeometry(t Gomega, device *evetest.EdgeDevice) { + t.Eventually(func(g Gomega) { + out, err := runEVE(device, partsLsblk) + g.Expect(err).NotTo(HaveOccurred()) + // ESP-B carries the fresh-install GUID ...30056; it exists only after the grow. + g.Expect(strings.ToLower(out)).NotTo(ContainSubstring("30056"), + "SMALL geometry must not have an ESP-B yet:\n%s", out) + imga := partSizeBytes(out, "IMGA") + imgb := partSizeBytes(out, "IMGB") + const oneGiB = int64(1) << 30 + g.Expect(imga).To(BeNumerically(">", 0), "could not read IMGA size:\n%s", out) + g.Expect(imga).To(BeNumerically("<", oneGiB), "IMGA is not SMALL (%d bytes):\n%s", imga, out) + g.Expect(imgb).To(BeNumerically("<", oneGiB), "IMGB is not SMALL (%d bytes):\n%s", imgb, out) + }, 3*time.Minute, 10*time.Second).Should(Succeed()) +} + +// assertGrownShrink asserts the LARGE layout after the conversion: the reserved +// ESP-B was created and IMGA/IMGB grew past 8 GiB (mirror capture-partitions.sh +// assert-grown). That the shrink path (not grow) was taken is confirmed earlier by +// assertCheckDecision(shrink). Retried: it runs right after the EVE-k boot. +func assertGrownShrink(t Gomega, device *evetest.EdgeDevice) { + t.Eventually(func(g Gomega) { + out, err := runEVE(device, partsLsblk) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(strings.ToLower(out)).To(ContainSubstring("30056"), + "grow must have created the reserved ESP-B (GUID ...30056):\n%s", out) + const eightGiB = int64(8) << 30 + g.Expect(partSizeBytes(out, "IMGA")).To(BeNumerically(">=", eightGiB), "IMGA did not grow to LARGE:\n%s", out) + g.Expect(partSizeBytes(out, "IMGB")).To(BeNumerically(">=", eightGiB), "IMGB did not grow to LARGE:\n%s", out) + }, 5*time.Minute, 10*time.Second).Should(Succeed()) +} + +// partSizeBytes extracts the SIZE of the partition with PARTLABEL==label from a +// `lsblk -b -P` dump. +func partSizeBytes(lsblk, label string) int64 { + for _, line := range strings.Split(lsblk, "\n") { + if !strings.Contains(line, fmt.Sprintf("PARTLABEL=%q", label)) { + continue + } + for _, field := range strings.Fields(line) { + if strings.HasPrefix(field, "SIZE=") { + var n int64 + fmt.Sscanf(strings.Trim(strings.TrimPrefix(field, "SIZE="), `"`), "%d", &n) + return n + } + } + } + return 0 +} + +// bootDiskPath resolves the boot disk (the parent of the IMGA partition), e.g. +// /dev/vda under evetest's virtio QEMU. +func bootDiskPath(device *evetest.EdgeDevice) (string, error) { + out, err := runEVE(device, `eve exec pillar sh -c 'lsblk -ndo pkname $(findfs PARTLABEL=IMGA)'`) + if err != nil { + return "", err + } + name := strings.TrimSpace(out) + if name == "" { + return "", fmt.Errorf("could not resolve boot disk (IMGA parent)") + } + return "/dev/" + name, nil +} + +// assertCheckDecision asserts storage-resizer's pre-flight check returns the +// expected decision on the live boot disk (mirror assert-check-decision.sh). +func assertCheckDecision(t Gomega, device *evetest.EdgeDevice, want string) { + t.Eventually(func(g Gomega) { + disk, err := bootDiskPath(device) + g.Expect(err).NotTo(HaveOccurred()) + out, err := runEVE(device, "eve exec pillar /usr/bin/storage-resizer check --disk "+disk+" --json") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(out).To(ContainSubstring(fmt.Sprintf("%q", want)), + "storage-resizer check did not decide %q:\n%s", want, out) + }, 3*time.Minute, 10*time.Second).Should(Succeed()) +} + +// settleVaultLocal reboots through the controller until the vault reports a local +// TPM unlock (UnlockMethod==1); a controller-key unlock (2) triggers a reboot, +// no-tpm (3) fails (mirror settle-vault-local.sh). +func settleVaultLocal(t Gomega, device *evetest.EdgeDevice) { + log := evetest.Logger() + readUnlock := func() string { + out, _, err := device.RunShellScript( + `eve exec pillar sh -c "cat /run/vaultmgr/VaultStatus/*.json 2>/dev/null"`, eveShellTimeout, 0) + if err != nil { + return "" + } + switch { + case strings.Contains(out, `"UnlockMethod":1`): + return "local" + case strings.Contains(out, `"UnlockMethod":2`): + return "controller" + case strings.Contains(out, `"UnlockMethod":3`): + return "no-tpm" + } + return "" + } + for attempt := 1; attempt <= 4; attempt++ { + var decided string + t.Eventually(func() string { decided = readUnlock(); return decided }, 12*time.Minute, 10*time.Second). + ShouldNot(BeEmpty(), "vault never reported an unlock method") + switch decided { + case "local": + log.Infof("vault settled: local TPM unlock") + return + case "no-tpm": + t.Expect(decided).NotTo(Equal("no-tpm"), "vault is no-tpm; this test requires TPM") + return + default: // controller-key: reboot to re-seal + log.Infof("vault unlock=controller-key (attempt %d); rebooting to re-seal", attempt) + device.RequestReboot(true) + } + } + t.Expect(false).To(BeTrue(), "vault did not settle to a local unlock after 4 reboots") +} + +// waitVolumemgrReady waits for volumemgr Initialized:true (mirror +// wait-for-volumemgr-ready.sh). The budget must exceed volumemgr's own pre-publish +// block on a slow EVE-k bring-up: up to 20m in WaitForKubernetes (node + KubeVirt + +// Longhorn) plus up to 20m more in storageWait before any VolumeMgrStatus is +// published, so 45m could expire right as volumemgr is released (eden 29346e15). +func waitVolumemgrReady(t Gomega, device *evetest.EdgeDevice) { + t.Eventually(func() string { + out, _, _ := device.RunShellScript( + "eve exec pillar cat /run/volumemgr/VolumeMgrStatus/volumemgr.json 2>/dev/null", eveShellTimeout, 0) + return out + }, 60*time.Minute, 15*time.Second).Should(ContainSubstring(`"Initialized":true`), + "volumemgr did not reach Initialized:true") +} + +// waitLonghornSC waits for the longhorn StorageClass (mirror wait-for-longhorn-sc.sh). +func waitLonghornSC(t Gomega, device *evetest.EdgeDevice) { + t.Eventually(func() string { + out, _, _ := device.RunShellScript("eve exec kube kubectl get sc 2>/dev/null", eveShellTimeout, 0) + return out + }, 50*time.Minute, 30*time.Second).Should(ContainSubstring("longhorn"), + "longhorn StorageClass not ready") +} + +// waitAppHasRoutableIPv4 blocks until the app reports at least one routable +// (non-link-local) IPv4 address in its ZInfoApp, or fatals after timeout. This is +// the app-network readiness signal: without a reported IPv4 the harness cannot +// build a direct app-IP SSH endpoint, so a still-converging app looks the same as +// an unreachable one. It logs the address it settles on. +func waitAppHasRoutableIPv4(t Gomega, device *evetest.EdgeDevice, appUUID uuid.UUID, timeout time.Duration) { + t.Eventually(func(t Gomega) { + info := device.GetAppInfo(appUUID) + t.Expect(info).NotTo(BeNil()) + var found string + for _, netInfo := range info.GetNetwork() { + for _, ipStr := range netInfo.GetIPAddrs() { + ip := net.ParseIP(ipStr) + if ip != nil && ip.To4() != nil && !ip.IsLinkLocalUnicast() { + found = ipStr + } + } + } + t.Expect(found).NotTo(BeEmpty(), "app reports no routable IPv4 yet") + evetest.Logger().Infof("app reports routable IPv4 %s", found) + }, timeout, 5*time.Second).Should(Succeed()) +} + +// assertAppSSH asserts the app is reachable over SSH (returns non-empty hostname). +func assertAppSSH(t Gomega, device *evetest.EdgeDevice, appUUID uuid.UUID, auth evetest.AuthMethod) { + t.Eventually(func(t Gomega) { + out, _, err := device.RunShellScriptInsideApp(appUUID, auth, "hostname", sshTimeout, 0) + t.Expect(err).NotTo(HaveOccurred()) + t.Expect(strings.TrimSpace(out)).NotTo(BeEmpty()) + }, 3*time.Minute, 5*time.Second).Should(Succeed()) +} + +// writeVolumeMarker writes a unique marker into the app's mounted data volume and +// fsyncs it, failing if the volume isn't mounted at mountDir. Run on EVE-kvm, +// where the runx shim formats and mounts the blank volume at MountDir. +func writeVolumeMarker(t Gomega, device *evetest.EdgeDevice, appUUID uuid.UUID, auth evetest.AuthMethod, mountDir, marker string) { + script := fmt.Sprintf( + "grep -q ' %s ' /proc/mounts || { echo NOT-MOUNTED; cat /proc/mounts; exit 1; }; "+ + "echo %q > %s/marker && sync && echo WROTE-OK", + mountDir, marker, mountDir) + t.Eventually(func(t Gomega) { + out, _, err := device.RunShellScriptInsideApp(appUUID, auth, script, 60*time.Second, 0) + t.Expect(err).NotTo(HaveOccurred()) + t.Expect(out).To(ContainSubstring("WROTE-OK")) + }, 2*time.Minute, 5*time.Second).Should(Succeed()) +} + +// assertVolumeMarker verifies the marker survived the conversion by reading it off +// the data volume's RAW block device from inside the app — EVE-k does not +// auto-mount the container data volume at MountDir (lf-edge/eve#6145). The data +// volume is a virtio disk after the rootfs (/dev/vd[b-z]); grep it for the marker. +func assertVolumeMarker(t Gomega, device *evetest.EdgeDevice, appUUID uuid.UUID, auth evetest.AuthMethod, marker string) { + script := fmt.Sprintf( + "for d in /dev/vd[b-z] /dev/sd[b-z]; do [ -b \"$d\" ] || continue; "+ + "grep -aq %q \"$d\" 2>/dev/null && { echo FOUND-ON $d; exit 0; }; done; "+ + "echo NOT-FOUND; ls -l /dev/vd* /dev/sd* 2>/dev/null; lsblk 2>/dev/null; blkid 2>/dev/null; exit 1", + marker) + t.Eventually(func(t Gomega) { + out, _, err := device.RunShellScriptInsideApp(appUUID, auth, script, 90*time.Second, 0) + t.Expect(err).NotTo(HaveOccurred()) + t.Expect(out).To(ContainSubstring("FOUND-ON")) + }, 3*time.Minute, 10*time.Second).Should(Succeed()) +} + +// isK3sReady reports whether the single k3s node is ready with healthy storage. +func isK3sReady(info *eveinfo.ZInfoKubeCluster) bool { + if info == nil || len(info.Nodes) != 1 { + return false + } + if info.Storage.Health != eveinfo.ServiceStatus_SERVICE_STATUS_HEALTHY { + return false + } + for _, cond := range info.Nodes[0].GetConditions() { + if cond.GetType() == eveinfo.KubeNodeConditionType_KUBE_NODE_CONDITION_TYPE_READY { + return cond.GetSet() + } + } + return false +} From 02c8a3330422a44fe65e2212eaf1bb89e5401ef2 Mon Sep 17 00:00:00 2001 From: eriknordmark Date: Mon, 27 Jul 2026 17:41:18 -0700 Subject: [PATCH 05/29] volverify: treat a full volume as a clean stop Writer.Run now returns the committed high-water mark and treats ENOSPC as a normal end state rather than an error: filling the volume is the expected end for the corruption soak, not a failure. The write CLI prints the committed index, and the app-volume shrink test passes that reported index to `verify --expect-committed` instead of assuming every requested op was written (a volume that fills first commits fewer). Signed-off-by: eriknordmark Co-Authored-By: Claude Opus 4.8 --- .../testapps/volverify/cmd/volverify/main.go | 5 ++- .../volverify/internal/verify/engine.go | 38 +++++++++++++++---- .../volverify/internal/verify/engine_test.go | 4 +- .../tests/appvolshrink/appvolshrink_test.go | 19 +++++++--- 4 files changed, 49 insertions(+), 17 deletions(-) diff --git a/evetest/testapps/volverify/cmd/volverify/main.go b/evetest/testapps/volverify/cmd/volverify/main.go index 0abeca762c7..c539f2e22a0 100644 --- a/evetest/testapps/volverify/cmd/volverify/main.go +++ b/evetest/testapps/volverify/cmd/volverify/main.go @@ -65,10 +65,11 @@ func main() { if err != nil { fatal(err) } - if err := w.Run(); err != nil { + committed, err := w.Run() + if err != nil { fatal(err) } - fmt.Println("write: complete") + fmt.Printf("write: complete committed=%d\n", committed) case "verify": rep, err := verify.Verify(*dir, cfg) if err != nil { diff --git a/evetest/testapps/volverify/internal/verify/engine.go b/evetest/testapps/volverify/internal/verify/engine.go index 16b06db1672..c6b13c9d472 100644 --- a/evetest/testapps/volverify/internal/verify/engine.go +++ b/evetest/testapps/volverify/internal/verify/engine.go @@ -4,10 +4,12 @@ package verify import ( + "errors" "fmt" "io" "os" "path/filepath" + "syscall" ) // Writer applies the deterministic op stream to a volume, fsyncing and advancing @@ -26,9 +28,13 @@ func NewWriter(volDir string, cfg Config) (*Writer, error) { return &Writer{volDir: volDir, cfg: cfg}, nil } -// Run applies ops until Config.Ops, resuming after a crash from the committed -// index. It is safe to call repeatedly across reboots on the same volume. -func (w *Writer) Run() error { +// Run applies ops until Config.Ops (or until the volume fills), resuming after a +// crash from the committed index. It is safe to call repeatedly across reboots on +// the same volume. It returns the final committed op index — the high-water mark +// the verifier should expect (equal to Ops-1 on full completion, or lower when the +// volume filled first). Filling the volume is the expected end state for the +// corruption soak, so ENOSPC is a clean stop, not an error. +func (w *Writer) Run() (int64, error) { gen := newGenerator(w.cfg) m := newModel(w.cfg) @@ -39,6 +45,7 @@ func (w *Writer) Run() error { m.apply(gen.next(m, n)) } gen64 := nextGeneration(w.volDir) + lastCommitted := committed touchedFiles := make(map[string]bool) touchedDirs := make(map[string]bool) @@ -58,6 +65,7 @@ func (w *Writer) Run() error { return err } gen64++ + lastCommitted = int64(index) touchedFiles = make(map[string]bool) touchedDirs = make(map[string]bool) return nil @@ -67,19 +75,35 @@ func (w *Writer) Run() error { for n := start; n < w.cfg.Ops; n++ { o := gen.next(m, n) if err := w.applyOp(o, touchedFiles, touchedDirs); err != nil { - return fmt.Errorf("op %d (%v): %w", n, o.typ, err) + if errors.Is(err, syscall.ENOSPC) { + // Volume full: stop cleanly at the last fully-written op. Drop the + // partial file (it is an uncommitted op the verifier ignores) and + // commit everything through n-1, then report that index. + if o.typ == opCreate { + _ = os.Remove(filepath.Join(w.volDir, filePathFor(w.cfg, o.fileID))) + } + if n > start { + if err := commit(n - 1); err != nil { + return lastCommitted, err + } + } + return lastCommitted, nil + } + return lastCommitted, fmt.Errorf("op %d (%v): %w", n, o.typ, err) } m.apply(o) if (n+1)%w.cfg.CommitEvery == 0 { if err := commit(n); err != nil { - return err + return lastCommitted, err } } } if w.cfg.Ops > start { - return commit(w.cfg.Ops - 1) + if err := commit(w.cfg.Ops - 1); err != nil { + return lastCommitted, err + } } - return nil + return lastCommitted, nil } // applyOp performs one op against the volume and records what it touched so the diff --git a/evetest/testapps/volverify/internal/verify/engine_test.go b/evetest/testapps/volverify/internal/verify/engine_test.go index 7192fc4e4e1..b0cf29550a9 100644 --- a/evetest/testapps/volverify/internal/verify/engine_test.go +++ b/evetest/testapps/volverify/internal/verify/engine_test.go @@ -28,7 +28,7 @@ func writeVolume(t *testing.T, cfg Config) (string, *model) { if err != nil { t.Fatal(err) } - if err := w.Run(); err != nil { + if _, err := w.Run(); err != nil { t.Fatal(err) } return dir, committedModel(dir, cfg) @@ -170,7 +170,7 @@ func TestCrashResumeIdempotent(t *testing.T) { // Running the writer again must resume from the commit and leave the volume // clean (idempotent re-application of already-committed ops). w, _ := NewWriter(dir, cfg) - if err := w.Run(); err != nil { + if _, err := w.Run(); err != nil { t.Fatal(err) } rep, _ := Verify(dir, cfg) diff --git a/evetest/tests/appvolshrink/appvolshrink_test.go b/evetest/tests/appvolshrink/appvolshrink_test.go index 3eb080c16e9..13fd242cdc5 100644 --- a/evetest/tests/appvolshrink/appvolshrink_test.go +++ b/evetest/tests/appvolshrink/appvolshrink_test.go @@ -235,14 +235,21 @@ func TestAppVolumeShrinkCorruption(test *testing.T) { appAuth := evetest.UsernamePasswordAuth{Username: appSSHUser, Password: appSSHPassword} // Fill the volume with the deterministic pattern before the shrink. The write - // of a multi-GiB volume can take a while, so allow a generous timeout. + // of a multi-GiB volume can take a while, so allow a generous timeout. It stops + // at Ops or when the volume fills (whichever first) and reports the committed + // high-water mark, which the post-shrink verify must expect. writeCmd := fmt.Sprintf("volverify write --dir %s --seed %d --ops %d --block-size %d", dataMountDir, seed, ops, blockSize) log.Infof("Filling data volume: %s", writeCmd) + writtenCommitted := -1 t.Eventually(func(t Gomega) { - _, stderr, err := device.RunShellScriptInsideApp(appUUID, appAuth, writeCmd, 60*time.Minute, 0) + stdout, stderr, err := device.RunShellScriptInsideApp(appUUID, appAuth, writeCmd, 60*time.Minute, 0) t.Expect(err).NotTo(HaveOccurred(), "write stderr: %s", stderr) + writtenCommitted = reportField(stdout, "committed") + t.Expect(writtenCommitted).To(BeNumerically(">=", 0), + "write did not report a committed index:\n%s", stdout) }, 65*time.Minute, 10*time.Second).Should(Succeed()) + log.Infof("volume filled through committed op %d", writtenCommitted) evetest.Checkpoint("volume-filled") // Drive the watchdog-interrupted kvm→k conversion (shrink relocates the volume). @@ -260,11 +267,11 @@ func TestAppVolumeShrinkCorruption(test *testing.T) { } device.WaitUntilAppIsRunning(appUUID, 5*time.Minute) - // Re-verify. The on-volume commit lives on the shrunk filesystem and may be - // cleared by fsck along with data, so pass the harness high-water mark - // (ops-1) as an off-volume floor (design §4.2). + // Re-verify against the write's high-water mark. The on-volume commit lives on + // the shrunk filesystem and may be cleared by fsck along with data, so pass the + // committed index the pre-shrink write reported as an off-volume floor (§4.2). verifyCmd := fmt.Sprintf("volverify verify --dir %s --seed %d --ops %d --block-size %d --expect-committed %d", - dataMountDir, seed, ops, blockSize, ops-1) + dataMountDir, seed, ops, blockSize, writtenCommitted) log.Infof("Verifying data volume: %s", verifyCmd) var report string t.Eventually(func(t Gomega) { From b5fc822c329e929b9ffc6d0f97bf8e86a9973e36 Mon Sep 17 00:00:00 2001 From: eriknordmark Date: Mon, 27 Jul 2026 20:03:25 -0700 Subject: [PATCH 06/29] evetest: fold appvol shrink test into resize MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The app-volume shrink-corruption test was written before the kvm→k conversion had ever completed under evetest, and it carried none of what that took: it provisioned from the installer image (whose ESP holds a 0-byte file the offline grow cannot copy), gave the app a local network instance that never reconverges onto the kubevirt VMI, went straight from the small 16.6.0 base to the kubevirt target without the intermediate hop that lands the conversion code, and left the device on the framework's 4 vCPU / 8 GiB defaults. It would have failed long before reaching a volume. Move it alongside TestKvmToKResize so it runs the sequence that is known to work and shares its assertions and diagnostics, and swap in volverify plus a blank data volume for the plain container app and its string marker. After the conversion the volume is no longer mounted at its MountDir, so the verify locates it among the guest's block devices and mounts it read-only without a journal replay, which would otherwise heal the torn state being measured. A volume that comes back empty is reported as such rather than as a lookup failure — that is what a run sees when the post-resize manifest check removed a torn volume and EVE recreated it — and the manifest state is captured alongside it. The default volume size drops to 256 MiB, below the size at which the app redeploy wedges in a Longhorn CSI create race that would mask the result. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: eriknordmark --- .../tests/appvolshrink/appvolshrink_test.go | 309 ----------- evetest/tests/resize/appvolshrink_test.go | 493 ++++++++++++++++++ 2 files changed, 493 insertions(+), 309 deletions(-) delete mode 100644 evetest/tests/appvolshrink/appvolshrink_test.go create mode 100644 evetest/tests/resize/appvolshrink_test.go diff --git a/evetest/tests/appvolshrink/appvolshrink_test.go b/evetest/tests/appvolshrink/appvolshrink_test.go deleted file mode 100644 index 13fd242cdc5..00000000000 --- a/evetest/tests/appvolshrink/appvolshrink_test.go +++ /dev/null @@ -1,309 +0,0 @@ -// Copyright (c) 2026 Zededa, Inc. -// SPDX-License-Identifier: Apache-2.0 - -package appvolshrink_test - -import ( - "fmt" - "strings" - "testing" - "time" - - // revive:disable:dot-imports - . "github.com/onsi/gomega" - - eveconfig "github.com/lf-edge/eve-api/go/config" - "github.com/lf-edge/eve-api/go/evecommon" - eveinfo "github.com/lf-edge/eve-api/go/info" - "github.com/lf-edge/eve/evetest" - "github.com/lf-edge/eve/evetest/constants" - "github.com/lf-edge/eve/evetest/matchers" - "github.com/lf-edge/eve/evetest/netmodels" - "github.com/lf-edge/eve/pkg/pillar/types" -) - -const ( - initialEVEVersionParamKey = "INITIAL_EVE_VERSION" - initialHypervisorParamKey = "INITIAL_HYPERVISOR" - seedParamKey = "VOLVERIFY_SEED" - opsParamKey = "VOLVERIFY_OPS" - volSizeMiBParamKey = "DATA_VOLUME_MB" - volverifyImageParamKey = "VOLVERIFY_IMAGE" - - // defaultVolverifyImage is the canonical published testapp image; override - // VOLVERIFY_IMAGE to run against a personal registry copy. - defaultVolverifyImage = "lfedge/evetest-volverify" - - appSSHUser = "root" - appSSHPassword = "testpassword" - appSSHFwdPort = 2222 - dataMountDir = "/mnt/data" - blockSize = 4096 -) - -// TestAppVolumeShrinkCorruption checks whether a watchdog-interrupted EVE-kvm→EVE-k -// offline filesystem shrink corrupts an application data volume, and if so whether -// the corruption is detectable. -// -// It deploys the volverify app (github.com/lf-edge/eve/evetest/testapps/volverify) -// with a large BLANK data volume, fills it with a deterministic self-verifying -// pattern before the upgrade, drives the kvm→kubevirt upgrade (which repartitions -// and shrinks /persist, relocating the volume's blocks), and then re-verifies the -// pattern. The watchdog fault is baked into the target EVE image (the fork#7 no-pet -// stress build fires the HW watchdog inside the offline resizer), so this test just -// upgrades TO that build; the soak loops the test externally (design §5, §7). -// -// The target build must also relax the shrink+volumes gate (§3.1) — the shippable -// EVE refuses a cross-flavor shrink while a volume is present. -// -// Parameters: -// - EVE_VERSION / HYPERVISOR: target build + hypervisor (default hypervisor kubevirt here). -// - INITIAL_EVE_VERSION (required, e.g. "16.6.0") / INITIAL_HYPERVISOR (default kvm). -// - DISK_SIZE_MB: device disk size (default 131072 = 128 GiB). -// - DATA_VOLUME_MB: blank data-volume size (default 40960 = 40 GiB). -// - VOLVERIFY_SEED / VOLVERIFY_OPS: pattern seed and op count. -func TestAppVolumeShrinkCorruption(test *testing.T) { - evetestT := evetest.Init(test) - t := NewGomegaWithT(evetestT) - defer evetest.Close() - - evetest.DefineTestParameters( - evetest.EVEVersionParameter(), - evetest.HypervisorParameter(), - evetest.TPMParameter(), - evetest.DiskSizeMiBParameter(), - evetest.TestParameterDefinition{ - Key: initialEVEVersionParamKey, - DefaultValue: "16.6.0", - Description: evetest.TestParameterDescription{ - Summary: "EVE-kvm version to start on (before the kvm→k conversion)", - Default: "16.6.0", - }, - }, - evetest.TestParameterDefinition{ - Key: initialHypervisorParamKey, - DefaultValue: evetest.HypervisorKVM, - Description: evetest.TestParameterDescription{ - Summary: "Hypervisor of the initial (pre-upgrade) EVE version", - Default: "kvm", - AllowedValues: "kvm|kubevirt", - }, - }, - evetest.TestParameterDefinition{ - Key: volSizeMiBParamKey, - DefaultValue: uint32(40960), - Description: evetest.TestParameterDescription{ - Summary: "Blank data-volume size in MiB (should exceed the shrink evacuation zone)", - Default: "40960", - }, - }, - evetest.TestParameterDefinition{ - Key: seedParamKey, - DefaultValue: uint64(20260723), - Description: evetest.TestParameterDescription{ - Summary: "volverify master seed for the fill/delete pattern", - Default: "20260723", - }, - }, - evetest.TestParameterDefinition{ - Key: opsParamKey, - DefaultValue: uint64(200000), - Description: evetest.TestParameterDescription{ - Summary: "volverify op count (creates/deletes) to fill the volume", - Default: "200000", - }, - }, - evetest.TestParameterDefinition{ - Key: volverifyImageParamKey, - DefaultValue: defaultVolverifyImage, - Description: evetest.TestParameterDescription{ - Summary: "Docker Hub repo of the volverify test app (override for a personal registry copy)", - Default: defaultVolverifyImage, - }, - }, - ) - - withTPM := evetest.GetTPMParameterValue() - diskSizeMiB := evetest.GetDiskSizeMiBParameterValue() - if diskSizeMiB == 0 { - diskSizeMiB = 131072 // 128 GiB — enough headroom for a real shrink - } - initialVersion := evetest.GetTestParameter[string](initialEVEVersionParamKey) - if initialVersion == "" { - evetestT.Fatalf("%s%s is required", constants.EnvPrefix, initialEVEVersionParamKey) - } - initialHypervisor := evetest.GetTestParameter[evetest.Hypervisor](initialHypervisorParamKey) - targetVersion := evetest.GetEVEVersionParameterValue() - targetHypervisor := evetest.GetHypervisorParameterValue() - volSizeMiB := evetest.GetTestParameter[uint32](volSizeMiBParamKey) - seed := evetest.GetTestParameter[uint64](seedParamKey) - ops := evetest.GetTestParameter[uint64](opsParamKey) - volverifyImage := evetest.GetTestParameter[string](volverifyImageParamKey) - - const devName = "edge-dev" - evetest.Setup( - evetest.RequireEdgeDevice{ - Name: devName, - WithEVEVersion: initialVersion, - WithHypervisor: initialHypervisor, - WithTPM: withTPM, - MinDiskSizeInMiB: diskSizeMiB, - DeviceReusePolicy: evetest.CreateFromScratchWithInstaller, - }, - evetest.RequireNetworkModel{NetworkModel: netmodels.SingleEthWithDHCP}, - ) - device := evetest.GetEdgeDevice(devName) - log := evetest.Logger() - - devConfig := evetest.NewEdgeDeviceConfig(devName) - networkUUID := devConfig.AddNetwork(evetest.DHCPNetworkConfig{ - NetworkType: evecommon.NetworkType_V4, - }) - devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ - LogicalLabel: "eth0", - PhysicalLabel: "eth0", - InterfaceName: "eth0", - NetworkUUID: networkUUID, - Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, - }) - - const nodeReadyCond = eveinfo.KubeNodeConditionType_KUBE_NODE_CONDITION_TYPE_READY - isK3sReady := func(info *eveinfo.ZInfoKubeCluster) bool { - if info == nil || len(info.Nodes) != 1 { - return false - } - if info.Storage.Health != eveinfo.ServiceStatus_SERVICE_STATUS_HEALTHY { - return false - } - for _, cond := range info.Nodes[0].GetConditions() { - if cond.GetType() == nodeReadyCond { - return cond.GetSet() - } - } - return false - } - - if initialHypervisor == evetest.HypervisorKubevirt { - clusterUpdates, stopClusterWatch := device.WatchClusterInfo() - defer stopClusterWatch() - device.ApplyConfig(devConfig, true, true) - t.Eventually(clusterUpdates, 20*time.Minute).Should(Receive( - matchers.SatisfyPredicate("K3s node is ready", isK3sReady))) - evetest.Checkpoint("k3s-ready") - } - - niUUID := devConfig.AddNetworkInstance(evetest.LocalNetworkInstanceConfig{ - DisplayName: "local-ni", - Port: "eth0", - Subnet: evetest.IPSubnet("10.11.12.0/24"), - DHCPRange: types.IPRange{ - Start: evetest.IPAddress("10.11.12.2"), - End: evetest.IPAddress("10.11.12.254"), - }, - Gateway: evetest.IPAddress("10.11.12.1"), - MTU: 1500, - }) - appUUID := devConfig.AddApplication(evetest.ApplicationInstanceConfig{ - DisplayName: "volverify-app", - Activate: true, - Image: evetest.DockerContainer{ - ImageName: volverifyImage, - Tag: "1.0", - }, - VirtualizationMode: eveconfig.VmMode_HVM, - CPUs: 1, - MemoryBytes: 512 * evetest.MiB, - DataVolumes: []evetest.DataVolumeConfig{ - {SizeBytes: uint64(volSizeMiB) * evetest.MiB, MountDir: dataMountDir}, - }, - NetworkAdapters: []evetest.AppNetworkAdapter{ - evetest.VirtualNetworkAdapter{ - LogicalLabel: "vif0", - NetworkInstanceUUID: niUUID, - PortFwdRules: []evetest.PortFwdRule{ - {Protocol: evetest.NetworkProtocolTCP, EdgeNodePort: appSSHFwdPort, AppPort: 22}, - }, - ACLAllowRules: []evetest.ACLAllowRule{ - {Protocol: evetest.NetworkProtocolAny, RemoteSubnet: evetest.IPSubnet("0.0.0.0/0")}, - }, - }, - }, - }) - device.ApplyConfig(devConfig, false, false) - device.WaitUntilAppIsRunning(appUUID, 5*time.Minute) - - appAuth := evetest.UsernamePasswordAuth{Username: appSSHUser, Password: appSSHPassword} - - // Fill the volume with the deterministic pattern before the shrink. The write - // of a multi-GiB volume can take a while, so allow a generous timeout. It stops - // at Ops or when the volume fills (whichever first) and reports the committed - // high-water mark, which the post-shrink verify must expect. - writeCmd := fmt.Sprintf("volverify write --dir %s --seed %d --ops %d --block-size %d", - dataMountDir, seed, ops, blockSize) - log.Infof("Filling data volume: %s", writeCmd) - writtenCommitted := -1 - t.Eventually(func(t Gomega) { - stdout, stderr, err := device.RunShellScriptInsideApp(appUUID, appAuth, writeCmd, 60*time.Minute, 0) - t.Expect(err).NotTo(HaveOccurred(), "write stderr: %s", stderr) - writtenCommitted = reportField(stdout, "committed") - t.Expect(writtenCommitted).To(BeNumerically(">=", 0), - "write did not report a committed index:\n%s", stdout) - }, 65*time.Minute, 10*time.Second).Should(Succeed()) - log.Infof("volume filled through committed op %d", writtenCommitted) - evetest.Checkpoint("volume-filled") - - // Drive the watchdog-interrupted kvm→k conversion (shrink relocates the volume). - device.UpgradeEVE(targetVersion, targetHypervisor, true, false) - evetest.Checkpoint("upgrade-complete") - - if targetHypervisor == evetest.HypervisorKubevirt { - if !isK3sReady(device.GetClusterInfo()) { - clusterUpdates, stopClusterWatch := device.WatchClusterInfo() - defer stopClusterWatch() - t.Eventually(clusterUpdates, 20*time.Minute).Should(Receive( - matchers.SatisfyPredicate("K3s node is ready", isK3sReady))) - } - evetest.Checkpoint("k3s-ready-post-upgrade") - } - device.WaitUntilAppIsRunning(appUUID, 5*time.Minute) - - // Re-verify against the write's high-water mark. The on-volume commit lives on - // the shrunk filesystem and may be cleared by fsck along with data, so pass the - // committed index the pre-shrink write reported as an off-volume floor (§4.2). - verifyCmd := fmt.Sprintf("volverify verify --dir %s --seed %d --ops %d --block-size %d --expect-committed %d", - dataMountDir, seed, ops, blockSize, writtenCommitted) - log.Infof("Verifying data volume: %s", verifyCmd) - var report string - t.Eventually(func(t Gomega) { - stdout, _, _ := device.RunShellScriptInsideApp(appUUID, appAuth, verifyCmd, 30*time.Minute, 0) - t.Expect(stdout).NotTo(BeEmpty()) - report = strings.TrimSpace(stdout) - }, 32*time.Minute, 10*time.Second).Should(Succeed()) - log.Infof("volverify report:\n%s", report) - - // present-corrupt is the dangerous silent case (§2.4): a torn-but-present - // volume EVE would serve to the app as-is. orphaned/lost are the recoverable - // modes (recreate path). Fail on present-corrupt; surface the rest for the - // soak to tally. Phase 3's detect→recreate should drive present-corrupt to 0. - presentCorrupt := reportField(report, "present-corrupt") - t.Expect(presentCorrupt).To(BeNumerically(">=", 0), - "could not parse present-corrupt count from report:\n%s", report) - t.Expect(presentCorrupt).To(Equal(0), - "data volume served corrupt-but-present after the interrupted shrink:\n%s", report) - evetest.Checkpoint("volume-verified") -} - -// reportField extracts an integer "key=N" field from a volverify summary line. -func reportField(report, key string) int { - for _, tok := range strings.Fields(report) { - if strings.HasPrefix(tok, key+"=") { - var n int - _, err := fmt.Sscanf(strings.TrimPrefix(tok, key+"="), "%d", &n) - if err != nil { - return -1 - } - return n - } - } - return -1 -} diff --git a/evetest/tests/resize/appvolshrink_test.go b/evetest/tests/resize/appvolshrink_test.go new file mode 100644 index 00000000000..003d3fa1d92 --- /dev/null +++ b/evetest/tests/resize/appvolshrink_test.go @@ -0,0 +1,493 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package resize_test + +import ( + "fmt" + "strings" + "testing" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + uuid "github.com/satori/go.uuid" + + eveconfig "github.com/lf-edge/eve-api/go/config" + "github.com/lf-edge/eve-api/go/evecommon" + "github.com/lf-edge/eve/evetest" + "github.com/lf-edge/eve/evetest/constants" + "github.com/lf-edge/eve/evetest/matchers" + "github.com/lf-edge/eve/evetest/netmodels" +) + +const ( + seedParamKey = "VOLVERIFY_SEED" + opsParamKey = "VOLVERIFY_OPS" + volverifyImageParamKey = "VOLVERIFY_IMAGE" + + // defaultVolverifyImage is where the volverify testapp is currently published. + // The canonical lfedge/evetest-volverify repository does not exist yet; point + // VOLVERIFY_IMAGE at it once it does. + defaultVolverifyImage = "eriknordmark/evetest-volverify" + + // volverifyBlockSize is the on-disk block size of the verification pattern; it + // must be identical for the write and the verify, which replay the same stream. + volverifyBlockSize = 4096 + + // volverifyCommitDir is volverify's on-volume committed-index directory. Its + // presence identifies the data volume among the guest's block devices after + // the conversion, when the volume is no longer mounted at its MountDir. + volverifyCommitDir = ".vv-commit" + + // stageCDataVolMiB is the data-volume size this test defaults to. The app + // redeploy on EVE-k wedges in a Longhorn CSI CreateVolume race above ~256 MiB + // (sweep: 100/256 MiB pass, >=512 MiB wedge), which would mask the corruption + // result behind an unrelated failure, so stay under that ceiling by default. + stageCDataVolMiB = 256 +) + +// TestAppVolumeShrinkCorruption checks whether the EVE-kvm→EVE-k offline boot-disk +// shrink corrupts an application data volume when it is interrupted part-way, and +// whether the corruption is detectable. +// +// It follows the same conversion sequence as TestKvmToKResize — that ordering and +// those readiness gates are load-bearing, see the comments there — and swaps the +// plain container app for volverify with a blank data volume. The volume is filled +// with a deterministic self-verifying pattern on EVE-kvm, the conversion relocates +// its blocks, and the pattern is re-verified on EVE-k. +// +// The fault is baked into the target EVE image rather than driven from here: the +// fork#7 stress build runs the offline resizer under a no-pet hardware watchdog +// whose timeout escalates with the retry count, so early attempts are cut mid +// shrink/grow and a later one converges. Run this against a non-stress build to +// get the same measurement with no fault injected. +// +// Parameters: as TestKvmToKResize, plus +// - DATAVOL_MB: blank data-volume size (default 256; see stageCDataVolMiB). +// - VOLVERIFY_SEED / VOLVERIFY_OPS: pattern seed and op count. The writer stops +// early when the volume fills and reports the committed high-water mark, which +// the verify is then held to. +// - VOLVERIFY_IMAGE: Docker repo of the volverify testapp. +func TestAppVolumeShrinkCorruption(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.EVEVersionParameter(), + evetest.HypervisorParameter(), + evetest.TPMParameter(), + evetest.DiskSizeMiBParameter(), + evetest.TestParameterDefinition{ + Key: ramSizeMiBParamKey, + DefaultValue: uint32(minDeviceRAMInMiB), + Description: evetest.TestParameterDescription{ + Summary: "Device RAM in MiB (EVE-k + Longhorn need >= 16 GiB)", + Default: "16384 (16 GiB)", + }, + }, + evetest.TestParameterDefinition{ + Key: cpusParamKey, + DefaultValue: uint8(minDeviceCPUs), + Description: evetest.TestParameterDescription{ + Summary: "Device vCPUs (EVE-k + Longhorn need >= 8)", + Default: "8", + }, + }, + evetest.TestParameterDefinition{ + Key: dataVolMiBParamKey, + DefaultValue: uint32(stageCDataVolMiB), + Description: evetest.TestParameterDescription{ + Summary: "App data-volume size in MiB (stay <= 256 to avoid the EVE-k CSI create race)", + Default: "256", + }, + }, + evetest.TestParameterDefinition{ + Key: seedParamKey, + DefaultValue: uint64(20260723), + Description: evetest.TestParameterDescription{ + Summary: "volverify master seed for the fill/delete pattern", + Default: "20260723", + }, + }, + evetest.TestParameterDefinition{ + Key: opsParamKey, + DefaultValue: uint64(200000), + Description: evetest.TestParameterDescription{ + Summary: "volverify op count; the writer stops early once the volume fills", + Default: "200000", + }, + }, + evetest.TestParameterDefinition{ + Key: volverifyImageParamKey, + DefaultValue: defaultVolverifyImage, + Description: evetest.TestParameterDescription{ + Summary: "Docker repo of the volverify test app", + Default: defaultVolverifyImage, + }, + }, + evetest.TestParameterDefinition{ + Key: initialEVEVersionParamKey, + DefaultValue: "16.6.0", + Description: evetest.TestParameterDescription{ + Summary: "SMALL-geometry EVE-kvm base to start on (pre-large-geometry)", + Default: "16.6.0", + }, + }, + evetest.TestParameterDefinition{ + Key: initialHypervisorParamKey, + DefaultValue: evetest.HypervisorKVM, + Description: evetest.TestParameterDescription{ + Summary: "Hypervisor of the initial (small) base", + Default: "kvm", + AllowedValues: "kvm", + }, + }, + ) + + withTPM := evetest.GetTPMParameterValue() + diskSizeMiB := evetest.GetDiskSizeMiBParameterValue() + initialVersion := evetest.GetTestParameter[string](initialEVEVersionParamKey) + if initialVersion == "" { + evetestT.Fatalf("%s%s is required", constants.EnvPrefix, initialEVEVersionParamKey) + } + initialHypervisor := evetest.GetTestParameter[evetest.Hypervisor](initialHypervisorParamKey) + convVersion := evetest.GetEVEVersionParameterValue() + targetHypervisor := evetest.GetHypervisorParameterValue() + + effectiveDiskMiB := diskSizeMiB + if effectiveDiskMiB == 0 { + effectiveDiskMiB = constants.DefaultEVEDeviceDiskSizeInMiB + } + if effectiveDiskMiB < constants.DefaultEVEDeviceDiskSizeInMiB { + evetestT.Fatalf("boot disk %d MiB is too small for the kvm→k conversion; "+ + "need at least %d MiB (64 GiB) — set DISK_SIZE_MB accordingly", + effectiveDiskMiB, constants.DefaultEVEDeviceDiskSizeInMiB) + } + effectiveRAMMiB := evetest.GetTestParameter[uint32](ramSizeMiBParamKey) + if effectiveRAMMiB == 0 { + effectiveRAMMiB = constants.DefaultEVEDeviceRAMInMiB + } + if effectiveRAMMiB < minDeviceRAMInMiB { + evetestT.Fatalf("device RAM %d MiB is too small for the kvm→k conversion "+ + "(EVE-k + Longhorn); need at least %d MiB (16 GiB) — set RAM_SIZE_MB accordingly", + effectiveRAMMiB, minDeviceRAMInMiB) + } + effectiveCPUs := evetest.GetTestParameter[uint8](cpusParamKey) + if effectiveCPUs == 0 { + effectiveCPUs = constants.DefaultEVEDeviceCPUs + } + if effectiveCPUs < minDeviceCPUs { + evetestT.Fatalf("device vCPUs %d is too few for the kvm→k conversion "+ + "(EVE-k + Longhorn); need at least %d — set CPUS accordingly", + effectiveCPUs, minDeviceCPUs) + } + dataVolMiB := evetest.GetTestParameter[uint32](dataVolMiBParamKey) + dataVolBytes := uint64(dataVolMiB) * evetest.MiB + seed := evetest.GetTestParameter[uint64](seedParamKey) + ops := evetest.GetTestParameter[uint64](opsParamKey) + volverifyImage := evetest.GetTestParameter[string](volverifyImageParamKey) + + // Cap a single file at a sixteenth of the volume. volverify's default large-file + // bound is 256 MiB, which on a volume this size would let one op consume the + // whole thing and leave the pattern with almost no files to place — and so + // almost no coverage of the relocated block range. + maxBlocks := uint64(dataVolMiB) * evetest.MiB / 16 / volverifyBlockSize + if maxBlocks < 256 { + maxBlocks = 256 + } + volverifyArgs := fmt.Sprintf("--dir %s --seed %d --ops %d --block-size %d --max-blocks %d", + dataMountDir, seed, ops, volverifyBlockSize, maxBlocks) + + const devName = "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithEVEVersion: initialVersion, + WithHypervisor: initialHypervisor, + WithTPM: withTPM, + MinDiskSizeInMiB: diskSizeMiB, + MinRAMInMiB: effectiveRAMMiB, + MinCPUs: effectiveCPUs, + // Provision from the LIVE image, not the installer: the installer ESP + // carries a 0-byte boot/.boot_repository that the offline grow's FAT32 + // copy rejects with "invalid start cluster: 0" (diskfs/go-diskfs#417). + DeviceReusePolicy: evetest.CreateFromScratchWithLiveImage, + }, + evetest.RequireNetworkModel{NetworkModel: netmodels.SingleEthWithDHCP}, + ) + device := evetest.GetEdgeDevice(devName) + log := evetest.Logger() + log.Infof("volverify app %s:1.0 on a %d MiB data volume (max file %d blocks)", + volverifyImage, dataVolMiB, maxBlocks) + + devConfig := evetest.NewEdgeDeviceConfig(devName) + networkUUID := devConfig.AddNetwork(evetest.DHCPNetworkConfig{NetworkType: evecommon.NetworkType_V4}) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "eth0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: networkUUID, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + device.ApplyConfig(devConfig, false, false) + + log.Infof("baseline: asserting SMALL boot-disk geometry") + assertSmallGeometry(t, device) + evetest.Checkpoint("baseline-small") + + log.Infof("kvm→kvm hop: upgrading to the conversion-capable build %s (kvm)", convVersion) + device.UpgradeEVE(convVersion, evetest.HypervisorKVM, true, false) + assertSmallGeometry(t, device) + assertCheckDecision(t, device, "shrink") + evetest.Checkpoint("kvm-hop-done") + + log.Infof("settling vault to a local TPM unlock") + settleVaultLocal(t, device) + evetest.Checkpoint("vault-settled") + + // Switch (L2-bridged) NI: the carried app gets its own DHCP address on the eth0 + // segment and is reached directly. A local NI does not reconverge onto the + // kubevirt VMI after the conversion (verified stuck, not slow). + niUUID := devConfig.AddNetworkInstance(evetest.SwitchNetworkInstanceConfig{ + DisplayName: "switch-ni", + Port: "eth0", + }) + appUUID := devConfig.AddApplication(evetest.ApplicationInstanceConfig{ + DisplayName: "volverify-app", + Activate: true, + Image: evetest.DockerContainer{ImageName: volverifyImage, Tag: "1.0"}, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 512 * evetest.MiB, + NetworkAdapters: []evetest.AppNetworkAdapter{ + evetest.VirtualNetworkAdapter{ + LogicalLabel: "vif0", + NetworkInstanceUUID: niUUID, + PortFwdRules: []evetest.PortFwdRule{ + {Protocol: evetest.NetworkProtocolTCP, EdgeNodePort: appSSHFwdPort, AppPort: 22}, + }, + ACLAllowRules: []evetest.ACLAllowRule{ + {Protocol: evetest.NetworkProtocolAny, RemoteSubnet: evetest.IPSubnet("0.0.0.0/0")}, + }, + }, + }, + DataVolumes: []evetest.DataVolumeConfig{ + {SizeBytes: dataVolBytes, MountDir: dataMountDir}, + }, + }) + device.ApplyConfig(devConfig, false, false) + device.WaitUntilAppIsRunning(appUUID, 5*time.Minute) + appAuth := evetest.UsernamePasswordAuth{Username: appSSHUser, Password: appSSHPassword} + assertAppSSH(t, device, appUUID, appAuth) + captureAppNet(device, "pre-conversion EVE-kvm (SSH OK)") + + // Fill the volume on EVE-kvm, where the runx shim has formatted and mounted it + // at MountDir. The writer stops at Ops or when the volume fills, whichever comes + // first, and reports the committed high-water mark the verify is held to. + log.Infof("pre-conversion: filling the data volume with the volverify pattern") + writtenCommitted := writeVolverifyPattern(t, device, appUUID, appAuth, volverifyArgs) + log.Infof("volume filled through committed op %d", writtenCommitted) + capturePersist(device, "pre-conversion EVE-kvm (volume filled)") + evetest.Checkpoint("volume-filled") + + log.Infof("kvm→k conversion: upgrading to %s (%s) — triggers the offline shrink+grow", convVersion, targetHypervisor) + conversionOK := false + defer func() { + if !conversionOK { + dumpConversionFailure(device) + } + }() + device.UpgradeEVE(convVersion, targetHypervisor, true, false) + conversionOK = true + device.ExpectAdditionalReboots(1) + evetest.Checkpoint("conversion-complete") + + assertGrownShrink(t, device) + evetest.Checkpoint("geometry-grown") + + stopPersistSampler := startPersistSampler(device, 45*time.Second) + defer stopPersistSampler() + + if targetHypervisor == evetest.HypervisorKubevirt { + log.Infof("(a) waiting for k3s node ready") + if !isK3sReady(device.GetClusterInfo()) { + clusterUpdates, stop := device.WatchClusterInfo() + defer stop() + t.Eventually(clusterUpdates, 20*time.Minute).Should(Receive( + matchers.SatisfyPredicate("K3s node is ready", isK3sReady))) + } + log.Infof("(b) waiting for volumemgr Initialized") + waitVolumemgrReady(t, device) + log.Infof("(c) waiting for longhorn StorageClass ready") + waitLonghornSC(t, device) + } + log.Infof("(d) waiting for the app to reach RUNNING on the target") + device.WaitUntilAppIsRunning(appUUID, 65*time.Minute) + appSSHOK := false + defer func() { + if !appSSHOK { + captureAppNet(device, "post-conversion EVE-k (app net FAILED)") + captureAppNetFailure(device, appUUID) + } + }() + log.Infof("(e0) post-conversion: waiting up to 10m for the app to report a routable IPv4") + waitAppHasRoutableIPv4(t, device, appUUID, 10*time.Minute) + log.Infof("(e) post-conversion: app must be SSH-reachable") + assertAppSSH(t, device, appUUID, appAuth) + appSSHOK = true + + log.Infof("(f) post-conversion: re-verifying the data-volume pattern") + captureVolManifest(device) + state := mountDataVolumeRO(t, device, appUUID, appAuth, dataMountDir) + if state == volumeStateBlank { + // The pattern is gone but the volume is intact and empty: either the + // post-resize manifest check found the volume torn and removed it, so EVE + // recreated it blank, or the shrink destroyed the filesystem outright. The + // manifest capture above says which. Either way nothing corrupt is being + // served to the app, so this is not the failure this test gates on. + log.Errorf("data volume came back BLANK — the pattern did not survive; " + + "check the volume-manifest capture for whether the detector removed it") + evetest.Checkpoint("volume-recreated-blank") + return + } + + report := verifyVolverifyPattern(t, device, appUUID, appAuth, volverifyArgs, writtenCommitted) + log.Infof("volverify report:\n%s", report) + + // present-corrupt is the silent case: a torn-but-present volume that EVE would + // serve to the app as-is, invisible to fsck and to qemu-img. orphaned and lost + // files are the recoverable modes — EVE recreates a missing volume file blank — + // so surface them but do not fail on them. + presentCorrupt := reportField(report, "present-corrupt") + t.Expect(presentCorrupt).To(BeNumerically(">=", 0), + "could not parse present-corrupt from the volverify report:\n%s", report) + t.Expect(presentCorrupt).To(Equal(0), + "data volume served corrupt-but-present after the interrupted shrink:\n%s", report) + evetest.Checkpoint("volume-verified") +} + +// writeVolverifyPattern fills the app's data volume with the deterministic pattern +// and returns the committed op index the writer reached. It requires the volume to +// be mounted at its MountDir, which the runx shim does on EVE-kvm. +func writeVolverifyPattern(t Gomega, device *evetest.EdgeDevice, appUUID uuid.UUID, + auth evetest.AuthMethod, args string) int { + script := fmt.Sprintf( + "grep -q ' %s ' /proc/mounts || { echo NOT-MOUNTED; cat /proc/mounts; exit 1; }; "+ + "volverify write %s && sync", dataMountDir, args) + committed := -1 + t.Eventually(func(t Gomega) { + stdout, stderr, err := device.RunShellScriptInsideApp(appUUID, auth, script, 60*time.Minute, 0) + t.Expect(err).NotTo(HaveOccurred(), "volverify write failed:\n%s%s", stdout, stderr) + committed = reportField(stdout, "committed") + t.Expect(committed).To(BeNumerically(">=", 0), + "volverify write reported no committed index:\n%s", stdout) + }, 65*time.Minute, 10*time.Second).Should(Succeed()) + return committed +} + +// Outcomes of locating the app's data volume after the conversion. +const ( + volumeStatePattern = "PATTERN" // mounted and still carrying the volverify pattern + volumeStateBlank = "BLANK" // the volume exists but the pattern is gone +) + +// mountDataVolumeRO finds the app's data volume among the guest's block devices +// and mounts it at mountDir, returning which of the states above it is in. EVE-k +// does not auto-mount a container app's data volume at its MountDir +// (lf-edge/eve#6145), so after the conversion the verify has to do it; volverify's +// committed-index directory is what identifies the volume. +// +// The mount is read-only and skips the ext4 journal: the verify only reads, and +// replaying the journal would heal exactly the torn state this test is measuring. +// +// A volume that holds no pattern is reported as BLANK rather than as a failure — +// that is what the caller sees when the post-resize manifest check removed a torn +// volume and EVE recreated it empty. Only finding no data volume at all is an +// error, so the block-device inventory is dumped either way. +func mountDataVolumeRO(t Gomega, device *evetest.EdgeDevice, appUUID uuid.UUID, + auth evetest.AuthMethod, mountDir string) string { + script := fmt.Sprintf(`set -u +mkdir -p %[1]s +[ -d %[1]s/%[2]s ] && { echo STATE=%[3]s already-mounted; exit 0; } +mountpoint -q %[1]s && umount %[1]s +found= +for d in /dev/vd[b-z] /dev/sd[b-z]; do + [ -b "$d" ] || continue + found="$found $d" + mount -o ro,noload -t ext4 "$d" %[1]s 2>/dev/null || continue + if [ -d %[1]s/%[2]s ]; then echo "STATE=%[3]s $d"; exit 0; fi + umount %[1]s +done +echo "candidates:$found" +lsblk 2>/dev/null; blkid 2>/dev/null; cat /proc/mounts +[ -n "$found" ] && { echo STATE=%[4]s; exit 0; } +echo STATE=NO-DATA-DEVICE +exit 1`, mountDir, volverifyCommitDir, volumeStatePattern, volumeStateBlank) + state := "" + t.Eventually(func(t Gomega) { + stdout, stderr, err := device.RunShellScriptInsideApp(appUUID, auth, script, 2*time.Minute, 0) + t.Expect(err).NotTo(HaveOccurred(), + "no data volume among the guest block devices after the conversion:\n%s%s", stdout, stderr) + for _, tok := range strings.Fields(stdout) { + if s, ok := strings.CutPrefix(tok, "STATE="); ok { + state = s + } + } + t.Expect(state).To(Or(Equal(volumeStatePattern), Equal(volumeStateBlank)), + "could not classify the data volume:\n%s", stdout) + evetest.Logger().Infof("data volume state %s:\n%s", state, strings.TrimSpace(stdout)) + }, 5*time.Minute, 15*time.Second).Should(Succeed()) + return state +} + +// captureVolManifest reports whether the post-resize volume-manifest check ran and +// what it concluded — whether it found the pre-shrink hashes, judged any volume +// torn, and removed it. This is what separates "the shrink left the volume alone" +// from "the detector cleaned up after it". Best-effort; never fails the test. +func captureVolManifest(device *evetest.EdgeDevice) { + log := evetest.Logger() + log.Errorf("=== volume-manifest (post-resize detect+recreate) ===") + probes := []struct{ what, script string }{ + {"manifest files on /persist", `eve exec pillar sh -c 'ls -l /persist/vault/volumes/.sha256 /persist/clear/volumes/.sha256 2>&1'`}, + {"volmanifest / recreate signatures (newlog)", `eve exec pillar sh -c 'grep -ahiE "volmanifest|recreateCorruptVolumes|verifyVolumes|torn by the resize" /persist/newlog/collect/*.log 2>/dev/null | tail -40' || echo none`}, + {"app volume files", `eve exec pillar sh -c 'ls -l /persist/vault/volumes /persist/clear/volumes 2>&1'`}, + } + for _, p := range probes { + out, errOut, err := device.RunShellScript(p.script, 60*time.Second, 0) + log.Errorf("[volmanifest:%s]\n%s%s(err=%v)", p.what, strings.TrimSpace(out), errOut, err) + } +} + +// verifyVolverifyPattern replays the pattern against the volume and returns the +// report. volverify exits non-zero on a dirty report, so the exit status is not an +// assertion failure here — the caller classifies the report instead. +func verifyVolverifyPattern(t Gomega, device *evetest.EdgeDevice, appUUID uuid.UUID, + auth evetest.AuthMethod, args string, expectCommitted int) string { + cmd := fmt.Sprintf("volverify verify %s --expect-committed %d", args, expectCommitted) + var report string + t.Eventually(func(t Gomega) { + stdout, _, _ := device.RunShellScriptInsideApp(appUUID, auth, cmd, 30*time.Minute, 0) + t.Expect(stdout).NotTo(BeEmpty(), "volverify verify produced no output") + report = strings.TrimSpace(stdout) + }, 32*time.Minute, 10*time.Second).Should(Succeed()) + return report +} + +// reportField extracts an integer "key=N" field from a volverify summary line, +// returning -1 when the field is absent or unparsable. +func reportField(report, key string) int { + for _, tok := range strings.Fields(report) { + if !strings.HasPrefix(tok, key+"=") { + continue + } + var n int + if _, err := fmt.Sscanf(strings.TrimPrefix(tok, key+"="), "%d", &n); err != nil { + return -1 + } + return n + } + return -1 +} From c84b96a098a4c9c9bbee7077d33fca3944c9cf1b Mon Sep 17 00:00:00 2001 From: eriknordmark Date: Mon, 27 Jul 2026 22:07:44 -0700 Subject: [PATCH 07/29] evetest: capture whether the resize was interrupted A fault-injected conversion that converges is indistinguishable from a clean one at the harness level: the device reboots, comes back on the target, and the geometry asserts pass either way. The first Stage C run passed with a spotless volume and left no evidence of whether the stress watchdog had cut the offline resize at all, so the result could not be read as either "the shrink preserves the volume" or "nothing was injected". Capture the resize attempt counter after the conversion, which storage-resize.sh advances once per resize boot and which therefore distinguishes a single clean pass from a re-driven one, together with the watchdog device node and the recorded reboot reasons. The resizer's watchdog exits quietly on a guest that has no watchdog device, which would silently reduce the whole test to a no-fault run. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: eriknordmark --- evetest/tests/resize/appvolshrink_test.go | 28 +++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/evetest/tests/resize/appvolshrink_test.go b/evetest/tests/resize/appvolshrink_test.go index 003d3fa1d92..7c6d51c0bbc 100644 --- a/evetest/tests/resize/appvolshrink_test.go +++ b/evetest/tests/resize/appvolshrink_test.go @@ -303,6 +303,7 @@ func TestAppVolumeShrinkCorruption(test *testing.T) { device.UpgradeEVE(convVersion, targetHypervisor, true, false) conversionOK = true device.ExpectAdditionalReboots(1) + captureResizeEvidence(device) evetest.Checkpoint("conversion-complete") assertGrownShrink(t, device) @@ -443,6 +444,33 @@ exit 1`, mountDir, volverifyCommitDir, volumeStatePattern, volumeStateBlank) return state } +// captureResizeEvidence records whether the offline resize was actually +// interrupted, which is the whole premise of this test and is otherwise +// invisible: a clean conversion and a fault-injected one that happened to +// converge look identical from the harness side. +// +// The decisive artifact is the resize attempt counter on the CONFIG partition — +// storage-resize.sh increments it once per resize boot, so a value above 1 means +// the resize was re-driven, i.e. something cut it. The stress watchdog can only +// cut anything if the guest has a watchdog device at all (the resizer's +// run-watchdog exits quietly when /dev/watchdog is absent), so the device node +// and the recorded reboot reasons are captured next to it. +func captureResizeEvidence(device *evetest.EdgeDevice) { + log := evetest.Logger() + log.Errorf("=== offline-resize fault evidence ===") + probes := []struct{ what, script string }{ + {"resize attempt counter", `cat /config/resize-reboots 2>/dev/null || echo ABSENT`}, + {"resize-failed.json", `cat /config/resize-failed.json 2>/dev/null || echo NONE`}, + {"watchdog device present?", `ls -l /dev/watchdog* 2>&1; eve exec pillar wdctl /dev/watchdog 2>&1 | head -20`}, + {"reboot reasons", `cat /persist/reboot-reason.log 2>/dev/null || eve exec pillar sh -c 'cat /persist/log/reboot-reason.log 2>/dev/null' || echo NONE`}, + {"resizer/watchdog log lines", `eve exec pillar sh -c 'grep -ahiE "run-watchdog|storage-resizer|resize did not converge|watchdog" /persist/newlog/collect/*.log 2>/dev/null | tail -30' || echo none`}, + } + for _, p := range probes { + out, errOut, err := device.RunShellScript(p.script, 60*time.Second, 0) + log.Errorf("[resize:%s]\n%s%s(err=%v)", p.what, strings.TrimSpace(out), errOut, err) + } +} + // captureVolManifest reports whether the post-resize volume-manifest check ran and // what it concluded — whether it found the pre-shrink hashes, judged any volume // torn, and removed it. This is what separates "the shrink left the volume alone" From 892c6adef03f08fd5ecfe834c4991c5db456fe23 Mon Sep 17 00:00:00 2001 From: eriknordmark Date: Mon, 27 Jul 2026 22:27:47 -0700 Subject: [PATCH 08/29] evetest: let the chipset watchdog reset the guest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A device under evetest survives a hardware-watchdog timeout that would reset real hardware. The q35 machine brings the ICH9 LPC bridge, and with it the iTCO watchdog EVE drives through /dev/watchdog, but the bridge only latches a status bit when the timer expires unless its no-reboot property is cleared and an expiry action is set. So the guest runs on as if nothing had happened. That hides any regression in EVE's own watchdog handling, and it silently disarms tests that deliberately fire the watchdog — a fault-injection run then looks indistinguishable from a clean one, which is how a spotless result can be mistaken for evidence that the code under test is sound. Configure the same two QEMU settings the eden-based harness already uses, so the guest behaves like the hardware it stands in for. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: eriknordmark --- evetest/broker/provider/qemu.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/evetest/broker/provider/qemu.go b/evetest/broker/provider/qemu.go index 5325c7e5af9..8faada23c72 100644 --- a/evetest/broker/provider/qemu.go +++ b/evetest/broker/provider/qemu.go @@ -1482,6 +1482,15 @@ func (dev *qemuDevice) buildArgs() []string { args := []string{ "-enable-kvm", "-machine", "q35", + // Let the chipset watchdog actually reset the guest. q35 brings the ICH9 + // LPC bridge and with it the iTCO watchdog EVE drives through + // /dev/watchdog, but the bridge only latches a status bit on timeout + // unless noreboot is cleared. Without these two a guest silently survives + // a watchdog it should have died on, which hides watchdog regressions and + // quietly disarms any test that injects one. eden configures its QEMU the + // same way. + "-global", "ICH9-LPC.noreboot=false", + "-watchdog-action", "reset", "-cpu", "host", "-smp", fmt.Sprintf("%d", dev.spec.CPUs), "-m", fmt.Sprintf("%d", dev.spec.MemoryBytes>>20), From 7c5a638f5682b7c1447add3d22bf4eab1e40f847 Mon Sep 17 00:00:00 2001 From: eriknordmark Date: Mon, 27 Jul 2026 22:48:47 -0700 Subject: [PATCH 09/29] evetest: let a test widen the upgrade budget The harness allows a fixed 20 minutes for a device to come back running an upgraded version. That suits an ordinary base-OS upgrade, but a cross-flavor boot-disk conversion also repartitions the disk offline across several restarts and then brings up a whole container-cluster stack, which takes very nearly the whole budget: two conversion runs on the same image measured 19m46s and just over 20m, so the second was failed as "timed out" while it was still making progress and had in fact already booted the target image. Let a test raise the wait for its own device handle, and do so in the two conversion tests. Without this the pass/fail verdict on a conversion is decided by a couple of minutes of host load rather than by the conversion. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: eriknordmark --- evetest/edgedevice.go | 27 +++++++++++++++++++++- evetest/tests/resize/kvmtok_resize_test.go | 5 ++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/evetest/edgedevice.go b/evetest/edgedevice.go index 21c3b9a067b..0955700668d 100644 --- a/evetest/edgedevice.go +++ b/evetest/edgedevice.go @@ -38,6 +38,9 @@ type EdgeDevice struct { // Set of app UUIDs (strings) for which WaitUntilAppIsRunning is active. // Used by WatchAppInfo to suppress duplicate state logging. appsBeingWaited sync.Map + // Per-handle override of how long to wait for an upgrade; see + // SetUpgradeTimeout. Zero means use the package default. + upgradeTimeout time.Duration } // GetEdgeDevice returns a handle to an onboarded EdgeDevice identified by devName. @@ -483,7 +486,7 @@ func (d *EdgeDevice) waitForUpgrade(targetShortVersion string) { } defer unsub() - ctx, cancel := context.WithTimeout(d.th.ctx, eveUpgradeTimeout) + ctx, cancel := context.WithTimeout(d.th.ctx, d.upgradeWaitTimeout()) defer cancel() var lastLoggedState, lastLoggedStatus string @@ -638,6 +641,28 @@ func (d *EdgeDevice) ExpectAdditionalReboots(n int) { } } +// SetUpgradeTimeout overrides how long UpgradeEVE waits for the device to come +// back running the target version, for this handle only. +// +// The default is sized for an ordinary base-OS upgrade: download, one reboot, +// done. A cross-flavor boot-disk conversion is a different animal — it also runs +// an offline shrink+grow across several reboots and then brings up a whole +// container-cluster stack — and it lands close enough to the default that a +// healthy conversion and an expired budget are decided by a couple of minutes of +// host load. A test driving one should raise the timeout, otherwise it reports a +// conversion that was still progressing as a failure. +func (d *EdgeDevice) SetUpgradeTimeout(timeout time.Duration) { + d.upgradeTimeout = timeout +} + +// upgradeWaitTimeout is the effective upgrade wait for this handle. +func (d *EdgeDevice) upgradeWaitTimeout() time.Duration { + if d.upgradeTimeout > 0 { + return d.upgradeTimeout + } + return eveUpgradeTimeout +} + // rebootAndWait executes triggerFn to initiate a device reboot and, if // wait is true, blocks until the device confirms the reboot by reporting // a ZInfoDevice.lastRebootTime strictly after the moment triggerFn was called. diff --git a/evetest/tests/resize/kvmtok_resize_test.go b/evetest/tests/resize/kvmtok_resize_test.go index 5be64375a2f..1cff3e34c90 100644 --- a/evetest/tests/resize/kvmtok_resize_test.go +++ b/evetest/tests/resize/kvmtok_resize_test.go @@ -208,6 +208,11 @@ func TestKvmToKResize(test *testing.T) { }) device.ApplyConfig(devConfig, false, false) + // The conversion is several reboots plus an EVE-k bring-up and runs close to + // the framework's default upgrade budget, so give it room; without this a + // conversion that is merely slow is reported as a failed one. + device.SetUpgradeTimeout(45 * time.Minute) + // Step 1: baseline — the boot disk must be SMALL, else the conversion is a no-op. log.Infof("baseline: asserting SMALL boot-disk geometry") assertSmallGeometry(t, device) From 123c4a4b77964634ca5f441f3cf3a2426d92e227 Mon Sep 17 00:00:00 2001 From: eriknordmark Date: Mon, 27 Jul 2026 22:49:03 -0700 Subject: [PATCH 10/29] evetest: assert the guest has a watchdog device MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The app-volume shrink test interrupts the offline resize by having the resizer arm /dev/watchdog and stop feeding it. When the guest has no such device the resizer exits quietly instead, nothing is interrupted, and the conversion runs to completion — so the volume verifies perfectly and the run reads exactly like evidence that an interrupted shrink preserves application data. That is the worst possible failure mode for a fault-injection test: it produces a reassuring result rather than an error. Assert the device node up front, on the baseline, so a guest that cannot deliver the fault fails in seconds with an explanation. Also correct what the post-conversion evidence capture reads. The resize attempt counter looked like the ideal witness — it advances once per resize attempt — but it is deleted on the success path, so after a completed conversion it always reads empty no matter how many attempts it took. Capture the recorded boot reasons instead, since a watchdog reset is reported as one, and leave a note so it is not reintroduced. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: eriknordmark --- evetest/tests/resize/appvolshrink_test.go | 46 ++++++++++++++++++----- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/evetest/tests/resize/appvolshrink_test.go b/evetest/tests/resize/appvolshrink_test.go index 7c6d51c0bbc..1d27f2cfeab 100644 --- a/evetest/tests/resize/appvolshrink_test.go +++ b/evetest/tests/resize/appvolshrink_test.go @@ -234,8 +234,14 @@ func TestAppVolumeShrinkCorruption(test *testing.T) { }) device.ApplyConfig(devConfig, false, false) + // The conversion is several reboots plus an EVE-k bring-up and runs close to + // the framework's default upgrade budget, so give it room; without this a + // conversion that is merely slow is reported as a failed one. + device.SetUpgradeTimeout(45 * time.Minute) + log.Infof("baseline: asserting SMALL boot-disk geometry") assertSmallGeometry(t, device) + assertWatchdogPresent(t, device) evetest.Checkpoint("baseline-small") log.Infof("kvm→kvm hop: upgrading to the conversion-capable build %s (kvm)", convVersion) @@ -444,25 +450,47 @@ exit 1`, mountDir, volverifyCommitDir, volumeStatePattern, volumeStateBlank) return state } +// assertWatchdogPresent fails the run if the guest has no watchdog device. +// +// This is a check on the QEMU setup, not on EVE. The stress build's resizer arms +// /dev/watchdog and then deliberately stops feeding it, which is how this test +// interrupts the offline resize — but when the device node is missing the resizer +// exits quietly and the interruption never happens. The conversion then completes +// cleanly and the volume verifies perfectly, which reads exactly like evidence +// that an interrupted shrink preserves the data. Assert it up front, on the +// baseline, so a misconfigured guest fails in seconds instead of producing a +// reassuring result an hour later. +func assertWatchdogPresent(t Gomega, device *evetest.EdgeDevice) { + t.Eventually(func(g Gomega) { + out, err := runEVE(device, + `[ -c /dev/watchdog ] && echo WATCHDOG-PRESENT || echo WATCHDOG-MISSING; wdctl /dev/watchdog 2>&1 | head -8`) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(out).To(ContainSubstring("WATCHDOG-PRESENT"), + "guest has no /dev/watchdog, so the stress resizer cannot interrupt the "+ + "resize and a clean result here would mean nothing; check that QEMU "+ + "exposes a watchdog and that the chipset is allowed to reset:\n%s", out) + evetest.Logger().Infof("watchdog device present:\n%s", strings.TrimSpace(out)) + }, 2*time.Minute, 10*time.Second).Should(Succeed()) +} + // captureResizeEvidence records whether the offline resize was actually // interrupted, which is the whole premise of this test and is otherwise // invisible: a clean conversion and a fault-injected one that happened to // converge look identical from the harness side. // -// The decisive artifact is the resize attempt counter on the CONFIG partition — -// storage-resize.sh increments it once per resize boot, so a value above 1 means -// the resize was re-driven, i.e. something cut it. The stress watchdog can only -// cut anything if the guest has a watchdog device at all (the resizer's -// run-watchdog exits quietly when /dev/watchdog is absent), so the device node -// and the recorded reboot reasons are captured next to it. +// Note the resize attempt counter on the CONFIG partition is NOT usable here: +// storage-resize.sh deletes it on the success path, so by the time the conversion +// has finished it always reads empty regardless of how many attempts it took. The +// durable evidence is what EVE recorded about why it rebooted — a watchdog reset +// is reported as its own boot reason — plus whatever the resizer left in the logs. func captureResizeEvidence(device *evetest.EdgeDevice) { log := evetest.Logger() log.Errorf("=== offline-resize fault evidence ===") probes := []struct{ what, script string }{ - {"resize attempt counter", `cat /config/resize-reboots 2>/dev/null || echo ABSENT`}, + {"boot / reboot reasons", `for f in /persist/boot-reason /persist/reboot-reason /persist/status/boot-reason /persist/status/reboot-reason /persist/log/reboot-reason.log; do [ -f "$f" ] && { echo "--- $f"; cat "$f"; }; done 2>/dev/null || echo NONE`}, + {"watchdog boot reason in logs", `eve exec pillar sh -c 'grep -ahoE "BootReason[A-Za-z]+" /persist/newlog/collect/*.log 2>/dev/null | sort | uniq -c | sort -rn | head' || echo none`}, {"resize-failed.json", `cat /config/resize-failed.json 2>/dev/null || echo NONE`}, - {"watchdog device present?", `ls -l /dev/watchdog* 2>&1; eve exec pillar wdctl /dev/watchdog 2>&1 | head -20`}, - {"reboot reasons", `cat /persist/reboot-reason.log 2>/dev/null || eve exec pillar sh -c 'cat /persist/log/reboot-reason.log 2>/dev/null' || echo NONE`}, + {"watchdog device", `[ -c /dev/watchdog ] && echo PRESENT || echo MISSING; wdctl /dev/watchdog 2>&1 | head -8`}, {"resizer/watchdog log lines", `eve exec pillar sh -c 'grep -ahiE "run-watchdog|storage-resizer|resize did not converge|watchdog" /persist/newlog/collect/*.log 2>/dev/null | tail -30' || echo none`}, } for _, p := range probes { From 53414e9f1d5c90f674c13a3d679caec7d3d72624 Mon Sep 17 00:00:00 2001 From: eriknordmark Date: Mon, 27 Jul 2026 23:33:26 -0700 Subject: [PATCH 11/29] evetest: check the watchdog driver, not the node MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A /dev/watchdog character node can exist with no driver behind it, so testing for the node proves nothing about whether the guest can deliver a watchdog reset — a run observed the node present while wdctl could not read any information about it. Key the check on sysfs instead, where a driver only appears once it has registered. Do not test that the node opens, either: EVE's watchdog service holds it once the system is up, so a busy device is the healthy case here rather than a fault. Note in passing what this check cannot establish, since it runs with everything up whereas the resizer runs from an onboot container long before that. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: eriknordmark --- evetest/tests/resize/appvolshrink_test.go | 40 ++++++++++++++--------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/evetest/tests/resize/appvolshrink_test.go b/evetest/tests/resize/appvolshrink_test.go index 1d27f2cfeab..251b8a6ddbf 100644 --- a/evetest/tests/resize/appvolshrink_test.go +++ b/evetest/tests/resize/appvolshrink_test.go @@ -241,7 +241,7 @@ func TestAppVolumeShrinkCorruption(test *testing.T) { log.Infof("baseline: asserting SMALL boot-disk geometry") assertSmallGeometry(t, device) - assertWatchdogPresent(t, device) + assertWatchdogDriverBound(t, device) evetest.Checkpoint("baseline-small") log.Infof("kvm→kvm hop: upgrading to the conversion-capable build %s (kvm)", convVersion) @@ -450,26 +450,34 @@ exit 1`, mountDir, volverifyCommitDir, volumeStatePattern, volumeStateBlank) return state } -// assertWatchdogPresent fails the run if the guest has no watchdog device. +// assertWatchdogDriverBound fails the run if no watchdog driver is bound in the +// guest. It checks sysfs rather than the device node: a /dev/watchdog character +// node can exist with nothing behind it, so its presence alone proves nothing, +// whereas an entry under /sys/class/watchdog means a driver registered. It does +// not try to open the node — once EVE's watchdog service is up it holds it, and +// EBUSY here would be a false alarm. // -// This is a check on the QEMU setup, not on EVE. The stress build's resizer arms +// This checks the QEMU setup, not EVE. The stress build's resizer arms // /dev/watchdog and then deliberately stops feeding it, which is how this test -// interrupts the offline resize — but when the device node is missing the resizer -// exits quietly and the interruption never happens. The conversion then completes -// cleanly and the volume verifies perfectly, which reads exactly like evidence -// that an interrupted shrink preserves the data. Assert it up front, on the -// baseline, so a misconfigured guest fails in seconds instead of producing a -// reassuring result an hour later. -func assertWatchdogPresent(t Gomega, device *evetest.EdgeDevice) { +// interrupts the offline resize — but if the resizer cannot open the device it +// exits quietly and nothing is interrupted. The conversion then completes and the +// volume verifies perfectly, which reads exactly like evidence that an interrupted +// shrink preserves the data. +// +// Necessary but not sufficient: this runs with the system fully up, whereas the +// resizer runs from an onboot container much earlier. Only the resizer's own +// console output confirms it armed the watchdog on that path. +func assertWatchdogDriverBound(t Gomega, device *evetest.EdgeDevice) { t.Eventually(func(g Gomega) { out, err := runEVE(device, - `[ -c /dev/watchdog ] && echo WATCHDOG-PRESENT || echo WATCHDOG-MISSING; wdctl /dev/watchdog 2>&1 | head -8`) + `ls /sys/class/watchdog/ 2>/dev/null | grep -q watchdog && echo WATCHDOG-DRIVER-BOUND || echo WATCHDOG-DRIVER-MISSING; `+ + `ls /sys/class/watchdog/ 2>&1; cat /sys/class/watchdog/watchdog0/identity 2>/dev/null`) g.Expect(err).NotTo(HaveOccurred()) - g.Expect(out).To(ContainSubstring("WATCHDOG-PRESENT"), - "guest has no /dev/watchdog, so the stress resizer cannot interrupt the "+ - "resize and a clean result here would mean nothing; check that QEMU "+ - "exposes a watchdog and that the chipset is allowed to reset:\n%s", out) - evetest.Logger().Infof("watchdog device present:\n%s", strings.TrimSpace(out)) + g.Expect(out).To(ContainSubstring("WATCHDOG-DRIVER-BOUND"), + "no watchdog driver bound in the guest, so the stress resizer cannot "+ + "interrupt the resize and a clean result here would mean nothing; check "+ + "that QEMU exposes a watchdog and the chipset may reset:\n%s", out) + evetest.Logger().Infof("watchdog driver:\n%s", strings.TrimSpace(out)) }, 2*time.Minute, 10*time.Second).Should(Succeed()) } From 3c1cc787107853964fe1554820ad6d478886d4f8 Mon Sep 17 00:00:00 2001 From: eriknordmark Date: Tue, 28 Jul 2026 14:32:22 -0700 Subject: [PATCH 12/29] evetest: recover the app PVC wedge in the shrink test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Data volumes above roughly 256 MiB make the app redeploy on EVE-k wedge: a PVC stays Pending forever behind a Longhorn CSI create/verify race. It is stuck rather than slow, so the existing wait can only ever time out, which caps the test at a volume size far too small to be relocated by the shrink in any interesting quantity. Wait in rounds instead and, between them, delete the Pending PVCs so the provisioner re-drives them — the documented recovery for this wedge. Never delete the data volume's PVC or its import scratch. That volume holds the pattern the test verifies and EVE recreates a deleted volume empty, so the verify would find a blank volume and the run would report a clean pass: a false negative that looks like a result. The size comparison is therefore biased towards protecting, and when the wedged PVC is the protected one no recovery is attempted and the run fails honestly. Sizes are used because the harness does not know the volume UUIDs EVE assigns; the classification is logged so a protected-everything round is visible rather than silent. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: eriknordmark --- evetest/tests/resize/appvolshrink_test.go | 103 +++++++++++++++++++++- 1 file changed, 102 insertions(+), 1 deletion(-) diff --git a/evetest/tests/resize/appvolshrink_test.go b/evetest/tests/resize/appvolshrink_test.go index 251b8a6ddbf..3ab1507a76c 100644 --- a/evetest/tests/resize/appvolshrink_test.go +++ b/evetest/tests/resize/appvolshrink_test.go @@ -16,6 +16,7 @@ import ( eveconfig "github.com/lf-edge/eve-api/go/config" "github.com/lf-edge/eve-api/go/evecommon" + eveinfo "github.com/lf-edge/eve-api/go/info" "github.com/lf-edge/eve/evetest" "github.com/lf-edge/eve/evetest/constants" "github.com/lf-edge/eve/evetest/matchers" @@ -332,7 +333,7 @@ func TestAppVolumeShrinkCorruption(test *testing.T) { waitLonghornSC(t, device) } log.Infof("(d) waiting for the app to reach RUNNING on the target") - device.WaitUntilAppIsRunning(appUUID, 65*time.Minute) + waitAppRunningWithPVCRecovery(t, device, appUUID, dataVolMiB) appSSHOK := false defer func() { if !appSSHOK { @@ -450,6 +451,106 @@ exit 1`, mountDir, volverifyCommitDir, volumeStatePattern, volumeStateBlank) return state } +// Rounds of "wait, then try the documented PVC recovery" allowed before the app +// is declared stuck, and how long each round waits. +const ( + pvcRecoveryRounds = 3 + pvcRecoveryWait = 12 * time.Minute +) + +// waitAppRunningWithPVCRecovery waits for the app to reach RUNNING on EVE-k, +// applying the documented recovery for the known app-PVC wedge between attempts. +// +// The wedge is a Longhorn CSI create/verify race that leaves a PVC Pending +// forever: it is "stuck, not slow", so simply waiting longer never rescues it. +// Deleting the PVC lets the provisioner re-drive it cleanly, which is what makes +// data volumes above ~256 MiB usable at all — below that the race is rare, above +// it the wedge is the norm. +func waitAppRunningWithPVCRecovery(t Gomega, device *evetest.EdgeDevice, + appUUID uuid.UUID, dataVolMiB uint32) { + log := evetest.Logger() + for round := 1; round <= pvcRecoveryRounds; round++ { + if waitAppRunningQuietly(device, appUUID, pvcRecoveryWait) { + return + } + log.Errorf("app not RUNNING after %s (round %d/%d) — trying the PVC-wedge recovery", + pvcRecoveryWait, round, pvcRecoveryRounds) + if !recoverWedgedAppPVCs(device, dataVolMiB) { + log.Errorf("nothing safe to recover this round") + } + } + t.Expect(appIsRunning(device, appUUID)).To(BeTrue(), + "app never reached RUNNING after %d PVC-wedge recovery attempts", pvcRecoveryRounds) +} + +// appIsRunning reports the app's current state without asserting. +func appIsRunning(device *evetest.EdgeDevice, appUUID uuid.UUID) bool { + info := device.GetAppInfo(appUUID) + return info != nil && info.GetState() == eveinfo.ZSwState_RUNNING +} + +// waitAppRunningQuietly polls until the app is RUNNING or the timeout expires, +// returning whether it got there. Unlike the framework's waiter it does not fail +// the test on timeout, so the caller can intervene and keep waiting. +func waitAppRunningQuietly(device *evetest.EdgeDevice, appUUID uuid.UUID, timeout time.Duration) bool { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if appIsRunning(device, appUUID) { + return true + } + time.Sleep(15 * time.Second) + } + return appIsRunning(device, appUUID) +} + +// recoverWedgedAppPVCs deletes Pending PVCs in the app namespace so the +// provisioner re-drives them, and reports whether it deleted any. +// +// It must never delete the data volume's PVC or that volume's CDI scratch: the +// data volume holds the pattern this test verifies, and EVE recreates a deleted +// volume BLANK — the verify would then find an empty volume and the run would +// report a clean pass, a false negative wearing the clothes of a result. So the +// size check is deliberately biased towards protecting: anything within reach of +// the data volume's size is left alone, and if that is the wedged PVC then no +// recovery happens and the run is allowed to fail honestly. +// +// The discriminator is size because the harness does not know the volume UUIDs +// EVE assigns. That is sound while the data volume is comfortably larger than the +// app's image PVC (a few hundred MiB) — which is exactly the case where recovery +// is needed. At data-volume sizes near the image size the app's own PVC gets +// protected too and recovery no-ops; the classification is logged so that is +// visible rather than silent. +func recoverWedgedAppPVCs(device *evetest.EdgeDevice, dataVolMiB uint32) bool { + log := evetest.Logger() + script := fmt.Sprintf(`set -u +DV=%d +LIST=$(eve exec kube kubectl -n eve-kube-app get pvc \ + -o custom-columns=N:.metadata.name,P:.status.phase,R:.spec.resources.requests.storage \ + --no-headers 2>/dev/null) +[ -z "$LIST" ] && { echo NO-PVCS; exit 0; } +echo "$LIST" | sed 's/^/PVC: /' +VICTIMS=$(echo "$LIST" | awk -v dv="$DV" ' + function mib(s) { + if (s ~ /Gi$/) { sub(/Gi$/, "", s); return s * 1024 } + if (s ~ /Mi$/) { sub(/Mi$/, "", s); return s + 0 } + return 0 + } + $2 == "Pending" { + if (mib($3) >= dv - 32) { printf "PROTECTED %%s (%%s)\n", $1, $3; next } + printf "VICTIM %%s (%%s)\n", $1, $3 + }') +echo "$VICTIMS" +NAMES=$(echo "$VICTIMS" | awk '$1=="VICTIM" {print $2}') +[ -z "$NAMES" ] && { echo NOTHING-TO-RECOVER; exit 0; } +for v in $NAMES; do + echo "RECOVERING $v" + eve exec kube kubectl -n eve-kube-app delete pvc "$v" --wait=false 2>&1 | sed 's/^/ /' +done`, dataVolMiB) + out, errOut, err := device.RunShellScript(script, 3*time.Minute, 0) + log.Errorf("[pvc-recovery]\n%s%s(err=%v)", strings.TrimSpace(out), errOut, err) + return strings.Contains(out, "RECOVERING ") +} + // assertWatchdogDriverBound fails the run if no watchdog driver is bound in the // guest. It checks sysfs rather than the device node: a /dev/watchdog character // node can exist with nothing behind it, so its presence alone proves nothing, From d90dae0557caf0e49d71a56fc9904e4d21234265 Mon Sep 17 00:00:00 2001 From: eriknordmark Date: Tue, 28 Jul 2026 16:38:09 -0700 Subject: [PATCH 13/29] evetest: gate the shrink test on the app, not the commit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The framework's upgrade wait returns only once EVE has committed the new partition, which is a trial period lasting well past the point where the device is already running the target. On a conversion that is ten minutes of a twenty-six minute wait, and none of it is needed here: by then the boot disk has been repartitioned and what remains to be established is whether the application and its volume come back. Return as soon as the target is the booted partition and drive the existing readiness gates instead, so a failure lands on the step that actually failed — the cluster, the storage class, the volume — rather than on a timer that expired while the conversion was progressing normally. A rejected image still surfaces immediately, since the wait fails fast when EVE flags one FAILED. Whether the partition was ever committed is recorded at the end, where it explains a later oddity without gating anything. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: eriknordmark --- evetest/tests/resize/appvolshrink_test.go | 68 ++++++++++++++++++++++- 1 file changed, 66 insertions(+), 2 deletions(-) diff --git a/evetest/tests/resize/appvolshrink_test.go b/evetest/tests/resize/appvolshrink_test.go index 3ab1507a76c..c52a68b253a 100644 --- a/evetest/tests/resize/appvolshrink_test.go +++ b/evetest/tests/resize/appvolshrink_test.go @@ -307,7 +307,14 @@ func TestAppVolumeShrinkCorruption(test *testing.T) { dumpConversionFailure(device) } }() - device.UpgradeEVE(convVersion, targetHypervisor, true, false) + // Do not wait for EVE to commit the new partition. Committing is a trial + // period that runs long after the device is already up on the target, and + // nothing this test asserts depends on it: the geometry is grown by then and + // what matters is whether the app and its volume come back. Waiting for it + // only delays — and can fail — a conversion that is doing fine. The commit is + // observed at the end instead, where it costs nothing. + device.UpgradeEVE(convVersion, targetHypervisor, false, false) + waitDeviceOnTarget(t, device, targetHypervisor, 45*time.Minute) conversionOK = true device.ExpectAdditionalReboots(1) captureResizeEvidence(device) @@ -324,7 +331,7 @@ func TestAppVolumeShrinkCorruption(test *testing.T) { if !isK3sReady(device.GetClusterInfo()) { clusterUpdates, stop := device.WatchClusterInfo() defer stop() - t.Eventually(clusterUpdates, 20*time.Minute).Should(Receive( + t.Eventually(clusterUpdates, 30*time.Minute).Should(Receive( matchers.SatisfyPredicate("K3s node is ready", isK3sReady))) } log.Infof("(b) waiting for volumemgr Initialized") @@ -375,6 +382,12 @@ func TestAppVolumeShrinkCorruption(test *testing.T) { t.Expect(presentCorrupt).To(Equal(0), "data volume served corrupt-but-present after the interrupted shrink:\n%s", report) evetest.Checkpoint("volume-verified") + + // Whether EVE committed the new partition is recorded rather than asserted: + // the volume result above does not depend on it, but a target still sitting at + // "inprogress" here means the trial period had not finished, and one that + // reverted would explain an otherwise puzzling later failure. + logPartitionState(device) } // writeVolverifyPattern fills the app's data volume with the deterministic pattern @@ -451,6 +464,57 @@ exit 1`, mountDir, volverifyCommitDir, volumeStatePattern, volumeStateBlank) return state } +// logPartitionState records what EVE currently reports for each base image, so a +// run that did not commit its target (or reverted) is visible after the fact. +// Best-effort; never fails the test. +func logPartitionState(device *evetest.EdgeDevice) { + info := device.GetDeviceInfo() + if info == nil { + evetest.Logger().Errorf("[partition-state] no device info") + return + } + for _, sw := range info.GetSwList() { + evetest.Logger().Errorf("[partition-state] %s partition=%s status=%s %s", + sw.GetShortVersion(), sw.GetPartitionState(), sw.GetUserStatus(), sw.GetSubStatusStr()) + } +} + +// waitDeviceOnTarget waits until the device is RUNNING the target flavor, which +// is a weaker and much earlier condition than the framework's upgrade wait: that +// one blocks until EVE commits the partition (state "active"), whereas this +// returns as soon as the target is the booted partition ("inprogress" counts). +// +// It fails fast if EVE flags any base image FAILED, so a rejected conversion still +// surfaces immediately rather than burning the whole budget. +func waitDeviceOnTarget(t Gomega, device *evetest.EdgeDevice, + hv evetest.Hypervisor, timeout time.Duration) { + // The target's short version carries a flavor suffix; for the conversion the + // only thing that distinguishes it from the kvm hop is that suffix. + suffix := "-kvm-" + if hv == evetest.HypervisorKubevirt { + suffix = "-k-" + } + t.Eventually(func(g Gomega) { + info := device.GetDeviceInfo() + g.Expect(info).NotTo(BeNil()) + var seen []string + for _, sw := range info.GetSwList() { + ver, state := sw.GetShortVersion(), sw.GetPartitionState() + seen = append(seen, fmt.Sprintf("%s[%s/%s]", ver, state, sw.GetUserStatus())) + if !strings.Contains(ver, suffix) { + continue + } + g.Expect(sw.GetUserStatus()).NotTo(Equal(eveinfo.BaseOsStatus_FAILED), + "EVE flagged %s FAILED: %s", ver, sw.GetSubStatusStr()) + if state == "inprogress" || state == "active" { + evetest.Logger().Infof("device is running %s (partition %s)", ver, state) + return + } + } + g.Expect(false).To(BeTrue(), "device not running a %s image yet: %v", suffix, seen) + }, timeout, 15*time.Second).Should(Succeed()) +} + // Rounds of "wait, then try the documented PVC recovery" allowed before the app // is declared stuck, and how long each round waits. const ( From 8a7bcf3f6b2afd9c613a414200996f9b7e0761ab Mon Sep 17 00:00:00 2001 From: eriknordmark Date: Wed, 29 Jul 2026 00:09:23 -0700 Subject: [PATCH 14/29] evetest: place the app volume where the shrink will move it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On a nearly empty /persist the app's data volume is allocated in low blocks, well below the size the shrink cuts the filesystem down to, so the shrink never has to relocate it — and with almost nothing to move the shrink finishes in about a second, too fast for the stress watchdog to interrupt. Runs therefore came back clean for a reason that had nothing to do with whether an interrupted relocation is safe. Fill /persist with incompressible data before deploying the app, so its volume is allocated at the top of the filesystem, then delete the lowest files once the pattern is written. That leaves the volume and a few GiB of survivors above the boundary: enough relocation work for the shrink to run long enough to be cut, while freeing the low blocks the shrink needs to fit and the space EVE-k needs afterwards. The bytes must be incompressible or the qcow2 backing file stores them sparsely and the relocation reads and writes nothing. Then refuse to continue unless the volume really does have blocks above the boundary the resizer reports, since a volume sitting entirely below it is untouched by the shrink and a clean verify would prove nothing. Creating a volume on a deliberately full filesystem also needs EVE's disk check disabled, as the fill is transient and trimmed away before the conversion. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: eriknordmark --- evetest/tests/resize/appvolshrink_test.go | 191 ++++++++++++++++++++++ 1 file changed, 191 insertions(+) diff --git a/evetest/tests/resize/appvolshrink_test.go b/evetest/tests/resize/appvolshrink_test.go index c52a68b253a..2cf41965817 100644 --- a/evetest/tests/resize/appvolshrink_test.go +++ b/evetest/tests/resize/appvolshrink_test.go @@ -4,6 +4,7 @@ package resize_test import ( + "encoding/base64" "fmt" "strings" "testing" @@ -17,6 +18,8 @@ import ( eveconfig "github.com/lf-edge/eve-api/go/config" "github.com/lf-edge/eve-api/go/evecommon" eveinfo "github.com/lf-edge/eve-api/go/info" + pillartypes "github.com/lf-edge/eve/pkg/pillar/types" + "github.com/lf-edge/eve/evetest" "github.com/lf-edge/eve/evetest/constants" "github.com/lf-edge/eve/evetest/matchers" @@ -42,6 +45,19 @@ const ( // the conversion, when the volume is no longer mounted at its MountDir. volverifyCommitDir = ".vv-commit" + fillPeakPctParamKey = "FILL_PEAK_PCT" + fillKeepGiBParamKey = "FILL_KEEP_GIB" + + // Fill /persist to this percentage before the app is deployed, then trim back + // to this many GiB once its volume is written. The peak has to sit well above + // the shrink boundary (~38 GiB of a 61.7 GiB /persist on a 64 GiB boot disk) so + // that the volume, allocated last, lands above it; the keep figure has to stay + // under the resizer's own limit (~34 GiB) and leave room for EVE-k's images + // (~8 GiB measured), while still leaving enough high-residing data that the + // shrink runs long enough for the watchdog to interrupt it. + defaultFillPeakPct = 90 + defaultFillKeepGiB = 15 + // stageCDataVolMiB is the data-volume size this test defaults to. The app // redeploy on EVE-k wedges in a Longhorn CSI CreateVolume race above ~256 MiB // (sweep: 100/256 MiB pass, >=512 MiB wedge), which would mask the corruption @@ -105,6 +121,22 @@ func TestAppVolumeShrinkCorruption(test *testing.T) { Default: "256", }, }, + evetest.TestParameterDefinition{ + Key: fillPeakPctParamKey, + DefaultValue: uint32(defaultFillPeakPct), + Description: evetest.TestParameterDescription{ + Summary: "Fill /persist to this % before deploying the app, so its volume lands in the blocks the shrink evacuates (0 disables)", + Default: "90", + }, + }, + evetest.TestParameterDefinition{ + Key: fillKeepGiBParamKey, + DefaultValue: uint32(defaultFillKeepGiB), + Description: evetest.TestParameterDescription{ + Summary: "Trim /persist back to this many GiB after the volume is written (must leave room for EVE-k)", + Default: "15", + }, + }, evetest.TestParameterDefinition{ Key: seedParamKey, DefaultValue: uint64(20260723), @@ -190,6 +222,11 @@ func TestAppVolumeShrinkCorruption(test *testing.T) { seed := evetest.GetTestParameter[uint64](seedParamKey) ops := evetest.GetTestParameter[uint64](opsParamKey) volverifyImage := evetest.GetTestParameter[string](volverifyImageParamKey) + fillPeakPct := evetest.GetTestParameter[uint32](fillPeakPctParamKey) + fillKeepGiB := evetest.GetTestParameter[uint32](fillKeepGiBParamKey) + if fillPeakPct > 0 { + t.Expect(fillKeepGiB).To(BeNumerically(">", 0), "FILL_KEEP_GIB must be set when filling") + } // Cap a single file at a sixteenth of the volume. volverify's default large-file // bound is 256 MiB, which on a volume this size would let one op consume the @@ -225,6 +262,17 @@ func TestAppVolumeShrinkCorruption(test *testing.T) { volverifyImage, dataVolMiB, maxBlocks) devConfig := evetest.NewEdgeDeviceConfig(devName) + if fillPeakPct > 0 { + // The volume is deliberately created on a nearly-full /persist, which EVE + // would otherwise refuse: volumemgr declines a volume whose size exceeds the + // remaining space, counting a dom0 reservation of 20% on top. The fill is + // transient and trimmed away before the conversion, so the check is what is + // wrong here, not the request. + devConfig.ConfigItems = append(devConfig.ConfigItems, &eveconfig.ConfigItem{ + Key: string(pillartypes.IgnoreDiskCheckForApps), + Value: "true", + }) + } networkUUID := devConfig.AddNetwork(evetest.DHCPNetworkConfig{NetworkType: evecommon.NetworkType_V4}) devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ LogicalLabel: "eth0", @@ -255,6 +303,16 @@ func TestAppVolumeShrinkCorruption(test *testing.T) { settleVaultLocal(t, device) evetest.Checkpoint("vault-settled") + // Fill /persist before the app exists, so its data volume is allocated at the + // top of the filesystem — inside the range the shrink has to evacuate. On an + // almost-empty /persist the volume lands low, the shrink finishes in about a + // second, and the watchdog only ever interrupts the grow. + if fillPeakPct > 0 { + log.Infof("filling /persist to %d%% so the app's volume lands in the shrink's evacuation zone", fillPeakPct) + fillPersistToPct(t, device, int(fillPeakPct)) + evetest.Checkpoint("persist-filled") + } + // Switch (L2-bridged) NI: the carried app gets its own DHCP address on the eth0 // segment and is reached directly. A local NI does not reconverge onto the // kubevirt VMI after the conversion (verified stuck, not slow). @@ -297,6 +355,17 @@ func TestAppVolumeShrinkCorruption(test *testing.T) { log.Infof("pre-conversion: filling the data volume with the volverify pattern") writtenCommitted := writeVolverifyPattern(t, device, appUUID, appAuth, volverifyArgs) log.Infof("volume filled through committed op %d", writtenCommitted) + + // Free the low blocks now that the volume is placed. This is what lets the + // shrink fit at all, and it leaves the relocation work — including the volume — + // concentrated above the boundary. + if fillPeakPct > 0 { + log.Infof("trimming /persist back to %d GiB (lowest blocks first)", fillKeepGiB) + trimPersistToGiB(t, device, int(fillKeepGiB)) + log.Infof("asserting the data volume actually lies above the shrink boundary") + assertVolumeAboveShrinkBoundary(t, device) + evetest.Checkpoint("volume-placed-high") + } capturePersist(device, "pre-conversion EVE-kvm (volume filled)") evetest.Checkpoint("volume-filled") @@ -464,6 +533,128 @@ exit 1`, mountDir, volverifyCommitDir, volumeStatePattern, volumeStateBlank) return state } +// runOnEVEScript runs a shell script on EVE inside the pillar container, passing +// it through base64 so nothing has to survive the ssh → `eve exec pillar sh -c` +// quoting layers (the same trick the eden scripts use). args are appended as $1… +func runOnEVEScript(device *evetest.EdgeDevice, script string, + timeout time.Duration, args ...string) (string, error) { + b64 := base64.StdEncoding.EncodeToString([]byte(script)) + cmd := fmt.Sprintf( + `eve exec pillar sh -c 'echo %s | base64 -d > /tmp/evetest-frag.sh; sh /tmp/evetest-frag.sh %s; rm -f /tmp/evetest-frag.sh'`, + b64, strings.Join(args, " ")) + out, errOut, err := device.RunShellScript(cmd, timeout, 0) + if err != nil { + return out + errOut, err + } + return out, nil +} + +// fillPersistToPct fills /persist with incompressible files until it is pct% full, +// so that the app's data volume — created afterwards, while the filesystem is at +// its peak — is allocated in the high block groups that the shrink must evacuate. +// +// The bytes have to be incompressible. Zeroes would let the qcow2 backing file +// store the blocks sparsely, and the relocation would then read and write nothing, +// leaving the shrink as fast as it is on an empty filesystem — which is the whole +// problem this is here to fix. +func fillPersistToPct(t Gomega, device *evetest.EdgeDevice, pct int) { + const script = `set -u +PCT=$1 +DIR=/persist/tmp/stressfill +rm -rf "$DIR"; mkdir -p "$DIR" +cap=$(df -k /persist | tail -1 | awk '{print $2}') +want=$(( cap * PCT / 100 )) +n=0 +while [ "$(df -k /persist | tail -1 | awk '{print $3}')" -lt "$want" ]; do + f=$(printf "%s/%06d" "$DIR" "$n") + dd if=/dev/urandom of="$f" bs=1M count=256 2>/dev/null || break + n=$((n + 1)) +done +sync +echo "FILLED files=$n $(df -h /persist | tail -1)"` + out, err := runOnEVEScript(device, script, 40*time.Minute, fmt.Sprintf("%d", pct)) + t.Expect(err).NotTo(HaveOccurred(), "filling /persist failed:\n%s", out) + t.Expect(out).To(ContainSubstring("FILLED"), "fill did not report completion:\n%s", out) + evetest.Logger().Infof("filled /persist to ~%d%%: %s", pct, strings.TrimSpace(out)) +} + +// trimPersistToGiB deletes the LOWEST-numbered fill files until /persist usage is +// down to keepGiB, which leaves the survivors — and the data volume created at peak +// — concentrated in the high blocks above the future shrink boundary. +// +// Trimming is what makes the shrink possible at all: the resizer refuses when the +// filesystem cannot fit in the target size, and EVE-k needs room afterwards for +// its own images. Deleting from the bottom is what keeps the relocation work high. +func trimPersistToGiB(t Gomega, device *evetest.EdgeDevice, keepGiB int) { + const script = `set -u +KEEP_KB=$(( $1 * 1024 * 1024 )) +DIR=/persist/tmp/stressfill +[ -d "$DIR" ] || { echo "TRIMMED no-fill-dir"; exit 0; } +for f in $(ls -1 "$DIR" 2>/dev/null | sort); do + [ "$(df -k /persist | tail -1 | awk '{print $3}')" -le "$KEEP_KB" ] && break + rm -f "$DIR/$f" +done +sync +echo "TRIMMED remaining=$(ls -1 "$DIR" 2>/dev/null | wc -l) $(df -h /persist | tail -1)"` + out, err := runOnEVEScript(device, script, 10*time.Minute, fmt.Sprintf("%d", keepGiB)) + t.Expect(err).NotTo(HaveOccurred(), "trimming /persist failed:\n%s", out) + t.Expect(out).To(ContainSubstring("TRIMMED"), "trim did not report completion:\n%s", out) + evetest.Logger().Infof("trimmed /persist to ~%d GiB: %s", keepGiB, strings.TrimSpace(out)) +} + +// assertVolumeAboveShrinkBoundary fails the run unless the app's data volume has +// blocks above the size the shrink will cut /persist down to — i.e. unless the +// shrink will actually have to relocate it. +// +// Without this the test can pass for the wrong reason: a volume that sits entirely +// below the boundary is untouched by the shrink, so a clean verify says nothing +// about whether an interrupted relocation corrupts data. The boundary comes from +// the resizer's own check (TargetBytes) rather than an assumption, and the block +// placement from filefrag, which reports physical extents relative to the +// filesystem the file lives on. +func assertVolumeAboveShrinkBoundary(t Gomega, device *evetest.EdgeDevice) { + disk, err := bootDiskPath(device) + t.Expect(err).NotTo(HaveOccurred()) + out, err := runEVE(device, "eve exec pillar /usr/bin/storage-resizer check --disk "+disk+" --json") + t.Expect(err).NotTo(HaveOccurred()) + // TargetBytes is the post-shrink filesystem size. + var target int64 + if i := strings.Index(out, `"TargetBytes"`); i >= 0 { + fmt.Sscanf(out[i:], `"TargetBytes": %d`, &target) + } + t.Expect(target).To(BeNumerically(">", 0), + "could not read TargetBytes from the resizer check:\n%s", out) + + const script = `set -u +FF=/usr/sbin/filefrag +[ -x "$FF" ] || FF=$(command -v filefrag 2>/dev/null || echo filefrag) +max=0 +for d in /persist/vault/volumes /persist/clear/volumes; do + [ -d "$d" ] || continue + for f in "$d"/*; do + [ -f "$f" ] || continue + e=$("$FF" -b4096 -v "$f" 2>/dev/null | awk '$1 ~ /^[0-9]+:$/ {gsub(/\.\./," ",$4); print $4}' | tr -d ':' | sort -n | tail -1) + [ -n "$e" ] || continue + echo "VOL $f top4k=$e" + [ "$e" -gt "$max" ] && max=$e + done +done +echo "MAXTOP4K $max"` + fragOut, err := runOnEVEScript(device, script, 5*time.Minute) + t.Expect(err).NotTo(HaveOccurred(), "reading volume block placement failed:\n%s", fragOut) + var top4k int64 + if i := strings.Index(fragOut, "MAXTOP4K "); i >= 0 { + fmt.Sscanf(fragOut[i:], "MAXTOP4K %d", &top4k) + } + topByte := top4k * 4096 + evetest.Logger().Infof("volume top block %d (%.1f GiB) vs shrink target %.1f GiB\n%s", + top4k, float64(topByte)/(1<<30), float64(target)/(1<<30), strings.TrimSpace(fragOut)) + t.Expect(topByte).To(BeNumerically(">", target), + "the data volume lies entirely below the shrink boundary (top %.1f GiB vs target %.1f GiB), "+ + "so the shrink would not relocate it and a clean verify would prove nothing", + float64(topByte)/(1<<30), float64(target)/(1<<30)) +} + // logPartitionState records what EVE currently reports for each base image, so a // run that did not commit its target (or reverted) is visible after the fact. // Best-effort; never fails the test. From b8077900419c5f0f8b8449db3837bb6ef2b8fe35 Mon Sep 17 00:00:00 2001 From: eriknordmark Date: Wed, 29 Jul 2026 10:57:05 -0700 Subject: [PATCH 15/29] evetest: read the resizer JSON key as emitted The placement assertion looked for TargetBytes where the resizer emits targetBytes, so it could never find the shrink boundary and failed every run before the conversion started. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: eriknordmark --- evetest/tests/resize/appvolshrink_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/evetest/tests/resize/appvolshrink_test.go b/evetest/tests/resize/appvolshrink_test.go index 2cf41965817..f05050bf7cf 100644 --- a/evetest/tests/resize/appvolshrink_test.go +++ b/evetest/tests/resize/appvolshrink_test.go @@ -617,13 +617,13 @@ func assertVolumeAboveShrinkBoundary(t Gomega, device *evetest.EdgeDevice) { t.Expect(err).NotTo(HaveOccurred()) out, err := runEVE(device, "eve exec pillar /usr/bin/storage-resizer check --disk "+disk+" --json") t.Expect(err).NotTo(HaveOccurred()) - // TargetBytes is the post-shrink filesystem size. + // targetBytes is the post-shrink filesystem size (ShrinkResult.TargetBytes). var target int64 - if i := strings.Index(out, `"TargetBytes"`); i >= 0 { - fmt.Sscanf(out[i:], `"TargetBytes": %d`, &target) + if i := strings.Index(out, `"targetBytes"`); i >= 0 { + fmt.Sscanf(out[i:], `"targetBytes": %d`, &target) } t.Expect(target).To(BeNumerically(">", 0), - "could not read TargetBytes from the resizer check:\n%s", out) + "could not read targetBytes from the resizer check:\n%s", out) const script = `set -u FF=/usr/sbin/filefrag From c49c17034c01206c33c565664ea2ef0385fdbefd Mon Sep 17 00:00:00 2001 From: eriknordmark Date: Wed, 29 Jul 2026 11:16:20 -0700 Subject: [PATCH 16/29] evetest: fix two untested paths in the shrink test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Neither of these had ever executed, so both were wrong in ways no run had yet exposed. The size comparison that decides whether a Pending PVC may be deleted understood only Gi and Mi suffixes, but kubectl prints these PVCs as a plain byte count. Every size therefore measured as zero and every Pending PVC was a candidate for deletion — including the data volume and its import scratch, the two it exists to protect. Deleting those does not merely lose a recovery: EVE recreates the volume empty, the verify then finds nothing wrong, and the run reports success. Handle each form kubectl may print and protect anything unrecognised, so a parsing gap can only cost a recovery rather than the result. The volume placement check read the physical extent's start column as though it were the end, and would have broken outright had filefrag closed up the space inside its "start.. end" range. Close that space first and take the range's second half, which holds regardless of alignment. Verified against real filefrag output in both spacings and against a file with no extents. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: eriknordmark --- evetest/tests/resize/appvolshrink_test.go | 30 ++++++++++++++++++----- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/evetest/tests/resize/appvolshrink_test.go b/evetest/tests/resize/appvolshrink_test.go index f05050bf7cf..a4ef7bfa123 100644 --- a/evetest/tests/resize/appvolshrink_test.go +++ b/evetest/tests/resize/appvolshrink_test.go @@ -625,15 +625,23 @@ func assertVolumeAboveShrinkBoundary(t Gomega, device *evetest.EdgeDevice) { t.Expect(target).To(BeNumerically(">", 0), "could not read targetBytes from the resizer check:\n%s", out) + // The physical range is the third column once any space inside "start.. end" + // is closed up, which keeps the parse independent of filefrag's alignment; the + // second half of that range is the file's highest block. Verified against real + // filefrag output for both spacings and for a file with no extents. const script = `set -u +sync FF=/usr/sbin/filefrag [ -x "$FF" ] || FF=$(command -v filefrag 2>/dev/null || echo filefrag) max=0 -for d in /persist/vault/volumes /persist/clear/volumes; do +for d in /persist/vault/volumes /persist/clear/volumes /persist/vault/volumes-kvm /persist/clear/volumes-kvm; do [ -d "$d" ] || continue for f in "$d"/*; do [ -f "$f" ] || continue - e=$("$FF" -b4096 -v "$f" 2>/dev/null | awk '$1 ~ /^[0-9]+:$/ {gsub(/\.\./," ",$4); print $4}' | tr -d ':' | sort -n | tail -1) + e=$("$FF" -b4096 -v "$f" 2>/dev/null | awk ' + { line=$0; gsub(/\.\.[ \t]+/, "..", line); $0=line } + $1 ~ /^[0-9]+:$/ { split($3, r, /\.\./); x=r[2]; sub(/:$/, "", x); if (x+0 > m) m=x+0 } + END { print m+0 }') [ -n "$e" ] || continue echo "VOL $f top4k=$e" [ "$e" -gt "$max" ] && max=$e @@ -775,6 +783,12 @@ func waitAppRunningQuietly(device *evetest.EdgeDevice, appUUID uuid.UUID, timeou // is needed. At data-volume sizes near the image size the app's own PVC gets // protected too and recovery no-ops; the classification is logged so that is // visible rather than silent. +// +// Sizes arrive in whatever form kubectl prints, which for these PVCs is a plain +// byte count rather than the Gi/Mi suffixes one might expect, so every form is +// handled and anything unrecognised is protected rather than deleted. Getting this +// wrong is not a missed recovery but a destroyed volume followed by a run that +// passes because the volume came back empty. func recoverWedgedAppPVCs(device *evetest.EdgeDevice, dataVolMiB uint32) bool { log := evetest.Logger() script := fmt.Sprintf(`set -u @@ -786,12 +800,16 @@ LIST=$(eve exec kube kubectl -n eve-kube-app get pvc \ echo "$LIST" | sed 's/^/PVC: /' VICTIMS=$(echo "$LIST" | awk -v dv="$DV" ' function mib(s) { - if (s ~ /Gi$/) { sub(/Gi$/, "", s); return s * 1024 } - if (s ~ /Mi$/) { sub(/Mi$/, "", s); return s + 0 } - return 0 + if (s ~ /^[0-9]+$/) { return s / 1048576 } + if (s ~ /^[0-9]+Ki$/) { sub(/Ki$/, "", s); return s / 1024 } + if (s ~ /^[0-9]+Mi$/) { sub(/Mi$/, "", s); return s + 0 } + if (s ~ /^[0-9]+Gi$/) { sub(/Gi$/, "", s); return s * 1024 } + if (s ~ /^[0-9]+Ti$/) { sub(/Ti$/, "", s); return s * 1048576 } + return -1 } $2 == "Pending" { - if (mib($3) >= dv - 32) { printf "PROTECTED %%s (%%s)\n", $1, $3; next } + v = mib($3) + if (v < 0 || v >= dv - 32) { printf "PROTECTED %%s (%%s)\n", $1, $3; next } printf "VICTIM %%s (%%s)\n", $1, $3 }') echo "$VICTIMS" From 11af0682aca9d6a06700b5267d7b12b2d69fb2f4 Mon Sep 17 00:00:00 2001 From: eriknordmark Date: Wed, 29 Jul 2026 11:33:42 -0700 Subject: [PATCH 17/29] evetest: record the fsck verdict beside the content verify MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The test asserted that no file came back corrupt but never recorded what a filesystem check made of the same volume, so it could not answer the question it exists to answer: when an interrupted relocation damages application data, would anything routine have noticed? That gap matters because the two verdicts are expected to disagree. A structural check is blind to data blocks relocated wrongly but left self-consistent — on a loopback filesystem, zeroing four mebibytes of data left e2fsck reporting a clean filesystem with an empty lost+found while the content verify found seventy-eight files silently full of zeroes. Either verdict alone is ambiguous; the pair is the finding. Check the volume before anything mounts it, since a mount replays the journal and can repair the damage under measurement, and check read-only so the check itself changes nothing. Locate the device by size rather than by mounting it to look inside. Emit both verdicts as a single line per run, so a soak can accumulate hundreds of them and the disagreement rate becomes the result rather than an anecdote. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: eriknordmark --- evetest/tests/resize/appvolshrink_test.go | 91 +++++++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/evetest/tests/resize/appvolshrink_test.go b/evetest/tests/resize/appvolshrink_test.go index a4ef7bfa123..7f172a1194e 100644 --- a/evetest/tests/resize/appvolshrink_test.go +++ b/evetest/tests/resize/appvolshrink_test.go @@ -425,8 +425,16 @@ func TestAppVolumeShrinkCorruption(test *testing.T) { log.Infof("(f) post-conversion: re-verifying the data-volume pattern") captureVolManifest(device) + + // Check the filesystem before anything mounts it, so the structural verdict can + // be set against the content verdict below. A mount would replay the journal and + // could repair what is being measured. + volDev := findDataVolumeDevice(t, device, appUUID, appAuth, dataVolMiB) + fsckRC, fsckOut := fsckDataVolume(device, appUUID, appAuth, volDev) + state := mountDataVolumeRO(t, device, appUUID, appAuth, dataMountDir) if state == volumeStateBlank { + recordVolumeOutcome(dataVolMiB, state, fsckRC, "", fsckOut) // The pattern is gone but the volume is intact and empty: either the // post-resize manifest check found the volume torn and removed it, so EVE // recreated it blank, or the shrink destroyed the filesystem outright. The @@ -440,6 +448,7 @@ func TestAppVolumeShrinkCorruption(test *testing.T) { report := verifyVolverifyPattern(t, device, appUUID, appAuth, volverifyArgs, writtenCommitted) log.Infof("volverify report:\n%s", report) + recordVolumeOutcome(dataVolMiB, state, fsckRC, report, fsckOut) // present-corrupt is the silent case: a torn-but-present volume that EVE would // serve to the app as-is, invisible to fsck and to qemu-img. orphaned and lost @@ -484,6 +493,88 @@ const ( volumeStateBlank = "BLANK" // the volume exists but the pattern is gone ) +// recordVolumeOutcome emits one line pairing what the filesystem check concluded +// with what the content verify found, which is the row a soak accumulates over +// many iterations. Read on its own, either verdict is ambiguous; together they say +// whether corruption occurred and whether a structural check would have noticed. +// +// The line is deliberately single and grep-friendly, since the point is to compare +// hundreds of these rather than to read one. +func recordVolumeOutcome(dataVolMiB uint32, state string, fsckRC int, report, fsckOut string) { + fsckVerdict := fmt.Sprintf("rc=%d", fsckRC) + switch fsckRC { + case 0: + fsckVerdict += "(clean)" + case 4: + fsckVerdict += "(errors-found)" + } + if strings.Contains(fsckOut, "FILE SYSTEM WAS MODIFIED") { + fsckVerdict += "+modified" + } + if report == "" { + report = "no-content-verify" + } + evetest.Logger().Errorf("[VOLUME-OUTCOME] datavolMiB=%d state=%s fsck=%s verify=[%s]", + dataVolMiB, state, fsckVerdict, strings.ReplaceAll(report, "\n", " | ")) +} + +// fsckDataVolume runs a read-only filesystem check on the data volume's block +// device and returns e2fsck's exit status and output. +// +// This is the counterpart to the content verify, and the pair is the point: a +// structural check is blind to data blocks that were relocated wrongly but left +// self-consistent, so the interesting result is fsck reporting a clean filesystem +// while the content verify finds files quietly full of zeroes. Recording only one +// of the two would lose exactly the comparison this test exists to make. +// +// It must run before anything mounts the volume — a mount can replay the journal +// and repair the very damage being measured — and with -n so the check itself +// changes nothing. Exit status 0 means fsck saw a clean filesystem; 4 means it +// found errors it was not allowed to fix. +func fsckDataVolume(device *evetest.EdgeDevice, appUUID uuid.UUID, + auth evetest.AuthMethod, dev string) (int, string) { + script := fmt.Sprintf("e2fsck -fn %s 2>&1; echo FSCK_RC=$?", dev) + out, _, _ := device.RunShellScriptInsideApp(appUUID, auth, script, 20*time.Minute, 0) + rc := -1 + if i := strings.Index(out, "FSCK_RC="); i >= 0 { + fmt.Sscanf(out[i:], "FSCK_RC=%d", &rc) + } + evetest.Logger().Errorf("[fsck] %s exit=%d\n%s", dev, rc, strings.TrimSpace(out)) + return rc, strings.TrimSpace(out) +} + +// findDataVolumeDevice returns the guest block device holding the app's data +// volume, identified by size rather than by mounting anything, so the caller can +// check the filesystem before it is touched. The app's own root disk differs in +// size by orders of magnitude, so a size match is unambiguous here. +func findDataVolumeDevice(t Gomega, device *evetest.EdgeDevice, appUUID uuid.UUID, + auth evetest.AuthMethod, dataVolMiB uint32) string { + script := fmt.Sprintf(`set -u +WANT=%d +for d in /dev/vd[b-z] /dev/sd[b-z]; do + [ -b "$d" ] || continue + sz=$(blockdev --getsize64 "$d" 2>/dev/null) || continue + mib=$(( sz / 1048576 )) + echo "CAND $d ${mib}MiB" + diff=$(( mib - WANT )); [ "$diff" -lt 0 ] && diff=$(( -diff )) + [ "$diff" -le 128 ] && { echo "DEV=$d"; break; } +done`, dataVolMiB) + var dev string + t.Eventually(func(g Gomega) { + out, _, err := device.RunShellScriptInsideApp(appUUID, auth, script, 2*time.Minute, 0) + g.Expect(err).NotTo(HaveOccurred(), "listing guest block devices failed:\n%s", out) + for _, tok := range strings.Fields(out) { + if d, ok := strings.CutPrefix(tok, "DEV="); ok { + dev = d + } + } + g.Expect(dev).NotTo(BeEmpty(), + "no guest block device close to %d MiB — the data volume is not attached:\n%s", dataVolMiB, out) + }, 5*time.Minute, 15*time.Second).Should(Succeed()) + evetest.Logger().Infof("data volume device: %s", dev) + return dev +} + // mountDataVolumeRO finds the app's data volume among the guest's block devices // and mounts it at mountDir, returning which of the states above it is in. EVE-k // does not auto-mount a container app's data volume at its MountDir From bb10b8c151fcc0212c1f68747df975898fceaab6 Mon Sep 17 00:00:00 2001 From: eriknordmark Date: Wed, 29 Jul 2026 13:48:16 -0700 Subject: [PATCH 18/29] evetest: check the volume again with the journal replayed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The filesystem check taken before the volume is mounted cannot tell real damage from bookkeeping. The volume is never cleanly unmounted, so its journal is unreplayed and the superblock disagrees with the disk by construction: one run reported 235353 free blocks against an actual 4644. That check therefore reports errors every time and, taken alone, would have made the structural column a constant in a soak — useless for spotting the case worth finding, where a check calls a damaged volume clean. Check it a second time once the content verdict is recorded, with the journal replayed and repairs allowed, so anything still reported is genuine. That modifies the filesystem, which is why it runs last and why the volume is unmounted first. Record both readings: the pair separates stale accounting from damage. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: eriknordmark --- evetest/tests/resize/appvolshrink_test.go | 70 +++++++++++++++++++---- 1 file changed, 60 insertions(+), 10 deletions(-) diff --git a/evetest/tests/resize/appvolshrink_test.go b/evetest/tests/resize/appvolshrink_test.go index 7f172a1194e..62640e81dcb 100644 --- a/evetest/tests/resize/appvolshrink_test.go +++ b/evetest/tests/resize/appvolshrink_test.go @@ -434,7 +434,7 @@ func TestAppVolumeShrinkCorruption(test *testing.T) { state := mountDataVolumeRO(t, device, appUUID, appAuth, dataMountDir) if state == volumeStateBlank { - recordVolumeOutcome(dataVolMiB, state, fsckRC, "", fsckOut) + recordVolumeOutcome(dataVolMiB, state, fsckRC, "", fsckOut, -1, "") // The pattern is gone but the volume is intact and empty: either the // post-resize manifest check found the volume torn and removed it, so EVE // recreated it blank, or the shrink destroyed the filesystem outright. The @@ -448,7 +448,13 @@ func TestAppVolumeShrinkCorruption(test *testing.T) { report := verifyVolverifyPattern(t, device, appUUID, appAuth, volverifyArgs, writtenCommitted) log.Infof("volverify report:\n%s", report) - recordVolumeOutcome(dataVolMiB, state, fsckRC, report, fsckOut) + + // Content is captured, so the filesystem can now be checked with the journal + // replayed — the only reading that distinguishes real damage from the stale + // accounting an unclean unmount leaves behind. + log.Infof("re-checking the volume filesystem with the journal replayed") + replayRC, replayOut := fsckDataVolumeAfterVerify(device, appUUID, appAuth, volDev, dataMountDir) + recordVolumeOutcome(dataVolMiB, state, fsckRC, report, fsckOut, replayRC, replayOut) // present-corrupt is the silent case: a torn-but-present volume that EVE would // serve to the app as-is, invisible to fsck and to qemu-img. orphaned and lost @@ -500,22 +506,40 @@ const ( // // The line is deliberately single and grep-friendly, since the point is to compare // hundreds of these rather than to read one. -func recordVolumeOutcome(dataVolMiB uint32, state string, fsckRC int, report, fsckOut string) { - fsckVerdict := fmt.Sprintf("rc=%d", fsckRC) +func recordVolumeOutcome(dataVolMiB uint32, state string, fsckRC int, report, fsckOut string, + replayRC int, replayOut string) { + // Before the journal is replayed, count mismatches are expected and mean + // nothing; after it, anything found is real. + dirty := fmt.Sprintf("rc=%d", fsckRC) switch fsckRC { case 0: - fsckVerdict += "(clean)" + dirty += "(clean)" case 4: - fsckVerdict += "(errors-found)" + dirty += "(errors-unreplayed)" + } + if strings.Contains(fsckOut, "skipping journal recovery") { + dirty += "+journal-not-replayed" } - if strings.Contains(fsckOut, "FILE SYSTEM WAS MODIFIED") { - fsckVerdict += "+modified" + replayed := "not-run" + if replayRC >= 0 { + replayed = fmt.Sprintf("rc=%d", replayRC) + switch replayRC { + case 0: + replayed += "(clean)" + case 1, 2: + replayed += "(REAL-DAMAGE-FIXED)" + case 4: + replayed += "(errors-left)" + } + if strings.Contains(replayOut, "FILE SYSTEM WAS MODIFIED") { + replayed += "+modified" + } } if report == "" { report = "no-content-verify" } - evetest.Logger().Errorf("[VOLUME-OUTCOME] datavolMiB=%d state=%s fsck=%s verify=[%s]", - dataVolMiB, state, fsckVerdict, strings.ReplaceAll(report, "\n", " | ")) + evetest.Logger().Errorf("[VOLUME-OUTCOME] datavolMiB=%d state=%s fsck-dirty=%s fsck-replayed=%s verify=[%s]", + dataVolMiB, state, dirty, replayed, strings.ReplaceAll(report, "\n", " | ")) } // fsckDataVolume runs a read-only filesystem check on the data volume's block @@ -543,6 +567,32 @@ func fsckDataVolume(device *evetest.EdgeDevice, appUUID uuid.UUID, return rc, strings.TrimSpace(out) } +// fsckDataVolumeAfterVerify unmounts the volume and checks it again, this time +// letting e2fsck replay the journal and repair what it finds. +// +// The read-only check taken before the mount cannot distinguish real damage from +// bookkeeping: the volume was never cleanly unmounted, so its journal is unreplayed +// and the superblock's free counts necessarily disagree with the disk. That check +// therefore reports errors on every run and says nothing on its own. Replaying +// first removes that noise, so whatever is still wrong here is genuine — at the +// cost of modifying the filesystem, which is why it runs only after the content +// verify has already been recorded. +// +// Exit 0 means clean once the journal was applied; 1 means e2fsck found and fixed +// real structural damage. +func fsckDataVolumeAfterVerify(device *evetest.EdgeDevice, appUUID uuid.UUID, + auth evetest.AuthMethod, dev, mountDir string) (int, string) { + script := fmt.Sprintf( + "umount %s 2>/dev/null; e2fsck -fy %s 2>&1; echo FSCK_RC=$?", mountDir, dev) + out, _, _ := device.RunShellScriptInsideApp(appUUID, auth, script, 20*time.Minute, 0) + rc := -1 + if i := strings.Index(out, "FSCK_RC="); i >= 0 { + fmt.Sscanf(out[i:], "FSCK_RC=%d", &rc) + } + evetest.Logger().Errorf("[fsck-replayed] %s exit=%d\n%s", dev, rc, strings.TrimSpace(out)) + return rc, strings.TrimSpace(out) +} + // findDataVolumeDevice returns the guest block device holding the app's data // volume, identified by size rather than by mounting anything, so the caller can // check the filesystem before it is touched. The app's own root disk differs in From 7f433ff581ad2b713b26d96f914c0d4308cf7bbf Mon Sep 17 00:00:00 2001 From: eriknordmark Date: Thu, 30 Jul 2026 06:52:45 -0700 Subject: [PATCH 19/29] evetest: judge volume damage by findings, not exit status MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A volume captured while it was still being written has stale superblock counters, the orphan-file flag set and extent trees e2fsck would rather rewrite. Repairing those returns a non-zero status on every run, so labelling any non-zero status as damage marked healthy iterations as hits and would have buried the signal a soak exists to find — the same mistake as reading the pre-replay status, one step later. Classify on the findings instead: unattached or deleted inodes, crossed or illegal blocks, bitmap differences, anything reaching lost+found. Two soak iterations that reported damage under the old rule repaired only free-count and orphan-flag bookkeeping, with all file contents verifying clean. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: eriknordmark --- evetest/tests/resize/appvolshrink_test.go | 49 ++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/evetest/tests/resize/appvolshrink_test.go b/evetest/tests/resize/appvolshrink_test.go index 62640e81dcb..07f68ee2dee 100644 --- a/evetest/tests/resize/appvolshrink_test.go +++ b/evetest/tests/resize/appvolshrink_test.go @@ -527,7 +527,17 @@ func recordVolumeOutcome(dataVolMiB uint32, state string, fsckRC int, report, fs case 0: replayed += "(clean)" case 1, 2: - replayed += "(REAL-DAMAGE-FIXED)" + // A non-zero status is not by itself damage. A volume captured while it + // was still being written to has stale superblock counters, an orphan + // flag and extent trees e2fsck would rather rewrite, and it repairs all + // of those on every run while saying nothing about the data. Only + // findings that imply lost, crossed or unreachable blocks mean the + // interrupted shrink actually hurt the volume. + if fsckFoundStructuralDamage(replayOut) { + replayed += "(STRUCTURAL-DAMAGE)" + } else { + replayed += "(accounting-only)" + } case 4: replayed += "(errors-left)" } @@ -567,6 +577,43 @@ func fsckDataVolume(device *evetest.EdgeDevice, appUUID uuid.UUID, return rc, strings.TrimSpace(out) } +// fsckFoundStructuralDamage reports whether an e2fsck transcript contains findings +// that mean blocks or inodes were actually lost, crossed or orphaned — as opposed +// to the accounting an unclean capture always produces. +// +// Repairing stale free-block and free-inode counters, clearing the orphan-file +// feature flag and narrowing extent trees all happen on a volume that was simply +// snapshotted mid-write; treating those as damage would mark every iteration of a +// soak as a hit and bury the real signal. The patterns below are the ones that +// imply data actually went missing. +func fsckFoundStructuralDamage(out string) bool { + damage := []string{ + "Unattached inode", + "Unattached zero-length inode", + "multiply-claimed", + "Multiply-claimed", + "illegal block", + "Illegal block", + "illegal indirect block", + "lost+found", + "Inode bitmap differences", + "Block bitmap differences", + "Directory inode", + "has an incorrect filesize", + "Entry '", + "deleted/unused inode", + "root inode is not a directory", + "Corrupt", + "corrupted", + } + for _, d := range damage { + if strings.Contains(out, d) { + return true + } + } + return false +} + // fsckDataVolumeAfterVerify unmounts the volume and checks it again, this time // letting e2fsck replay the journal and repair what it finds. // From a6abcc400c5b71f3715974b42d18623d849a452a Mon Sep 17 00:00:00 2001 From: eriknordmark Date: Thu, 30 Jul 2026 10:52:24 -0700 Subject: [PATCH 20/29] evetest: capture wedge diags when app never runs The app failing to reach RUNNING after the kvm->k conversion is the most common way this test fails, and it was the one failure that produced no diagnostics: the deferred capture of PVC state, csi-provisioner and longhorn-manager logs was registered below the app-RUNNING wait, so a failure inside that wait skipped it entirely. Six consecutive wedges were recorded with nothing to explain them. Register the capture before the wait and report which stage was reached, so a run that never gets the app up still leaves the evidence behind. Co-Authored-By: Claude Opus 5 Signed-off-by: eriknordmark --- evetest/tests/resize/appvolshrink_test.go | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/evetest/tests/resize/appvolshrink_test.go b/evetest/tests/resize/appvolshrink_test.go index 07f68ee2dee..cbd87cbb06b 100644 --- a/evetest/tests/resize/appvolshrink_test.go +++ b/evetest/tests/resize/appvolshrink_test.go @@ -408,15 +408,25 @@ func TestAppVolumeShrinkCorruption(test *testing.T) { log.Infof("(c) waiting for longhorn StorageClass ready") waitLonghornSC(t, device) } - log.Infof("(d) waiting for the app to reach RUNNING on the target") - waitAppRunningWithPVCRecovery(t, device, appUUID, dataVolMiB) + // Registered above the wait, not below it: the app failing to reach RUNNING is the + // most common way this test fails, and a defer set up after the wait never runs on + // that path — which is exactly where the wedge evidence is needed. + appRunning := false appSSHOK := false defer func() { - if !appSSHOK { - captureAppNet(device, "post-conversion EVE-k (app net FAILED)") - captureAppNetFailure(device, appUUID) + if appSSHOK { + return } + stage := "never reached RUNNING" + if appRunning { + stage = "RUNNING but unreachable" + } + captureAppNet(device, "post-conversion EVE-k (app "+stage+")") + captureAppNetFailure(device, appUUID) }() + log.Infof("(d) waiting for the app to reach RUNNING on the target") + waitAppRunningWithPVCRecovery(t, device, appUUID, dataVolMiB) + appRunning = true log.Infof("(e0) post-conversion: waiting up to 10m for the app to report a routable IPv4") waitAppHasRoutableIPv4(t, device, appUUID, 10*time.Minute) log.Infof("(e) post-conversion: app must be SSH-reachable") From 5448260f1961729b44224ec7d13f7e6e9d35e485 Mon Sep 17 00:00:00 2001 From: eriknordmark Date: Thu, 30 Jul 2026 11:04:19 -0700 Subject: [PATCH 21/29] evetest: search all of newlog, not just collect/ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five diagnostic probes globbed /persist/newlog/collect/*.log. That directory holds only the last few minutes of records — newlogd closes a collect file and moves it to the gzipped queues once it passes 550000 bytes or a 300s timer expires — so the probes saw almost nothing of the window they were meant to explain. The glob was narrower still: the chunks are named dev.log.keep. and dev.log.upload., so *.log matched only current.device.log, a symlink to the live keep stream, missing even the upload half beside it. Every wedge and watchdog probe was reporting a near-empty result that read as "the event never happened". Route them through the retrieval form the eve-device-logs skill already documents, which walks the whole newlog tree one file at a time. Checked against a collected device tree of 133 chunks: the old glob found no BootReason records, the new form finds 85. Co-Authored-By: Claude Opus 5 Signed-off-by: eriknordmark --- evetest/tests/resize/appvolshrink_test.go | 6 +++--- evetest/tests/resize/kvmtok_resize_test.go | 20 ++++++++++++++++++-- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/evetest/tests/resize/appvolshrink_test.go b/evetest/tests/resize/appvolshrink_test.go index cbd87cbb06b..6f4786355e2 100644 --- a/evetest/tests/resize/appvolshrink_test.go +++ b/evetest/tests/resize/appvolshrink_test.go @@ -1068,10 +1068,10 @@ func captureResizeEvidence(device *evetest.EdgeDevice) { log.Errorf("=== offline-resize fault evidence ===") probes := []struct{ what, script string }{ {"boot / reboot reasons", `for f in /persist/boot-reason /persist/reboot-reason /persist/status/boot-reason /persist/status/reboot-reason /persist/log/reboot-reason.log; do [ -f "$f" ] && { echo "--- $f"; cat "$f"; }; done 2>/dev/null || echo NONE`}, - {"watchdog boot reason in logs", `eve exec pillar sh -c 'grep -ahoE "BootReason[A-Za-z]+" /persist/newlog/collect/*.log 2>/dev/null | sort | uniq -c | sort -rn | head' || echo none`}, + {"watchdog boot reason in logs", newlogProbe(`grep -ahoE "BootReason[A-Za-z]+" | sort | uniq -c | sort -rn | head`)}, {"resize-failed.json", `cat /config/resize-failed.json 2>/dev/null || echo NONE`}, {"watchdog device", `[ -c /dev/watchdog ] && echo PRESENT || echo MISSING; wdctl /dev/watchdog 2>&1 | head -8`}, - {"resizer/watchdog log lines", `eve exec pillar sh -c 'grep -ahiE "run-watchdog|storage-resizer|resize did not converge|watchdog" /persist/newlog/collect/*.log 2>/dev/null | tail -30' || echo none`}, + {"resizer/watchdog log lines", newlogProbe(`grep -ahiE "run-watchdog|storage-resizer|resize did not converge|watchdog" | tail -30`)}, } for _, p := range probes { out, errOut, err := device.RunShellScript(p.script, 60*time.Second, 0) @@ -1088,7 +1088,7 @@ func captureVolManifest(device *evetest.EdgeDevice) { log.Errorf("=== volume-manifest (post-resize detect+recreate) ===") probes := []struct{ what, script string }{ {"manifest files on /persist", `eve exec pillar sh -c 'ls -l /persist/vault/volumes/.sha256 /persist/clear/volumes/.sha256 2>&1'`}, - {"volmanifest / recreate signatures (newlog)", `eve exec pillar sh -c 'grep -ahiE "volmanifest|recreateCorruptVolumes|verifyVolumes|torn by the resize" /persist/newlog/collect/*.log 2>/dev/null | tail -40' || echo none`}, + {"volmanifest / recreate signatures (newlog)", newlogProbe(`grep -ahiE "volmanifest|recreateCorruptVolumes|verifyVolumes|torn by the resize" | tail -40`)}, {"app volume files", `eve exec pillar sh -c 'ls -l /persist/vault/volumes /persist/clear/volumes 2>&1'`}, } for _, p := range probes { diff --git a/evetest/tests/resize/kvmtok_resize_test.go b/evetest/tests/resize/kvmtok_resize_test.go index 1cff3e34c90..e04a022efde 100644 --- a/evetest/tests/resize/kvmtok_resize_test.go +++ b/evetest/tests/resize/kvmtok_resize_test.go @@ -405,6 +405,22 @@ func captureAppNet(device *evetest.EdgeDevice, label string) { // engaged: the scratch/data PVC state (Terminating? storageClass?), the cdi-upload // pod phase/reason, Longhorn volume+attachment state, the CDI upload-controller // reconcile errors, and the volumemgr / MOUNT-WEDGE-RECOVERY newlog signatures. +// newlogCat emits every device log record newlogd has kept, for a grep to consume. +// This is the retrieval form from the eve-device-logs skill (see also +// kvm-to-k-conversion-testing/scripts/assert-seal-v2.sh) — do not hand-roll another. +// Narrowing to collect/ finds almost nothing: newlogd moves a collect file into the +// gzipped queues once it passes 550000 bytes or a 300 s timer, so that directory is a +// ~5-minute window rather than a boot's worth of records. -exec … \; runs one zcat +// per file, which both keeps -f safe on the plaintext chunks and avoids an argument +// list that a device with tens of thousands of chunks overflows. +const newlogCat = `find /persist/newlog -name "dev.log.*" -exec zcat -f {} \; 2>/dev/null` + +// newlogProbe builds a diagnostic-table entry running pipeline over that history. +// pipeline must not contain single quotes. +func newlogProbe(pipeline string) string { + return `eve exec pillar sh -c '` + newlogCat + ` | ` + pipeline + `' || echo none` +} + func captureAppNetFailure(device *evetest.EdgeDevice, appUUID uuid.UUID) { log := evetest.Logger() log.Errorf("=== EVE-k app-pvc-not-ready / CDI-wedge diagnostics (app %s) ===", appUUID) @@ -419,8 +435,8 @@ func captureAppNetFailure(device *evetest.EdgeDevice, appUUID uuid.UUID) { {"longhorn StorageClass (replicas/config)", `eve exec kube kubectl get sc longhorn -o yaml 2>/dev/null || echo none`}, {"longhorn version (manager ds image)", `eve exec kube kubectl -n longhorn-system get ds longhorn-manager -o wide 2>/dev/null || echo none`}, {"CDI upload-controller log", `eve exec kube kubectl -n cdi logs deployment/cdi-deployment --tail=60 2>/dev/null || echo none`}, - {"volumemgr RolloutDiskToPVC / V5 signature (newlog)", `eve exec pillar sh -c 'grep -aiE "RolloutDiskToPVC|retryFailedClusterVolumeCreate|terminating:true|local-path" /persist/newlog/collect/*.log 2>/dev/null | tail -30' || echo none`}, - {"MOUNT-WEDGE-RECOVERY (detector fired?)", `eve exec pillar sh -c 'grep -ai MOUNT-WEDGE-RECOVERY /persist/newlog/collect/*.log 2>/dev/null | tail -10' || echo none`}, + {"volumemgr RolloutDiskToPVC / V5 signature (newlog)", newlogProbe(`grep -aiE "RolloutDiskToPVC|retryFailedClusterVolumeCreate|terminating:true|local-path" | tail -30`)}, + {"MOUNT-WEDGE-RECOVERY (detector fired?)", newlogProbe(`grep -ai MOUNT-WEDGE-RECOVERY | tail -10`)}, {"kubevirt vmi -A", `eve exec kube kubectl get vmi -A -o wide 2>/dev/null || echo none`}, } for _, p := range probes { From a7a3a7bdeffb7161a6599760c4f95d4985493825 Mon Sep 17 00:00:00 2001 From: eriknordmark Date: Thu, 30 Jul 2026 12:51:03 -0700 Subject: [PATCH 22/29] evetest: take the volume verdict without the app MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every check of the migrated data volume ran inside the app guest, so the post-conversion app wedge destroyed the whole verdict — filesystem check and content check alike. Seven consecutive iterations exercised the interrupted shrink and recorded nothing about the volume, including one that cut the shrink twice with the volume 17 GiB inside the relocated range. Read the volume on the device instead, before the cluster can ingest it. upgradeconverter parks the carried-over volumes in /persist/vault/volumes-kvm as plain ext4 images, and the data-volume path leaves the source in place when the PVC rollout fails, so a wedged import is exactly the case where the file is still there. containerd has already unpacked the app image, so the same volverify binary the app would run is on disk and needs no transfer. Verified live: fsck read-only on the image, loop-mounted read-only alongside an in-flight import with no contention, verify clean at committed=2944. The app-side path stays as the cross-check that covers the import itself. Co-Authored-By: Claude Opus 5 Signed-off-by: eriknordmark --- evetest/tests/resize/appvolshrink_test.go | 100 ++++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/evetest/tests/resize/appvolshrink_test.go b/evetest/tests/resize/appvolshrink_test.go index 6f4786355e2..90c8830dd19 100644 --- a/evetest/tests/resize/appvolshrink_test.go +++ b/evetest/tests/resize/appvolshrink_test.go @@ -395,6 +395,11 @@ func TestAppVolumeShrinkCorruption(test *testing.T) { stopPersistSampler := startPersistSampler(device, 45*time.Second) defer stopPersistSampler() + // Take the verdict here, before Longhorn can ingest the volume and long before + // the app is asked to boot. This is the only reading that survives the + // post-conversion app wedge, which has cost every verdict it has hit. + verifyRelocatedVolumeOnDevice(t, device, dataVolMiB, seed, ops, maxBlocks, writtenCommitted) + if targetHypervisor == evetest.HypervisorKubevirt { log.Infof("(a) waiting for k3s node ready") if !isK3sReady(device.GetClusterInfo()) { @@ -484,6 +489,101 @@ func TestAppVolumeShrinkCorruption(test *testing.T) { logPartitionState(device) } +// devsideVerifyScript checks the carried-over volume on the device itself. +// +// upgradeconverter moves the pre-conversion app volumes out of +// /persist/vault/volumes — which Longhorn must own on EVE-k — into +// /persist/vault/volumes-kvm, and volumemgr rolls them into PVCs lazily once the +// cluster is up. In between, and for as long as that import keeps failing, the file +// sits there as a plain ext4 image holding the volume exactly as the shrink left it. +// The data-volume path in csihandler returns on a failed RolloutDiskToPVC without +// removing the source, so a wedged import preserves it rather than consuming it. +// +// volverify does not have to be shipped in: containerd has already unpacked the app +// image, so the same statically-linked binary the app would run is on disk. Args: +// seed, ops, max-blocks, expect-committed. +const devsideVerifyScript = `set -u +VV=$(find /persist/vault/containerd -path '*usr/local/bin/volverify' 2>/dev/null | head -1) +[ -n "$VV" ] || { echo "DEVSIDE=no-volverify"; exit 0; } +F=$(ls /persist/vault/volumes-kvm/*.raw /persist/clear/volumes-kvm/*.raw 2>/dev/null | head -1) +[ -n "$F" ] || { echo "DEVSIDE=no-relocated-volume"; exit 0; } +echo "DEVSIDE-FILE=$F size=$(stat -c %s "$F")" +# Read the filesystem before anything mounts it: a mount replays the journal and can +# repair the very damage being measured. +e2fsck -fn "$F" > /tmp/dsfsck.out 2>&1 +echo "DEVSIDE-FSCK-RC=$?" +sed -n '1,40p' /tmp/dsfsck.out +mkdir -p /tmp/dsmnt +L=$(losetup -r -f --show "$F" 2>&1) || { echo "DEVSIDE=losetup-failed:$L"; exit 0; } +if mount -o ro,noload "$L" /tmp/dsmnt 2>&1; then + # Redirect rather than pipe, so the status is volverify's and not the pager's. + "$VV" verify --dir /tmp/dsmnt --seed "$1" --ops "$2" \ + --block-size 4096 --max-blocks "$3" --expect-committed "$4" > /tmp/dsvv.out 2>&1 + echo "DEVSIDE-VERIFY-RC=$?" + grep -aE 'committed=|verify:' /tmp/dsvv.out | tail -4 + umount /tmp/dsmnt || true +else + echo "DEVSIDE=mount-failed" +fi +losetup -d "$L" || true +rm -rf /tmp/dsmnt /tmp/dsvv.out /tmp/dsfsck.out` + +// verifyRelocatedVolumeOnDevice records the volume verdict without involving the app, +// Longhorn, CDI or a completed PVC import — see devsideVerifyScript. +// +// It is deliberately non-fatal about not finding anything to check: on a clear-volume +// or ZFS layout the file may not be where this looks, and that must not fail a run +// whose app-side verdict is still to come. A volume it does read and find corrupt is +// a different matter and fails the test, since that is the finding being hunted. +func verifyRelocatedVolumeOnDevice(t Gomega, device *evetest.EdgeDevice, dataVolMiB uint32, + seed, ops uint64, maxBlocks uint64, expectCommitted int) { + log := evetest.Logger() + out, err := runOnEVEScript(device, devsideVerifyScript, 15*time.Minute, + fmt.Sprintf("%d", seed), fmt.Sprintf("%d", ops), + fmt.Sprintf("%d", maxBlocks), fmt.Sprintf("%d", expectCommitted)) + if err != nil { + log.Errorf("device-side volume verify could not run:\n%s", out) + return + } + log.Infof("device-side volume verify:\n%s", strings.TrimSpace(out)) + + if i := strings.Index(out, "DEVSIDE="); i >= 0 { + var reason string + fmt.Sscanf(out[i:], "DEVSIDE=%s", &reason) + log.Errorf("device-side volume verify skipped: %s", reason) + return + } + + fsckRC, verifyRC := -1, -1 + if i := strings.Index(out, "DEVSIDE-FSCK-RC="); i >= 0 { + fmt.Sscanf(out[i:], "DEVSIDE-FSCK-RC=%d", &fsckRC) + } + if i := strings.Index(out, "DEVSIDE-VERIFY-RC="); i >= 0 { + fmt.Sscanf(out[i:], "DEVSIDE-VERIFY-RC=%d", &verifyRC) + } + evetest.Logger().Infof("[DEVICE-VOLUME-OUTCOME] datavolMiB=%d app-independent=true "+ + "fsck-rc=%d structural-damage=%t verify-rc=%d verify=[%s]", + dataVolMiB, fsckRC, fsckFoundStructuralDamage(out), verifyRC, volverifySummary(out)) + evetest.Checkpoint("devside-volume-verified") + + presentCorrupt := reportField(out, "present-corrupt") + t.Expect(presentCorrupt).To(BeNumerically("<=", 0), + "the relocated pre-conversion volume is present but CORRUPT (present-corrupt=%d) — "+ + "the interrupted shrink tore data the filesystem check cannot see:\n%s", + presentCorrupt, out) +} + +// volverifySummary pulls volverify's one-line tally out of the device-side output so +// the outcome record carries the counts rather than the whole transcript. +func volverifySummary(out string) string { + for _, line := range strings.Split(out, "\n") { + if strings.Contains(line, "committed=") && strings.Contains(line, "ok=") { + return strings.TrimSpace(line) + } + } + return "no-summary" +} + // writeVolverifyPattern fills the app's data volume with the deterministic pattern // and returns the committed op index the writer reached. It requires the volume to // be mounted at its MountDir, which the runx shim does on EVE-kvm. From 57e4f5d6988ebc10f5fe5bf38d0c9d360538cd1b Mon Sep 17 00:00:00 2001 From: eriknordmark Date: Thu, 30 Jul 2026 16:53:46 -0700 Subject: [PATCH 23/29] evetest: wait for the volume relocation, don't race it The device reports itself on the target before upgradeconverter's post-vault phase has necessarily moved the carried-over volumes into volumes-kvm, so sampling for the file immediately after the conversion is a race. Losing it looks like success: a missing file is treated as nothing to verify, and the iteration silently produces no volume verdict at all. One run sampled sixteen seconds after the device came up, skipped, and only then did volumemgr start logging import errors against the very file that was reported absent. Poll for the file for up to ten minutes instead, and report how long the wait took so a slow relocation is visible rather than inferred. Also read clear volumes from clear/volumes, where kvmMigratedSourcePath looks for them; the previous path had them under a -kvm sibling that only encrypted volumes use. Co-Authored-By: Claude Opus 5 Signed-off-by: eriknordmark --- evetest/tests/resize/appvolshrink_test.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/evetest/tests/resize/appvolshrink_test.go b/evetest/tests/resize/appvolshrink_test.go index 90c8830dd19..f0293bf7b32 100644 --- a/evetest/tests/resize/appvolshrink_test.go +++ b/evetest/tests/resize/appvolshrink_test.go @@ -505,8 +505,20 @@ func TestAppVolumeShrinkCorruption(test *testing.T) { const devsideVerifyScript = `set -u VV=$(find /persist/vault/containerd -path '*usr/local/bin/volverify' 2>/dev/null | head -1) [ -n "$VV" ] || { echo "DEVSIDE=no-volverify"; exit 0; } -F=$(ls /persist/vault/volumes-kvm/*.raw /persist/clear/volumes-kvm/*.raw 2>/dev/null | head -1) +# Wait for the relocation rather than sampling once. The device reports itself on the +# target before upgradeconverter's post-vault phase has necessarily moved the volumes, +# so an immediate check races it and loses — silently, since a missing file is treated +# as "nothing to verify". Encrypted volumes land in vault/volumes-kvm; clear ones stay +# in clear/volumes, which is what kvmMigratedSourcePath reads. +F="" +i=0 +while [ "$i" -lt 60 ]; do + F=$(ls /persist/vault/volumes-kvm/*.raw /persist/clear/volumes/*.raw 2>/dev/null | head -1) + [ -n "$F" ] && break + i=$((i + 1)); sleep 10 +done [ -n "$F" ] || { echo "DEVSIDE=no-relocated-volume"; exit 0; } +echo "DEVSIDE-WAITED=${i}0s" echo "DEVSIDE-FILE=$F size=$(stat -c %s "$F")" # Read the filesystem before anything mounts it: a mount replays the journal and can # repair the very damage being measured. From 8a37929e749751683b69d70fdb302039890cb2ec Mon Sep 17 00:00:00 2001 From: eriknordmark Date: Thu, 30 Jul 2026 17:13:31 -0700 Subject: [PATCH 24/29] evetest: wait for volverify too, not just the volume The device-side check looked for the volverify binary before entering the wait loop, so a containerd snapshot that was not yet unpacked made it exit early and skip the wait it had just been given. A run that ended up with both artifacts present recorded no volume verdict, reporting the binary as missing. Wait for the binary and the relocated volume in one loop, and report the elapsed wait either way so a slow appearance is visible rather than inferred from a skip. Validated on hardware: four consecutive conversions recorded a complete app-independent verdict with nothing skipped, three of them alongside a working app-side check that agreed with it. Each reported a zero-length wait, so the common case costs nothing; the wait still has not been observed absorbing a slow relocation, which is the case it exists for. Co-Authored-By: Claude Opus 5 --- evetest/tests/resize/appvolshrink_test.go | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/evetest/tests/resize/appvolshrink_test.go b/evetest/tests/resize/appvolshrink_test.go index f0293bf7b32..5a33dbd0929 100644 --- a/evetest/tests/resize/appvolshrink_test.go +++ b/evetest/tests/resize/appvolshrink_test.go @@ -503,22 +503,24 @@ func TestAppVolumeShrinkCorruption(test *testing.T) { // image, so the same statically-linked binary the app would run is on disk. Args: // seed, ops, max-blocks, expect-committed. const devsideVerifyScript = `set -u -VV=$(find /persist/vault/containerd -path '*usr/local/bin/volverify' 2>/dev/null | head -1) -[ -n "$VV" ] || { echo "DEVSIDE=no-volverify"; exit 0; } -# Wait for the relocation rather than sampling once. The device reports itself on the -# target before upgradeconverter's post-vault phase has necessarily moved the volumes, -# so an immediate check races it and loses — silently, since a missing file is treated -# as "nothing to verify". Encrypted volumes land in vault/volumes-kvm; clear ones stay -# in clear/volumes, which is what kvmMigratedSourcePath reads. +# Both artifacts appear on their own schedule after the conversion: containerd unpacks +# the app image, and the post-vault phase of upgradeconverter moves the volumes. The +# device reports itself on the target before either is guaranteed, so wait for both +# together — checking one first and exiting early skips the wait on the other. +# Encrypted volumes land in vault/volumes-kvm; clear ones stay in clear/volumes, which +# is where kvmMigratedSourcePath looks for them. +VV="" F="" i=0 while [ "$i" -lt 60 ]; do - F=$(ls /persist/vault/volumes-kvm/*.raw /persist/clear/volumes/*.raw 2>/dev/null | head -1) - [ -n "$F" ] && break + [ -n "$VV" ] || VV=$(find /persist/vault/containerd -path '*usr/local/bin/volverify' 2>/dev/null | head -1) + [ -n "$F" ] || F=$(ls /persist/vault/volumes-kvm/*.raw /persist/clear/volumes/*.raw 2>/dev/null | head -1) + [ -n "$VV" ] && [ -n "$F" ] && break i=$((i + 1)); sleep 10 done -[ -n "$F" ] || { echo "DEVSIDE=no-relocated-volume"; exit 0; } echo "DEVSIDE-WAITED=${i}0s" +[ -n "$VV" ] || { echo "DEVSIDE=no-volverify"; exit 0; } +[ -n "$F" ] || { echo "DEVSIDE=no-relocated-volume"; exit 0; } echo "DEVSIDE-FILE=$F size=$(stat -c %s "$F")" # Read the filesystem before anything mounts it: a mount replays the journal and can # repair the very damage being measured. From 2c87d08e19ed7e63f4a238e6a4773c52fceb8a28 Mon Sep 17 00:00:00 2001 From: eriknordmark Date: Mon, 3 Aug 2026 22:58:00 +0200 Subject: [PATCH 25/29] evetest: make a shrink iteration measure something MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A corruption soak only earns a row if the iteration actually examined a volume, and three things stopped that happening. EVE's post-resize check hashes each volume against a pre-resize manifest and deletes any that mismatches, so the torn volumes this test hunts were gone before it could look at them; a marker now has them quarantined instead, and the device-side verify reads the quarantined copy exactly like an intact one while recording that EVE had already condemned it. Finding nothing to verify is now reported instead of passing silently, and a missing filler directory fails the trim rather than continuing. The filler also stops short of the data volume's size and lives under /persist/log: filling to the requested peak and then writing the volume into what little remained drove the device into low-disk maintenance mode and a reboot, and onboot.sh removes /persist/tmp on every boot, voiding the placement the fill exists to create. DEVSIDE_ONLY ends the test once the volume verdict is in, skipping the app-side checks that account for most of a run's wall time. At data-volume sizes where the post-conversion app reliably wedges those checks re-derive a known failure, so a soak collects far more verdicts per day without them — at the cost of the wedge diagnostics, which nothing else produces. Signed-off-by: eriknordmark Co-Authored-By: Claude Opus 5 (1M context) --- evetest/tests/resize/appvolshrink_test.go | 132 +++++++++++++++++++--- 1 file changed, 118 insertions(+), 14 deletions(-) diff --git a/evetest/tests/resize/appvolshrink_test.go b/evetest/tests/resize/appvolshrink_test.go index 5a33dbd0929..51ae5240d14 100644 --- a/evetest/tests/resize/appvolshrink_test.go +++ b/evetest/tests/resize/appvolshrink_test.go @@ -47,6 +47,7 @@ const ( fillPeakPctParamKey = "FILL_PEAK_PCT" fillKeepGiBParamKey = "FILL_KEEP_GIB" + devsideOnlyParamKey = "DEVSIDE_ONLY" // Fill /persist to this percentage before the app is deployed, then trim back // to this many GiB once its volume is written. The peak has to sit well above @@ -137,6 +138,14 @@ func TestAppVolumeShrinkCorruption(test *testing.T) { Default: "15", }, }, + evetest.TestParameterDefinition{ + Key: devsideOnlyParamKey, + DefaultValue: false, + Description: evetest.TestParameterDescription{ + Summary: "Stop once the volume verdict is taken, skipping the app-side checks", + Default: "false", + }, + }, evetest.TestParameterDefinition{ Key: seedParamKey, DefaultValue: uint64(20260723), @@ -309,7 +318,7 @@ func TestAppVolumeShrinkCorruption(test *testing.T) { // second, and the watchdog only ever interrupts the grow. if fillPeakPct > 0 { log.Infof("filling /persist to %d%% so the app's volume lands in the shrink's evacuation zone", fillPeakPct) - fillPersistToPct(t, device, int(fillPeakPct)) + fillPersistToPct(t, device, int(fillPeakPct), dataVolMiB) evetest.Checkpoint("persist-filled") } @@ -369,6 +378,17 @@ func TestAppVolumeShrinkCorruption(test *testing.T) { capturePersist(device, "pre-conversion EVE-kvm (volume filled)") evetest.Checkpoint("volume-filled") + // EVE's post-resize check hashes each volume against a pre-resize manifest and + // DELETES any that mismatches, so a torn volume is gone before this test can look + // at it — which is why nine 8 GiB corruptions were recorded as "nothing to + // measure". The marker makes baseosmgr quarantine it as .corrupt instead, + // leaving the damaged bytes for fsck and volverify to characterize. + markerOut, _, markerErr := device.RunShellScript( + `eve exec pillar touch /persist/volmanifest-keep-corrupt`, eveShellTimeout, 0) + t.Expect(markerErr).NotTo(HaveOccurred(), + "could not arm the corrupt-volume quarantine marker:\n%s", markerOut) + log.Infof("armed /persist/volmanifest-keep-corrupt so a torn volume is kept, not deleted") + log.Infof("kvm→k conversion: upgrading to %s (%s) — triggers the offline shrink+grow", convVersion, targetHypervisor) conversionOK := false defer func() { @@ -400,6 +420,21 @@ func TestAppVolumeShrinkCorruption(test *testing.T) { // post-conversion app wedge, which has cost every verdict it has hit. verifyRelocatedVolumeOnDevice(t, device, dataVolMiB, seed, ops, maxBlocks, writtenCommitted) + // The volume verdict is complete here, about 32 minutes into a run whose + // remaining 48 minutes are the app-side checks. At data-volume sizes where the + // post-conversion app reliably wedges, those 48 minutes re-derive a known + // failure, so a corruption soak can stop here and collect roughly two and a + // half times as many volume verdicts per day. + // + // This forfeits the wedge diagnostics, which are produced by the app-failure + // path and nowhere else: leave it off when the run is meant to investigate why + // the app does not come back. + if evetest.GetTestParameter[bool](devsideOnlyParamKey) { + log.Infof("DEVSIDE_ONLY: volume verdict taken; skipping the app-side checks") + evetest.Checkpoint("devside-only-complete") + return + } + if targetHypervisor == evetest.HypervisorKubevirt { log.Infof("(a) waiting for k3s node ready") if !isK3sReady(device.GetClusterInfo()) { @@ -514,11 +549,31 @@ F="" i=0 while [ "$i" -lt 60 ]; do [ -n "$VV" ] || VV=$(find /persist/vault/containerd -path '*usr/local/bin/volverify' 2>/dev/null | head -1) - [ -n "$F" ] || F=$(ls /persist/vault/volumes-kvm/*.raw /persist/clear/volumes/*.raw 2>/dev/null | head -1) + # A *.raw.corrupt is EVE's post-resize hash check having caught the interrupted + # shrink tearing this volume, quarantined instead of deleted (see the marker this + # test drops before the conversion). That is the finding, not a missing volume: + # verify it exactly like an intact one, so fsck and volverify can say whether the + # damage is structurally visible or silent. + [ -n "$F" ] || F=$(ls /persist/vault/volumes-kvm/*.raw /persist/clear/volumes/*.raw \ + /persist/vault/volumes-kvm/*.raw.corrupt /persist/clear/volumes/*.raw.corrupt \ + 2>/dev/null | head -1) [ -n "$VV" ] && [ -n "$F" ] && break i=$((i + 1)); sleep 10 done echo "DEVSIDE-WAITED=${i}0s" +# Giving up is only actionable with the layout that defeated the search: an 8 GiB run +# put ~8 GiB somewhere under /persist/vault inside the wait window, yet matched nothing +# here, and du -d1 could not say where. List the candidates so the next occurrence +# reports the actual path instead of only that it is not the expected one. +if [ -z "$VV" ] || [ -z "$F" ]; then + echo "DEVSIDE-LAYOUT:" + for d in /persist/vault/volumes-kvm /persist/vault/volumes /persist/clear/volumes; do + echo " $d:" + ls -l "$d" 2>&1 + done + echo " largest under /persist/vault:" + du -x -B1 -d2 /persist/vault 2>/dev/null | sort -rn | head -12 +fi [ -n "$VV" ] || { echo "DEVSIDE=no-volverify"; exit 0; } [ -n "$F" ] || { echo "DEVSIDE=no-relocated-volume"; exit 0; } echo "DEVSIDE-FILE=$F size=$(stat -c %s "$F")" @@ -545,10 +600,13 @@ rm -rf /tmp/dsmnt /tmp/dsvv.out /tmp/dsfsck.out` // verifyRelocatedVolumeOnDevice records the volume verdict without involving the app, // Longhorn, CDI or a completed PVC import — see devsideVerifyScript. // -// It is deliberately non-fatal about not finding anything to check: on a clear-volume -// or ZFS layout the file may not be where this looks, and that must not fail a run -// whose app-side verdict is still to come. A volume it does read and find corrupt is -// a different matter and fails the test, since that is the finding being hunted. +// Not finding anything to check is non-fatal ONLY when the app-side verdict is still to +// come: on a clear-volume or ZFS layout the file may not be where this looks. Under +// DEVSIDE_ONLY there is no second verdict, so the same silence means the iteration +// measured nothing — and a run that measures nothing must not report PASS, or a soak +// banks clean-looking rows that never examined a volume. +// A volume it does read and find corrupt fails the test either way; that is the finding +// being hunted. func verifyRelocatedVolumeOnDevice(t Gomega, device *evetest.EdgeDevice, dataVolMiB uint32, seed, ops uint64, maxBlocks uint64, expectCommitted int) { log := evetest.Logger() @@ -565,6 +623,27 @@ func verifyRelocatedVolumeOnDevice(t Gomega, device *evetest.EdgeDevice, dataVol var reason string fmt.Sscanf(out[i:], "DEVSIDE=%s", &reason) log.Errorf("device-side volume verify skipped: %s", reason) + // The volume was present and written before the conversion, so its absence + // afterwards is upgradeconverter's account to give: stageKvmVolumes logs + // "relocated N, skipped M" per run, plus a line for each entry it declined to + // move and for the sentinel short-circuit. Without this the layout dump can + // only say the volume is not there, not whether it was moved elsewhere, + // skipped, or never seen. + for _, p := range []struct{ what, pipeline string }{ + {"upgradeconverter volume relocation", "grep -a stageKvmVolumes | tail -20"}, + {"upgradeconverter kube entry", "grep -a relocateKvmVolumesForKube | tail -10"}, + // stageKvmVolumes reports relocating everything it finds, so the volume is + // already gone when it runs. Grep for the file itself rather than any one + // agent's wording: whoever removed it had to name it. + {"every mention of a volume file", `grep -a "#0.raw" | tail -40`}, + {"volumemgr delete path", `grep -a volumemgr | grep -aiE "delete|destroy|purge|remov" | tail -30`}, + } { + o, _, e := device.RunShellScript(newlogProbe(p.pipeline), 90*time.Second, 0) + log.Errorf("[devside-miss] %s:\n%s(err=%v)", p.what, strings.TrimSpace(o), e) + } + t.Expect(evetest.GetTestParameter[bool](devsideOnlyParamKey)).To(BeFalse(), + "DEVSIDE_ONLY run banked no volume verdict (%s): this iteration measured "+ + "nothing about shrink corruption:\n%s", reason, out) return } @@ -575,9 +654,15 @@ func verifyRelocatedVolumeOnDevice(t Gomega, device *evetest.EdgeDevice, dataVol if i := strings.Index(out, "DEVSIDE-VERIFY-RC="); i >= 0 { fmt.Sscanf(out[i:], "DEVSIDE-VERIFY-RC=%d", &verifyRC) } + // eve-detected records whether EVE's own post-resize hash check had already + // condemned this volume: the file it hands us is then the quarantined copy. A + // clean volverify on such a file would mean the two checks disagree, so the + // distinction has to survive into the row rather than being inferred from a path. + eveDetected := strings.Contains(out, ".raw"+".corrupt") evetest.Logger().Infof("[DEVICE-VOLUME-OUTCOME] datavolMiB=%d app-independent=true "+ - "fsck-rc=%d structural-damage=%t verify-rc=%d verify=[%s]", - dataVolMiB, fsckRC, fsckFoundStructuralDamage(out), verifyRC, volverifySummary(out)) + "eve-detected-corrupt=%t fsck-rc=%d structural-damage=%t verify-rc=%d verify=[%s]", + dataVolMiB, eveDetected, fsckRC, fsckFoundStructuralDamage(out), verifyRC, + volverifySummary(out)) evetest.Checkpoint("devside-volume-verified") presentCorrupt := reportField(out, "present-corrupt") @@ -869,13 +954,28 @@ func runOnEVEScript(device *evetest.EdgeDevice, script string, // store the blocks sparsely, and the relocation would then read and write nothing, // leaving the shrink as fast as it is on an empty filesystem — which is the whole // problem this is here to fix. -func fillPersistToPct(t Gomega, device *evetest.EdgeDevice, pct int) { +// reserveMiB is the size of the data volume written next: the fill stops that much +// short of pct, so the peak INCLUDING the volume is the requested percentage. Filling +// all the way to pct and then writing the volume into what little is left collapses the +// allowance volumemgr grants apps — the filler counts as dom0 usage, which is subtracted +// from it — and at RemainingSpace 0 the device enters +// MAINTENANCE_MODE_REASON_LOW_DISK_SPACE and reboots. Observed 3/3 at 8 GiB (peak 95%, +// 3.0 GiB free) and 0/3 at 2 GiB, where the volume still fit in the slack. +func fillPersistToPct(t Gomega, device *evetest.EdgeDevice, pct int, reserveMiB uint32) { + // The filler lives under /persist/log, NOT /persist/tmp: onboot.sh removes + // /persist/tmp unconditionally on every boot (no threshold, no df), so a reboot + // anywhere between the fill and the trim silently voids the placement this step + // exists to create. /persist/log survives a boot and is still the first entry in + // onboot.sh's PERSIST_CLEANUPS list, so a device that genuinely runs out of space + // reclaims it instead of wedging. const script = `set -u PCT=$1 -DIR=/persist/tmp/stressfill +RESERVE_KB=$(( $2 * 1024 )) +DIR=/persist/log/stressfill rm -rf "$DIR"; mkdir -p "$DIR" cap=$(df -k /persist | tail -1 | awk '{print $2}') -want=$(( cap * PCT / 100 )) +want=$(( cap * PCT / 100 - RESERVE_KB )) +[ "$want" -lt 0 ] && want=0 n=0 while [ "$(df -k /persist | tail -1 | awk '{print $3}')" -lt "$want" ]; do f=$(printf "%s/%06d" "$DIR" "$n") @@ -884,7 +984,8 @@ while [ "$(df -k /persist | tail -1 | awk '{print $3}')" -lt "$want" ]; do done sync echo "FILLED files=$n $(df -h /persist | tail -1)"` - out, err := runOnEVEScript(device, script, 40*time.Minute, fmt.Sprintf("%d", pct)) + out, err := runOnEVEScript(device, script, 40*time.Minute, + fmt.Sprintf("%d", pct), fmt.Sprintf("%d", reserveMiB)) t.Expect(err).NotTo(HaveOccurred(), "filling /persist failed:\n%s", out) t.Expect(out).To(ContainSubstring("FILLED"), "fill did not report completion:\n%s", out) evetest.Logger().Infof("filled /persist to ~%d%%: %s", pct, strings.TrimSpace(out)) @@ -900,8 +1001,11 @@ echo "FILLED files=$n $(df -h /persist | tail -1)"` func trimPersistToGiB(t Gomega, device *evetest.EdgeDevice, keepGiB int) { const script = `set -u KEEP_KB=$(( $1 * 1024 * 1024 )) -DIR=/persist/tmp/stressfill -[ -d "$DIR" ] || { echo "TRIMMED no-fill-dir"; exit 0; } +DIR=/persist/log/stressfill +# A missing filler means the volume was not placed in the evacuation zone, so the +# iteration cannot say anything about shrink safety. Fail rather than continue: the +# silent version of this produced three clean-looking rows that had measured nothing. +[ -d "$DIR" ] || { echo "TRIM-FAILED no-fill-dir"; exit 1; } for f in $(ls -1 "$DIR" 2>/dev/null | sort); do [ "$(df -k /persist | tail -1 | awk '{print $3}')" -le "$KEEP_KB" ] && break rm -f "$DIR/$f" From c1388e0fb20cda94903b542bbec9c96aaab94d91 Mon Sep 17 00:00:00 2001 From: eriknordmark Date: Mon, 3 Aug 2026 22:58:01 +0200 Subject: [PATCH 26/29] evetest: capture the stalling CDI upload server MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the post-conversion app never gets its volume, the failure diagnostics named the CDI upload controller but never the upload server — the process that receives the transfer, and the one observed Ready and idle while the import never completes. It is captured now, including the previous container's log, because on smaller volumes the upload pod has already terminated by the time this path runs and the live fetch came back empty. Alongside it: namespace warning events, which outlive the pods that caused them and are where an attach failure surfaces; the per-PVC upload progress, which distinguishes a transfer that never started from one that stalled partway; and the Longhorn volume state underneath, since an upload can idle because the volume was never brought up on the uploader's node. The RolloutDiskToPVC window grew because retry noise had been filling it exactly, and the recovery actions moved to their own probe so that noise cannot crowd them out. The persist snapshot also runs the conversion pre-flight now, so a declined conversion comes with the sizes and the policy floor behind its one-line reason rather than only the reason. Signed-off-by: eriknordmark Co-Authored-By: Claude Opus 5 (1M context) --- evetest/tests/resize/kvmtok_resize_test.go | 41 +++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/evetest/tests/resize/kvmtok_resize_test.go b/evetest/tests/resize/kvmtok_resize_test.go index e04a022efde..ccf0e7b825d 100644 --- a/evetest/tests/resize/kvmtok_resize_test.go +++ b/evetest/tests/resize/kvmtok_resize_test.go @@ -435,7 +435,33 @@ func captureAppNetFailure(device *evetest.EdgeDevice, appUUID uuid.UUID) { {"longhorn StorageClass (replicas/config)", `eve exec kube kubectl get sc longhorn -o yaml 2>/dev/null || echo none`}, {"longhorn version (manager ds image)", `eve exec kube kubectl -n longhorn-system get ds longhorn-manager -o wide 2>/dev/null || echo none`}, {"CDI upload-controller log", `eve exec kube kubectl -n cdi logs deployment/cdi-deployment --tail=60 2>/dev/null || echo none`}, - {"volumemgr RolloutDiskToPVC / V5 signature (newlog)", newlogProbe(`grep -aiE "RolloutDiskToPVC|retryFailedClusterVolumeCreate|terminating:true|local-path" | tail -30`)}, + // The upload SERVER, not the controller: this is the process that receives the + // transfer, and the one observed Ready and idle while the upload never + // completes. Its own logs were never captured, so every wedge so far has been + // diagnosed without looking at the component that stalls. + // --previous as well as the live log: on a 512 MiB reproduction the upload pod + // reported Succeeded and was gone by the time this ran, so the live fetch + // returned nothing and the component's own account of the transfer was lost. + {"CDI upload-server pod log (the stalling component)", `eve exec kube kubectl -n eve-kube-app logs -l cdi.kubevirt.io=cdi-upload-server --tail=120 --prefix 2>/dev/null; eve exec kube kubectl -n eve-kube-app logs -l cdi.kubevirt.io=cdi-upload-server --tail=120 --prefix --previous 2>/dev/null; echo "--- end upload-server logs"`}, + // Warning events outlive the pods that caused them, so they survive an upload + // pod that terminated. This is where an attach failure shows up — the observed + // end state is a Bound claim whose Longhorn volume is detached, which no + // pod-scoped probe explains. + {"eve-kube-app warning events (attach/mount failures)", `eve exec kube kubectl -n eve-kube-app get events --field-selector type=Warning --sort-by=.lastTimestamp 2>/dev/null | tail -40 || echo none`}, + {"longhorn-system warning events", `eve exec kube kubectl -n longhorn-system get events --field-selector type=Warning --sort-by=.lastTimestamp 2>/dev/null | tail -30 || echo none`}, + // How far the transfer got. Stuck at 0% means it never started (proxy/route), + // stuck partway means it began and stalled — different faults entirely. + {"CDI upload progress annotations", `eve exec kube kubectl -n eve-kube-app get pvc -o custom-columns=NAME:.metadata.name,PHASE:.status.phase,PROGRESS:'.metadata.annotations.cdi\.kubevirt\.io/storage\.pod\.progress',PODPHASE:'.metadata.annotations.cdi\.kubevirt\.io/storage\.pod\.phase',RUNNING:'.metadata.annotations.cdi\.kubevirt\.io/storage\.condition\.running' 2>/dev/null || echo none`}, + // Whether the target volume is actually attachable underneath: an upload can + // idle because Longhorn never brought the volume up on the uploader's node. + {"longhorn volume detail for the upload target", `eve exec kube kubectl -n longhorn-system get volumes.longhorn.io -o custom-columns=NAME:.metadata.name,STATE:.status.state,ROBUST:.status.robustness,NODE:.status.currentNodeID,SIZE:.spec.size 2>/dev/null || echo none`}, + // A wedged import re-drives this call for tens of minutes, so 30 lines is all + // retry noise: on every wedged run of 2026-07-31 the window came back exactly + // full, crowding out the one line that said whether the recovery below had + // run. An absent line then cannot be told from a truncated one. + {"volumemgr RolloutDiskToPVC / V5 signature (newlog)", newlogProbe(`grep -aiE "RolloutDiskToPVC|retryFailedClusterVolumeCreate|terminating:true|local-path" | tail -200`)}, + // Recovery actions get their own probe so retry spam cannot bury them. + {"PVC recovery actions taken (scratch/target delete)", newlogProbe(`grep -aiE "deleted wedged scratch PVC|could not delete wedged scratch|deleted wedged target PVC" | tail -40`)}, {"MOUNT-WEDGE-RECOVERY (detector fired?)", newlogProbe(`grep -ai MOUNT-WEDGE-RECOVERY | tail -10`)}, {"kubevirt vmi -A", `eve exec kube kubectl get vmi -A -o wide 2>/dev/null || echo none`}, } @@ -462,6 +488,19 @@ func capturePersist(device *evetest.EdgeDevice, label string) { {"longhorn storage settings", `eve exec kube kubectl -n longhorn-system get settings.longhorn.io -o custom-columns=NAME:.metadata.name,VALUE:.value 2>/dev/null | grep -aiE "NAME|reserved|over-provisioning|minimal-available|soft-anti-affinity" || echo none`}, {"eve-kube-app pvc (requested sizes)", `eve exec kube kubectl -n eve-kube-app get pvc 2>/dev/null || echo none`}, } + // The same pre-flight baseosmgr runs to decide the conversion. When it declines, + // the controller carries only the one-line reason ("persist is too full to free + // the needed space"); the JSON carries the numbers behind it — needed/target/used + // bytes, the resize2fs floor estimate and the maxFullPercent policy — which is the + // difference between knowing that it declined and knowing why. + if disk, err := bootDiskPath(device); err == nil { + probes = append(probes, struct{ what, script string }{ + "storage-resizer check (conversion pre-flight)", + "eve exec pillar /usr/bin/storage-resizer check --disk " + disk + " --json 2>&1 || echo none", + }) + } else { + log.Errorf("[persist:%s] storage-resizer check: could not resolve boot disk: %v", label, err) + } for _, p := range probes { out, errOut, err := device.RunShellScript(p.script, 60*time.Second, 0) log.Errorf("[persist:%s] %s:\n%s%s(err=%v)", label, p.what, strings.TrimSpace(out), errOut, err) From f580a9c322dbf3161e737b659fe60812b3f034f2 Mon Sep 17 00:00:00 2001 From: eriknordmark Date: Tue, 4 Aug 2026 20:46:23 +0200 Subject: [PATCH 27/29] evetest: fail the shrink test on every volume verdict MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The device-side check asserted only present-corrupt, so a volume that lost committed files outright, resurrected deleted ones, or produced no verdict tuple at all still reported PASS — an absent count reads as -1 and satisfied the old bound, banking a clean-looking row for an iteration that measured nothing. volverify's exit status is now the catch-all, a missing count is a failure rather than a pass, and lost/resurrected fail alongside corruption. Orphaned stays benign: recovery into lost+found self-heals. Quarantined volumes are also selected ahead of intact siblings, since a *.raw.corrupt is the finding, and the candidate count is reported so a selection that discarded one is visible. Signed-off-by: eriknordmark Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01HevHMgqe74rWRzZsEYD8Fb --- evetest/tests/resize/appvolshrink_test.go | 31 +++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/evetest/tests/resize/appvolshrink_test.go b/evetest/tests/resize/appvolshrink_test.go index 51ae5240d14..cce4c5cd1b9 100644 --- a/evetest/tests/resize/appvolshrink_test.go +++ b/evetest/tests/resize/appvolshrink_test.go @@ -554,8 +554,11 @@ while [ "$i" -lt 60 ]; do # test drops before the conversion). That is the finding, not a missing volume: # verify it exactly like an intact one, so fsck and volverify can say whether the # damage is structurally visible or silent. - [ -n "$F" ] || F=$(ls /persist/vault/volumes-kvm/*.raw /persist/clear/volumes/*.raw \ - /persist/vault/volumes-kvm/*.raw.corrupt /persist/clear/volumes/*.raw.corrupt \ + # Quarantined copies are listed first: a *.raw.corrupt IS the finding, so picking + # an intact sibling ahead of it would report a clean verdict for a conversion that + # had already been caught tearing a volume. + [ -n "$F" ] || F=$(ls /persist/vault/volumes-kvm/*.raw.corrupt /persist/clear/volumes/*.raw.corrupt \ + /persist/vault/volumes-kvm/*.raw /persist/clear/volumes/*.raw \ 2>/dev/null | head -1) [ -n "$VV" ] && [ -n "$F" ] && break i=$((i + 1)); sleep 10 @@ -577,6 +580,11 @@ fi [ -n "$VV" ] || { echo "DEVSIDE=no-volverify"; exit 0; } [ -n "$F" ] || { echo "DEVSIDE=no-relocated-volume"; exit 0; } echo "DEVSIDE-FILE=$F size=$(stat -c %s "$F")" +# Whether the selection above had a choice at all: >1 means head -1 discarded a +# candidate and the verdict may describe the wrong volume. +echo "DEVSIDE-CANDIDATES=$(ls /persist/vault/volumes-kvm/*.raw.corrupt \ + /persist/clear/volumes/*.raw.corrupt /persist/vault/volumes-kvm/*.raw \ + /persist/clear/volumes/*.raw 2>/dev/null | wc -l)" # Read the filesystem before anything mounts it: a mount replays the journal and can # repair the very damage being measured. e2fsck -fn "$F" > /tmp/dsfsck.out 2>&1 @@ -665,11 +673,30 @@ func verifyRelocatedVolumeOnDevice(t Gomega, device *evetest.EdgeDevice, dataVol volverifySummary(out)) evetest.Checkpoint("devside-volume-verified") + // volverify exits non-zero on ANY anomaly, so its status is the catch-all: a + // verdict class not named individually below still fails the run. + t.Expect(verifyRC).To(BeNumerically("==", 0), + "volverify rejected the relocated volume (DEVSIDE-VERIFY-RC=%d):\n%s", verifyRC, out) + presentCorrupt := reportField(out, "present-corrupt") + // An absent field reads as -1, which would satisfy the bound below and bank a + // clean-looking row for a run that measured nothing. + t.Expect(presentCorrupt).To(BeNumerically(">=", 0), + "volverify produced no present-corrupt count, so this iteration measured "+ + "nothing about shrink corruption:\n%s", out) t.Expect(presentCorrupt).To(BeNumerically("<=", 0), "the relocated pre-conversion volume is present but CORRUPT (present-corrupt=%d) — "+ "the interrupted shrink tore data the filesystem check cannot see:\n%s", presentCorrupt, out) + // Losing a committed file outright, or resurrecting a committed-deleted one, is + // data loss just as much as torn content. Orphaned is excluded: recovery into + // lost+found self-heals to a blank/content-tree recreate. + for _, key := range []string{"lost", "resurrected"} { + n := reportField(out, key) + t.Expect(n).To(BeNumerically("<=", 0), + "the relocated pre-conversion volume lost committed data (%s=%d):\n%s", + key, n, out) + } } // volverifySummary pulls volverify's one-line tally out of the device-side output so From 1729bf1cefd1ee059c720a09bad7c7d5d738a63b Mon Sep 17 00:00:00 2001 From: eriknordmark Date: Wed, 5 Aug 2026 10:16:40 +0200 Subject: [PATCH 28/29] evetest: list params through a test's helper list-tests reads the parameters a test declares by walking that test function's own body, so a test which delegates to a shared helper reported no parameters at all. Calls to package-local functions are followed now, with a visited set so recursion terminates. Tests that declare their parameters inline list exactly as before. Signed-off-by: eriknordmark Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_0156usNe9ABT9Y5ofQH3na8w --- evetest/cmd/list-tests/main.go | 54 ++++++++++++++++++++++++++-------- 1 file changed, 42 insertions(+), 12 deletions(-) diff --git a/evetest/cmd/list-tests/main.go b/evetest/cmd/list-tests/main.go index 77b9f5c26d0..f7c51d08616 100644 --- a/evetest/cmd/list-tests/main.go +++ b/evetest/cmd/list-tests/main.go @@ -72,6 +72,7 @@ type pkgContext struct { constStrings map[string]string // const name → String() return value (for variant display) allConsts map[string]string // all simple const name → literal value (for variant display) varParams map[string]paramInfo + funcDecls map[string]*ast.FuncDecl // package-local func name → declaration } func main() { @@ -244,6 +245,17 @@ func buildPkgContext(files []*ast.File) pkgContext { constStrings: map[string]string{}, allConsts: map[string]string{}, varParams: map[string]paramInfo{}, + funcDecls: map[string]*ast.FuncDecl{}, + } + + for _, f := range files { + for _, decl := range f.Decls { + fd, ok := decl.(*ast.FuncDecl) + if !ok || fd.Name == nil || fd.Recv != nil { + continue + } + ctx.funcDecls[fd.Name.Name] = fd + } } // Pass 1: collect const values (package-level and function-local). @@ -428,26 +440,44 @@ func returnsParamDef(fd *ast.FuncDecl) bool { } // extractParams finds the evetest.DefineTestParameters call in fd and returns -// the resolved parameter definitions. +// the resolved parameter definitions. Calls to package-local functions are +// followed, so that a test which delegates its body to a shared helper — the +// shape two tests take when they differ only in a setup argument — still reports +// the parameters that helper defines. func extractParams( fd *ast.FuncDecl, ctx pkgContext, paramFuncs map[string]paramInfo) []paramInfo { + return extractParamsFrom(fd, ctx, paramFuncs, map[string]bool{}) +} + +// extractParamsFrom is extractParams over one function body; visited records the +// functions already descended into so that recursion terminates. +func extractParamsFrom(fd *ast.FuncDecl, ctx pkgContext, + paramFuncs map[string]paramInfo, visited map[string]bool) []paramInfo { + if fd.Body == nil || fd.Name == nil || visited[fd.Name.Name] { + return nil + } + visited[fd.Name.Name] = true var params []paramInfo ast.Inspect(fd.Body, func(n ast.Node) bool { call, ok := n.(*ast.CallExpr) if !ok { return true } - sel, ok := call.Fun.(*ast.SelectorExpr) - if !ok { - return true - } - pkg, ok := sel.X.(*ast.Ident) - if !ok || pkg.Name != "evetest" || sel.Sel.Name != "DefineTestParameters" { - return true - } - for _, arg := range call.Args { - if pi, ok := resolveParamArg(arg, ctx, paramFuncs); ok { - params = append(params, pi) + switch fun := call.Fun.(type) { + case *ast.SelectorExpr: + pkg, ok := fun.X.(*ast.Ident) + if !ok || pkg.Name != "evetest" || fun.Sel.Name != "DefineTestParameters" { + return true + } + for _, arg := range call.Args { + if pi, ok := resolveParamArg(arg, ctx, paramFuncs); ok { + params = append(params, pi) + } + } + case *ast.Ident: + if callee, ok := ctx.funcDecls[fun.Name]; ok { + params = append(params, + extractParamsFrom(callee, ctx, paramFuncs, visited)...) } } return true From e225e354fe1dae646e9ed24a4810858bbc36c3bd Mon Sep 17 00:00:00 2001 From: eriknordmark Date: Wed, 5 Aug 2026 10:16:49 +0200 Subject: [PATCH 29/29] evetest: add installer-provisioned resize variants MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The kvm→k conversion tests provisioned their device from the live image only. An installer-written boot disk differs exactly where the conversion is most exposed: its ESP carries a zero-length boot/.boot_repository, and the offline grow relocates the ESP by copying its FAT32 contents, which go-diskfs rejects with "invalid start cluster: 0" before diskfs/go-diskfs#419. A live-image ESP has no such file, so nothing covered that path. Each test gains a FromInstaller sibling differing only in how the boot disk comes into existence. The shared body moves into a helper taking the reuse policy, so the conversion sequence, the readiness gates and every assertion stay common to both. Measured on an installer-provisioned device: the 0-byte marker is present before the resize and intact after it, across an ESP grown from 36 MiB to 2 GiB. Signed-off-by: eriknordmark Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_0156usNe9ABT9Y5ofQH3na8w --- evetest/tests/resize/appvolshrink_test.go | 43 +++++++++++++++------ evetest/tests/resize/kvmtok_resize_test.go | 45 +++++++++++++++------- 2 files changed, 64 insertions(+), 24 deletions(-) diff --git a/evetest/tests/resize/appvolshrink_test.go b/evetest/tests/resize/appvolshrink_test.go index cce4c5cd1b9..012dc8ed12f 100644 --- a/evetest/tests/resize/appvolshrink_test.go +++ b/evetest/tests/resize/appvolshrink_test.go @@ -88,7 +88,31 @@ const ( // early when the volume fills and reports the committed high-water mark, which // the verify is then held to. // - VOLVERIFY_IMAGE: Docker repo of the volverify testapp. +// +// The device is provisioned from the LIVE image; see +// TestAppVolumeShrinkCorruptionFromInstaller for the installer-written variant. func TestAppVolumeShrinkCorruption(test *testing.T) { + runAppVolumeShrinkCorruption(test, evetest.CreateFromScratchWithLiveImage) +} + +// TestAppVolumeShrinkCorruptionFromInstaller takes the same corruption +// measurement against a boot disk the EVE installer laid out, instead of a +// pre-built live image written to the disk whole. An installer-written ESP carries +// a zero-length boot/.boot_repository, which the offline grow's FAT32 copy can only +// read with diskfs/go-diskfs#419 ("invalid start cluster: 0" without it); a +// live-image ESP has no such file, so only this variant covers that path. +// +// Same parameters as TestAppVolumeShrinkCorruption. Expect a longer setup: the +// installer VM boots and writes the disk before the test's own device is up. +func TestAppVolumeShrinkCorruptionFromInstaller(test *testing.T) { + runAppVolumeShrinkCorruption(test, evetest.CreateFromScratchWithInstaller) +} + +// runAppVolumeShrinkCorruption is the body shared by both variants. +// provisionPolicy selects how the device's boot disk comes into existence; +// everything after Setup is identical. +func runAppVolumeShrinkCorruption(test *testing.T, + provisionPolicy evetest.ExistingEdgeDeviceReusePolicy) { evetestT := evetest.Init(test) t := NewGomegaWithT(evetestT) defer evetest.Close() @@ -251,17 +275,14 @@ func TestAppVolumeShrinkCorruption(test *testing.T) { const devName = "edge-dev" evetest.Setup( evetest.RequireEdgeDevice{ - Name: devName, - WithEVEVersion: initialVersion, - WithHypervisor: initialHypervisor, - WithTPM: withTPM, - MinDiskSizeInMiB: diskSizeMiB, - MinRAMInMiB: effectiveRAMMiB, - MinCPUs: effectiveCPUs, - // Provision from the LIVE image, not the installer: the installer ESP - // carries a 0-byte boot/.boot_repository that the offline grow's FAT32 - // copy rejects with "invalid start cluster: 0" (diskfs/go-diskfs#417). - DeviceReusePolicy: evetest.CreateFromScratchWithLiveImage, + Name: devName, + WithEVEVersion: initialVersion, + WithHypervisor: initialHypervisor, + WithTPM: withTPM, + MinDiskSizeInMiB: diskSizeMiB, + MinRAMInMiB: effectiveRAMMiB, + MinCPUs: effectiveCPUs, + DeviceReusePolicy: provisionPolicy, }, evetest.RequireNetworkModel{NetworkModel: netmodels.SingleEthWithDHCP}, ) diff --git a/evetest/tests/resize/kvmtok_resize_test.go b/evetest/tests/resize/kvmtok_resize_test.go index ccf0e7b825d..0890d4d1524 100644 --- a/evetest/tests/resize/kvmtok_resize_test.go +++ b/evetest/tests/resize/kvmtok_resize_test.go @@ -66,7 +66,8 @@ const ( // TestKvmToKResize drives 16.6.0-kvm (small) → current EVE-kvm (same geometry, // lands the conversion code) → current EVE-k (cross-flavor ⇒ offline shrink+grow), -// keeping a container app across the conversion. +// keeping a container app across the conversion. The device is provisioned from +// the LIVE image, the way eden brings one up. // // Parameters: // - INITIAL_EVE_VERSION (required, e.g. "16.6.0") / INITIAL_HYPERVISOR (kvm) — @@ -77,6 +78,28 @@ const ( // - DISK_SIZE_MB — keep the boot disk full (no free tail) so the conversion // SHRINKS P3 rather than growing into free space (asserted: decision==shrink). func TestKvmToKResize(test *testing.T) { + runKvmToKResize(test, evetest.CreateFromScratchWithLiveImage) +} + +// TestKvmToKResizeFromInstaller runs the same conversion against a boot disk the +// EVE installer laid out, instead of a pre-built live image written to the disk +// whole. That distinction is what the variant is for: an installer-written ESP +// carries a zero-length boot/.boot_repository, and the offline grow relocates the +// ESP by copying its FAT32 contents, which go-diskfs can only read with +// diskfs/go-diskfs#419 ("invalid start cluster: 0" without it). A live-image ESP +// has no such file, so only this variant covers that path. Run it against an EVE +// image built with the fix. +// +// Same parameters as TestKvmToKResize. Expect a longer setup: the installer VM +// boots and writes the disk before the test's own device is up. +func TestKvmToKResizeFromInstaller(test *testing.T) { + runKvmToKResize(test, evetest.CreateFromScratchWithInstaller) +} + +// runKvmToKResize is the body shared by both variants. provisionPolicy selects +// how the device's boot disk comes into existence; everything after Setup is +// identical, since the conversion sequence does not depend on it. +func runKvmToKResize(test *testing.T, provisionPolicy evetest.ExistingEdgeDeviceReusePolicy) { evetestT := evetest.Init(test) t := NewGomegaWithT(evetestT) defer evetest.Close() @@ -177,18 +200,14 @@ func TestKvmToKResize(test *testing.T) { const devName = "edge-dev" evetest.Setup( evetest.RequireEdgeDevice{ - Name: devName, - WithEVEVersion: initialVersion, - WithHypervisor: initialHypervisor, - WithTPM: withTPM, - MinDiskSizeInMiB: diskSizeMiB, - MinRAMInMiB: effectiveRAMMiB, - MinCPUs: effectiveCPUs, - // Provision from the LIVE image (as eden does), not the installer. The - // installer ESP carries a 0-byte marker (boot/.boot_repository) that the - // offline grow's FAT32 copy (go-diskfs) rejects with "invalid start - // cluster: 0"; the live ESP has no such file. See the go-diskfs issue. - DeviceReusePolicy: evetest.CreateFromScratchWithLiveImage, + Name: devName, + WithEVEVersion: initialVersion, + WithHypervisor: initialHypervisor, + WithTPM: withTPM, + MinDiskSizeInMiB: diskSizeMiB, + MinRAMInMiB: effectiveRAMMiB, + MinCPUs: effectiveCPUs, + DeviceReusePolicy: provisionPolicy, }, evetest.RequireNetworkModel{NetworkModel: netmodels.SingleEthWithDHCP}, )