lf-edge · milan-zededa · Aug 5, 2026 · Aug 1, 2026 · Aug 4, 2026 · Aug 2, 2026
@@ -72,9 +72,13 @@ RUN if [ "$TARGETARCH" = "$BUILDARCH" ]; then export CGO_ENABLED=1; else export
 ###########################
 FROM alpine:${ALPINE_VERSION}
 
-# Install runtime dependencies
+# Install runtime dependencies.
+# qemu-img provides qemu-img/qemu-io/qemu-nbd; mtools provides mcopy, used to
+# overlay a device's config files onto the EVE config partition image exactly
+# as pkg/eve/runme.sh does. bash is not used by the broker itself -- it is for
+# interactive debugging via `docker exec`, where BusyBox ash is painful.
 # hadolint ignore=DL3018
-RUN apk add --no-cache libvirt iptables ip6tables qemu-img
+RUN apk add --no-cache libvirt iptables ip6tables qemu-img mtools bash
 
 # Copy the built broker binary from builder
 COPY --from=builder /go/bin/evetest-broker /usr/local/bin/evetest-broker

@@ -66,11 +66,14 @@ ENV GOFLAGS="-mod=readonly"
 # hadolint ignore=DL3018
 RUN apk add --no-cache iptables dnsmasq radvd swtpm
 
-# Install QEMU
+# Install QEMU. qemu-img also provides qemu-io/qemu-nbd; mtools provides mcopy,
+# used together to assemble a device's config partition and write it into a
+# template-backed disk image (see Dockerfile.broker).
 # hadolint ignore=DL3018
 RUN apk add --no-cache \
     qemu \
     qemu-img \
+    mtools \
     qemu-system-x86_64 \
     qemu-system-aarch64
 

@@ -46,14 +46,24 @@ ifndef NAME
 endif
 	$(eval EVETEST_NAME := $(NAME))
 endif
+	@# Default the EVE version to this checkout's, which is what the container
+	@# transport needs to name an image. Deliberately NOT defaulted under
+	@# EVETEST_EVE_LIVE_IMAGE: the live transport looks the version up among the
+	@# local builds under dist/, and `make version` reports the working tree as it
+	@# is *now* -- including a "-dirty-<timestamp>" suffix -- which is a directory
+	@# that generally does not exist. Left empty, the newest build (dist/<arch>/
+	@# current) is used and reports its own version. An EVETEST_EVE_VERSION the
+	@# operator sets explicitly is always honoured, by both transports.
 	$(eval EVE_VERSION_ENV :=)
 ifndef EVETEST_EVE_VERSION
+ifeq ($(strip $(filter-out false False FALSE 0 f F,$(EVETEST_EVE_LIVE_IMAGE))),)
 	$(eval EVETEST_EVE_VERSION := $(strip $(shell \
 		$(MAKE) -s -C $(REPO_ROOT) version 2>/dev/null \
 	)))
 	$(eval EVE_VERSION_ENV := -e EVETEST_EVE_VERSION=$(EVETEST_EVE_VERSION))
 endif
-	@echo "Running evetest $(EVETEST_NAME) for EVE $(EVETEST_EVE_VERSION) with evetest image version $(EVETEST_VERSION) (adam $(EVETEST_ADAM_VERSION))..."
+endif
+	@echo "Running evetest $(EVETEST_NAME) for EVE $(or $(EVETEST_EVE_VERSION),the newest local build) with evetest image version $(EVETEST_VERSION) (adam $(EVETEST_ADAM_VERSION))..."
 	$(eval ENV_VARS := $(shell env | grep '^EVETEST_' | grep -v '^EVETEST_NAME=' | sed 's/^/-e /'))
 	$(eval ARTIFACTS_MOUNT :=)
 ifneq ($(EVETEST_COLLECT_ARTIFACTS),)
@@ -105,6 +115,19 @@ endif
 	@# never available; omit the flag so the container starts and QEMU falls back
 	@# to TCG software emulation.
 	$(eval KVM_DEVICE := $(shell [ -e /dev/kvm ] && echo '--device=/dev/kvm'))
+	@# Mount the EVE build output directory so a locally built live.qcow2 is
+	@# visible inside the container. Same absolute path on both sides so that
+	@# EVETEST_EVE_DIST_DIR is valid both inside the container and in any error
+	@# message a developer sees on the host. Read-write: the harness caches the
+	@# image's sha256 in a sidecar file next to it. Only added when a local
+	@# build exists, so nothing breaks without one.
+	$(eval EVETEST_DIST_DIR := $(or $(EVETEST_EVE_DIST_DIR),$(REPO_ROOT)/dist))
+	$(eval DIST_DIR_MOUNT :=)
+	$(eval DIST_DIR_ENV :=)
+	$(if $(wildcard $(EVETEST_DIST_DIR)), \
+		$(eval DIST_DIR_MOUNT := -v $(EVETEST_DIST_DIR):$(EVETEST_DIST_DIR)) \
+		$(eval DIST_DIR_ENV := -e EVETEST_EVE_DIST_DIR=$(EVETEST_DIST_DIR)) \
+	)
 	@if docker inspect --type container evetest-$(EVETEST_API_PORT) >/dev/null 2>&1; then \
 		echo ""; \
 		echo "Error: evetest instance 'evetest-$(EVETEST_API_PORT)' is already running."; \
@@ -138,18 +161,24 @@ endif
 		$(EVETEST_HOME_MOUNT) \
 		$(BROKER_IMAGE_MOUNT) \
 		$(BROKER_PROXY_CA_MOUNT) \
+		$(DIST_DIR_MOUNT) \
 		$(GO_CACHE_MOUNT) \
 		$(DOCKER_CONFIG_MOUNT) \
 		$(ENV_VARS) \
 		$(EVE_VERSION_ENV) \
 		$(BROKER_IMAGE_ENV) \
+		$(DIST_DIR_ENV) \
 		$(GO_CACHE_ENV) \
 		-e EVETEST_NAME=$(EVETEST_NAME) \
 		-e EVETEST_HOME=$(EVETEST_HOME_DIR) \
 		-e EVETEST_HOST_UID=$(EVETEST_HOST_UID) \
 		-e EVETEST_HOST_GID=$(EVETEST_HOST_GID) \
 		$(EVETEST_IMAGE)
 
+# Only builds when the image is missing -- a locally built image is never
+# rebuilt just because harness source changed, so after editing evetest code
+# run `make build-container` explicitly before `make evetest`, or the run
+# silently uses the stale, previously-built harness.
 ensure-evetest-image:
 	@if ! docker image inspect $(EVETEST_IMAGE) >/dev/null 2>&1; then \
 		echo "Docker image $(EVETEST_IMAGE) not found locally, trying to pull..."; \

@@ -487,6 +487,110 @@ EVETEST_COLLECT_ARTIFACTS=/tmp/evetest-artifacts \
     make evetest NAME=TestDHCPIPv4Only
 ```
 
+### Testing a Local EVE Build
+
+A developer iterating on EVE can point evetest at their own `make live` output instead
+of pulling or building an EVE container image:
+
+```bash
+make live                                               # build EVE locally
+EVETEST_EVE_LIVE_IMAGE=true make evetest NAME=<TestName>
+```
+
+**Two independent settings.** Which EVE build runs and how its bits get delivered are
+separate questions:
+
+| | setting | values |
+|---|---|---|
+| **which build** | `EVETEST_EVE_VERSION` | a version (`16.0.0-lts`), or unset |
+| **how it is delivered** | `EVETEST_EVE_LIVE_IMAGE` | `true` = the artifacts `make live` wrote; unset/`false` = an EVE container image |
+
+A local build is not a transport: `make eve` produces a local *container* image, and the
+harness pushes it to the broker when the broker does not have it. So
+`EVETEST_EVE_LIVE_IMAGE` says only *how*, never *which* — it is a plain boolean and takes
+no path.
+
+With the live transport on, the version selects the build directory under
+`EVETEST_EVE_DIST_DIR` (set automatically by `make evetest`):
+
+- **`EVETEST_EVE_VERSION` unset** — the newest build, via `dist/<arch>/current`; the run
+  reports whatever version that turns out to be. Note the symlink is **arch-scoped**
+  (`dist/amd64/current`, not `dist/current`).
+- **`EVETEST_EVE_VERSION=<v>`** — `dist/<arch>/<v>/`. If that version is not built here,
+  the run **fails** rather than quietly fetching it from a registry: you asked for a
+  specific build *and* for the live transport, and silently delivering different bits is
+  the kind of thing that costs an afternoon to notice.
+- A version pinned by the test itself (`RequireEdgeDevice.WithEVEVersion`, e.g.
+  `TestEVEUpgrade`'s pre-upgrade version) names a *released* build and always comes from
+  a container image, whatever the transport setting.
+
+The harness content-hashes the image, then tells the broker both the hash and where the
+files are (`live.qcow2`, `installer/config.img`, `installer/firmware/*`), and the broker
+picks how to get them:
+
+- **It can read those paths itself** -- all-in-one mode, or a broker you started by hand
+  on your own machine. It installs the template straight from the dist directory, and
+  nothing is uploaded at all. Whether the paths are readable is decided by the broker,
+  by looking; the harness does not guess from its deployment mode.
+- **It cannot** (the usual distributed setup, where the broker is a different machine).
+  It reports the image as missing and the harness uploads it as a tar, exactly as
+  before.
+
+Either way the upload/read happens only when the broker does not already hold that hash,
+and the template is then installed through the machinery described in
+[The EVE Image Template Cache](#the-eve-image-template-cache). The version reported for the
+run is the resolved build directory's name (e.g. `0.0.0-my-branch-abc123`), which is the
+authority on what was actually delivered.
+
+A local read is only ever used when the file's size **and** its sha256 match what the
+harness declared, so a broker can never install different content under a hash another
+run will later ask for -- the same check the upload path applies to received bytes.
+
+This is dramatically faster than the container path: there is no container build and no
+multi-gigabyte pull. Rebuilding with `make live` produces a qcow2 with different content
+and therefore a new hash, so the next run performs exactly one new upload -- nothing is
+re-uploaded until the image actually changes.
+
+#### Upgrading *to* a local build
+
+`TestEVEUpgrade` boots `INITIAL_EVE_VERSION` and upgrades to `EVE_VERSION`, and the two
+axes make both useful shapes fall out without any special-casing:
+
+```bash
+# A released version upgraded to another released version
+EVETEST_INITIAL_EVE_VERSION=16.0.0-lts EVETEST_EVE_VERSION=16.1.0-lts \
+    make evetest NAME=TestEVEUpgrade
+
+# A released version upgraded to your working tree
+make live
+EVETEST_INITIAL_EVE_VERSION=16.0.0-lts EVETEST_EVE_LIVE_IMAGE=true \
+    make evetest NAME=TestEVEUpgrade
+```
+
+The pre-upgrade device pins `INITIAL_EVE_VERSION`, so it always comes from a container
+image — a released build is the point of that field. Only the *target* uses the live
+transport, and it needs a different artifact than a fresh device does: an upgrade installs
+a base OS image, so the harness serves the build's own `installer/rootfs.img` from its HTTP
+image server and takes the version EVE will report from `installer/eve_version`. No
+container is pulled and the broker is not involved at all, since the rootfs goes straight
+from the harness to the device.
+
+Two constraints to be aware of:
+
+- **Installer-based tests cannot use this path.** A live qcow2 cannot produce an
+  installer flow, so a test that also requests an installer
+  (`CreateFromScratchWithInstaller`) fails immediately with a clear error instead of
+  silently falling back to the container path.
+- **The broker must advertise `CAPABILITY_LOCAL_LIVE_IMAGE`.** All three providers do
+  today; a broker too old to support the feature, or a future provider that still builds
+  images per device (see
+  [The EVE Image Template Cache](#the-eve-image-template-cache)), fails the test with a
+  clear error rather than quietly falling back and testing a different EVE build than
+  the one requested.
+
+See [Essential Variables](#essential-variables) for the full reference on
+`EVETEST_EVE_LIVE_IMAGE`, `EVETEST_EVE_DIST_DIR`, and `EVETEST_EVE_FIRMWARE_DIR`.
+
 ### Code Coverage
 
 When EVE is built with `COVER=y`, the `zedbox` binary is instrumented for
@@ -709,6 +813,9 @@ non-default behavior.
 | `EVETEST_NAME` | Test or suite name to run (**required**) | -- |
 | `EVETEST_OUTPUT_FORMAT` | `go test` output format: `json` (machine-readable, for `gotestfmt`) or `quiet` (compact, no `-v`); default is verbose (`-v`). **Do not combine `quiet` with `EVETEST_PAUSE_ON_FAILURE` or `EVETEST_PAUSE_ON_CHECKPOINT`** — without `-v`, `go test` buffers all output until the test completes, so a pause appears frozen with no visible output. | -- |
 | `EVETEST_EVE_VERSION` | EVE version to test | current repo HEAD |
+| `EVETEST_EVE_LIVE_IMAGE` | **How** EVE's bits reach a device: `true` delivers the artifacts `make live` wrote under `EVETEST_EVE_DIST_DIR`, unset/`false` uses an EVE container image. A boolean only -- **which** build to run is `EVETEST_EVE_VERSION`'s business, so this takes no path, and a non-boolean value is an error. See [Testing a Local EVE Build](#testing-a-local-eve-build) | `false` |
+| `EVETEST_EVE_DIST_DIR` | EVE build output directory whose `<arch>/<version>/` subdirectories (and the `<arch>/current` symlink) hold the local builds. Must be an absolute path (the harness runs inside a container). Set automatically by `make evetest` when a local `dist/` directory exists | -- |
+| `EVETEST_EVE_FIRMWARE_DIR` | Overrides firmware discovery for a local live image, which otherwise looks for `OVMF*.fd` in `installer/firmware` next to the resolved qcow2 | -- |
 | `EVETEST_PREFERRED_ARCH` | Preferred CPU architecture (`amd64`, `arm64`) | `amd64` |
 | `EVETEST_LOG_LEVEL` | Framework log level (`debug`, `info`, `warn`) | `info` |
 | `EVETEST_COLLECT_ARTIFACTS` | Host path for artifacts (logs, collect-info) | -- |
@@ -782,6 +889,8 @@ Common to every provider:
 | `EVETEST_BROKER_MAX_CLIENTS` | Max concurrent evetest clients the broker will accept; new connections are rejected with an error once this many are already connected (reconnects of existing clients are never blocked) | `-1` (unlimited) |
 | `EVETEST_BROKER_DOCKER_IMAGE_RETENTION` | How long, in minutes, an unused, evetest-managed Docker image (one the broker itself pulled or built) is kept before the broker's periodic cleanup removes it | `10080` (7 days) |
 | `EVETEST_BROKER_DOCKER_DISK_USAGE_THRESHOLD` | Disk usage percent (on the filesystem backing Docker's storage) at or above which the broker aggressively evicts the oldest unused, evetest-managed Docker images, regardless of the retention setting above | `80` |
+| `EVETEST_BROKER_TEMPLATE_RETENTION` | How long, in minutes, an unused EVE disk-image template (see [The EVE Image Template Cache](#the-eve-image-template-cache)) is kept before the broker's periodic cleanup removes it. Deliberately generous, since templates let consecutive runs against the same EVE version skip the image build entirely; zero or negative disables age-based eviction, but disk-usage-based eviction still applies regardless. A template still backing a live VM is never removed regardless of this value | `10080` (7 days) |
+| `EVETEST_BROKER_TEMPLATE_DISK_USAGE_THRESHOLD` | Disk usage percent (on the filesystem backing the broker's image directory) at or above which the broker evicts the oldest unreferenced EVE image templates, regardless of the retention setting above. Deliberately higher than `EVETEST_BROKER_DOCKER_DISK_USAGE_THRESHOLD`: broker hosts routinely idle above 80%, so an 80% threshold would evict every unreferenced template on every pass and the cache would never stay warm; templates are also the wrong thing to give up first -- one is 1-2 GB, where the Docker image store is tens of GB | `90` |
 | `EVETEST_BROKER_PPROF_PORT` | Port for the broker's `net/http/pprof` debug endpoint (listens on all interfaces); `0` disables it | `0` (disabled) |
 
 **`libvirt` provider only:**
@@ -936,6 +1045,44 @@ for reuse across tests, and acts as a tunnel proxy forwarding IP packets between
 evetest container and the SDN VM. This tunneling allows the evetest container to operate
 without direct network connectivity to the VMs -- it only needs access to the broker.
 
+#### The EVE Image Template Cache
+
+Building an EVE image used to mean one full container build per device -- roughly 4
+minutes and 2 GB of I/O -- run serially, even though most of that work (unpacking the
+container, laying out the disk) does not depend on anything device-specific like the
+onboarding certificate. The broker now builds a **configuration-independent template**
+once per distinct (docker image content ID, disk size, installer flag, arch) and reuses
+it across every device and every test run that matches:
+
+- Templates are cached under `$EVETEST_BROKER_IMAGE_DIR/templates/<key>/`, keyed by
+  content rather than by EVE version string, so identical image content built under a
+  different tag still hits the cache.
+- Each device gets its own qcow2 copy-on-write overlay backed by the template's disk,
+  with that device's own 5 MiB FAT config partition written into the overlay's CONFIG
+  partition -- the template itself is never modified.
+- Templates are reference-counted: one currently backing a live VM is never deleted,
+  regardless of age or disk pressure.
+- An image-directory-wide `flock` guards template creation and eviction, so two brokers
+  sharing the same `EVETEST_BROKER_IMAGE_DIR` cannot destroy each other's state.
+- Unreferenced templates are evicted by age (`EVETEST_BROKER_TEMPLATE_RETENTION`) and by
+  disk pressure (`EVETEST_BROKER_TEMPLATE_DISK_USAGE_THRESHOLD`), mirroring the existing
+  Docker image cleanup.
+
+Every provider uses the cache; they differ only in how a device's disk is derived from a
+template. `libvirt` and `qemu` use **overlays**, since both attach local image files
+directly. `proxmox` uses a **standalone copy**: it uploads each device's disk to the PVE
+node, where a backing file would not exist. A live-image template is deliberately keyed
+without the disk size, so the per-device disk -- overlay or copy -- is grown to the
+requested size instead.
+
+With the `proxmox` provider that copy is what the broker uploads, so the per-device cost
+there is one full-size copy plus the upload to the node, against a ~4-minute container
+build per device before.
+
+With the `qemu` provider the broker lives inside the short-lived evetest container, so
+its 30-minute cleanup loop would rarely tick before the container exits. There the sweep
+runs once at broker startup instead -- same age and disk-pressure rules, same variables.
+
 ### SDN (Software Defined Network)
 
 The SDN is a lightweight LinuxKit-based VM that models physical network infrastructure