lf-edge · eriknordmark · Jul 14, 2026 · May 29, 2026 · Jul 1, 2026 · Jul 2, 2026
diff --git a/.github/workflows/yetus.yml b/.github/workflows/yetus.yml
@@ -31,11 +31,20 @@ jobs:
           git diff upstream/${{ github.base_ref }}...HEAD > ${{ github.workspace }}/pr.patch
           # Get back to upstream master so patch can be applied
           git checkout upstream/master
+          git submodule update --init --recursive
+
+      - name: Prepare gitconfig for container
+        run: |
+          cat > /tmp/gitconfig <<'EOF'
+          [safe]
+              directory = *
+          EOF
 
       - name: Yetus
         run: |
           docker run --rm \
             -v ${{ github.workspace }}:/workspace \
+            -v /tmp/gitconfig:/etc/gitconfig \
             lfedge/eve-yetus:0.15.1-eve-2 \
             test-patch \
             --basedir=/workspace/src \

diff --git a/Makefile b/Makefile
@@ -18,7 +18,7 @@ uniq = $(if $1,$(firstword $1) $(call uniq,$(filter-out $(firstword $1),$1)))
 
 # you are not supposed to tweak these variables -- they are effectively R/O
 HV_DEFAULT=kvm
-GOVER ?= 1.24.1
+GOVER ?= 1.25.11
 PKGBASE=github.com/lf-edge/eve
 GOMODULE=$(PKGBASE)/pkg/pillar
 GOTREE=$(CURDIR)/pkg/pillar

diff --git a/README.md b/README.md
@@ -290,6 +290,15 @@ Once your Raspberry Pi is happily running an EVE image you can start using EVE c
 Notice that the support for Raspberry Pi 5 on U-boot is still limited. USB is currently not available during the boot stage.
 This is due to the new RP1 south-bridge chip, which requires a proprietary firmware and driver not yet supported by U-Boot.
 
+#### How to install EVE to an NVMe on Raspberry Pi 5
+
+Since USB is not supported on u-boot for Raspberry Pi 5, the installation should be performed from an SD Card:
+
+1. Make sure to enable `dtparam=pciex1` in your config.txt
+2. Flash the EVE installer RAW image to an SD Card
+3. Boot from the SD Card and let the installation process finish
+4. Remove the SD Card
+
 ### How to use on an Onlogic FR201 ARM device
 
 Onlogic Factor 201 (FR201) is a device based on the Raspberry Pi Compute Module 4 (CM4). There are two methods to install EVE on the FR201: flashing a live image directly using rpiboot, or using the USB installer.

diff --git a/build-tools/src/scripts/Dockerfile b/build-tools/src/scripts/Dockerfile
@@ -1,5 +1,8 @@
-ARG GOVER=1.20.1
-FROM golang:${GOVER}-alpine
+ARG GOVER=1.25.11
+# Pin the Alpine minor version to match eve-alpine (3.22) so this builder tracks
+# the same toolchain/libc as the real EVE build; the unpinned golang:*-alpine
+# tag floats to whatever Alpine is newest.
+FROM golang:${GOVER}-alpine3.22
 ARG USER
 ARG GROUP
 ARG UID
@@ -23,7 +26,7 @@ RUN echo "${USER} ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/${USER}
 # coreutils's uname -o breaks above url generation.
 # hadolint ignore=DL3018
 RUN apk add --no-cache coreutils
-ENV ZFS_VERSION=2.3.3
+ENV ZFS_VERSION=2.3.6
 ENV ZFS_COMMIT=zfs-${ZFS_VERSION}
 ENV ZFS_REPO=https://github.com/openzfs/zfs
 
@@ -64,3 +67,7 @@ RUN mv /go/bin/* /usr/bin
 ENV HOME /home/${USER}
 ENV GOFLAGS=-mod=vendor
 ENV GO111MODULE=on
+# go-libzfs pulls in the OpenZFS libspl headers, which still call the glibc
+# LFS64 symbols fstat64/stat64. musl (>=1.2.4) dropped those, so remap them to
+# the plain 64-bit fstat/stat -- same as pkg/pillar/Dockerfile.
+ENV CGO_CFLAGS="-Dfstat64=fstat -Dstat64=stat"
diff --git a/docs/CONFIG-PROPERTIES.md b/docs/CONFIG-PROPERTIES.md
@@ -49,6 +49,7 @@ This document mirrors the key names, types, defaults, and ranges defined there.
 | storage.zfs.reserved.percent | integer percent | 20 | 1 | 99 | min. percent of persist partition reserved for zfs performance |
 | storage.longhorn.disk.reserved.gigabytes | integer GB | 2 | 0 | 1048576 | per-disk storage reserved by Longhorn on the local node; overrides Longhorn's default 25% reservation. 0 sets storageReserved to 0 bytes (no reservation). 1048576 disables EVE's override, leaving Longhorn's current value in place |
 | storage.longhorn.snapshot.cron | cron string | `0 0 * * *` | - | - | cron schedule for Longhorn recurring snapshots; empty string disables. Snapshots bound delta rebuilds after node power loss to writes since the last snapshot. Default daily at midnight UTC. Standard 5-field cron syntax. EVE-k only. |
+| storage.longhorn.node-drain-policy | string | `block-for-eviction-if-contains-last-replica` | - | - | Longhorn cluster-wide node-drain-policy setting. Controls whether a node drain is permitted when the node holds Longhorn replicas. Valid values: `block-for-eviction`, `block-for-eviction-if-contains-last-replica`, `allow-if-replica-is-stopped`, `always-allow`. EVE-k only. |
 | storage.apps.ignore.disk.check | boolean | false | - | - | Ignore disk usage check for Apps. Allows apps to create images bigger than available disk |
 | timer.appcontainer.stats.interval | integer in seconds | 300 (5 minutes) | 1 | 4294967295 (max uint32) | collect application container stats |
 | timer.vault.ready.cutoff | integer in seconds | 300 (5 minutes) | 60 (1 minute) | 4294967295 (max uint32) | reboot after inaccessible vault |
@@ -110,6 +111,13 @@ This document mirrors the key names, types, defaults, and ranges defined there.
 | dhcp.enable.vendorclassid | bool | true | - | - | Enables sending the DHCP Vendor Class Identifier (Option 60) to identify the device as EVE OS. This allows networks or DHCP servers to apply policies such as VLAN assignment or granting access to the EVE controller. Some badly configured DHCP servers may reject unknown vendor class IDs. Setting this to false disables sending the vendor class ID. |
 | igpu.gop | string | "" | - | - | Filename (basename only) of a proprietary Intel GOP Option ROM placed under `/persist/gop/`. Used for Intel iGPU passthrough to provide a pre-OS UEFI framebuffer. Empty (default) or a missing file falls back to the bundled open-source `igd.rom` (IgdAssignmentDxe only — OS display works but no pre-OS framebuffer). Path separators, `..`, and absolute paths are rejected to prevent directory traversal. See [INTEL-IGPU-PASSTHROUGH.md](INTEL-IGPU-PASSTHROUGH.md) for details. |
 | debug.enable.efi | boolean | false | - | - | When true, attaches an `isa-debugcon` device at I/O port 0x402 to each KVM guest, writing OVMF/EDK2 `DEBUG()` output to `/run/hypervisor/kvm/<domain>/efi-debug.log`. Only produces useful output with a `TARGET=DEBUG` OVMF build (default is `TARGET=RELEASE` which compiles `DEBUG()` macros out). Primary diagnostic for iGPU passthrough GOP failures. |
+| debug.qemu.process.core | boolean | true | - | - | When true (default), if a KVM guest's qemu process dies on a fatal signal (SIGBUS/SIGSEGV/SIGABRT) the host kernel writes a bounded process core; pillar compresses it (zstd) into the encrypted vault under `/persist/vault/qemu-trace/<uuid>/` and rotates it. See [QEMU-CRASH-DEBUGGING.md](QEMU-CRASH-DEBUGGING.md). |
+| debug.qemu.guest.core | boolean | true | - | - | When true (default), if a KVM guest enters `internal-error` (e.g. `KVM_RUN -EFAULT`), pillar captures the guest's physical RAM as an ELF core (`dump-guest-memory`), compresses it (zstd) into the encrypted vault, and rotates it. |
+| debug.qemu.process.core.guest.ram | boolean | false | - | - | When true, includes the guest's RAM in the qemu process core (`dump-guest-core = on`). Off by default: it makes the core as large as the VM's RAM and can hold guest secrets; the guest RAM is already captured far more cheaply by the guest core (`debug.qemu.guest.core`). Note the process core is capped at ~1 GiB (`RLIMIT_CORE`), so on a VM with more RAM this core is truncated — use the guest core for full guest RAM. |
+| debug.qemu.pause.on.crash | boolean | false | - | - | When true, on a guest `internal-error` domainmgr keeps qemu alive and freezes the domain (BROKEN, held) for live inspection instead of tearing it down, until an operator releases it or a ~30 min timeout expires. An opt-in inspection aid on top of the always-on guest-core dump; pair with `debug.qemu.gdb`. |
+| debug.qemu.gdb | boolean | false | - | - | When true, each KVM guest exposes a gdb stub on a per-domain UNIX socket at `/run/hypervisor/kvm/<domain>/gdb` for live vCPU/memory inspection (especially with `debug.qemu.pause.on.crash`). Off by default. |
+| debug.qemu.trace.events | string | "" | - | - | Comma-separated qemu trace-event names/globs and/or `@<preset>` macros (`@iommu`, `@barmap`, `@vfio`) to enable per-VM. Writes a binary simpletrace log to `/persist/vault/qemu-trace/<domain>.<ts>.trace`. Empty (default) disables tracing. See [QEMU-CRASH-DEBUGGING.md](QEMU-CRASH-DEBUGGING.md). |
+| debug.qemu.igpu.no.mmap | boolean | false | - | - | When true, adds `x-no-mmap=on` to the Intel iGPU vfio-pci device so every BAR access traps into qemu instead of being mmap'd direct to hardware. Debug aid: makes the guest's iGPU MMIO writes visible in the qemu trace (`debug.qemu.trace.events`). Large performance cost; off by default. Only affects the iGPU device. |
 
 ## Local Profile Server (LPS) intervals
 

diff --git a/docs/INTEL-IGPU-PASSTHROUGH.md b/docs/INTEL-IGPU-PASSTHROUGH.md
@@ -134,7 +134,7 @@ VfioIgdPkg builds `igd.rom`, an EFI Option ROM containing:
 
 ### Changes to QEMU's vfio-igd quirk
 
-The QEMU patches in `pkg/xen-tools` (patches 08–11) rework `hw/vfio/igd.c`:
+The QEMU patches in `pkg/xen-tools` (patches 08–11 and 15) rework `hw/vfio/igd.c`:
 
 **Patch 08 — igd_gen() backport**: upstream's `igd_gen()` returns correct generation
 numbers for Gen7 through Gen12 (Haswell through Raptor Lake).  The old function returned
@@ -172,6 +172,23 @@ Based on upstream QEMU commits:
 - [`f926baa0`](https://github.com/qemu/qemu/commit/f926baa03b7babb8291ea4c1cbeadaf224977dae)
   "vfio/igd: emulate BDSM in mmio bar0 for gen 6-10 devices" by Tomita Moeko
 
+**Patch 15 — DBUF_CTL POWER_STATE sanitize** (Gen9+): on some hosts the firmware
+POST modeset leaves the display data buffer (DBUF) powered, so the passed-through
+`DBUF_CTL` slice registers (S1..S4) read back `POWER_STATE` (bit30) = 1 while
+`POWER_REQUEST` (bit31) = 0 — a legitimate-but-inconsistent leftover (the device
+is not display-reset on assignment; `POWER_STATE` is a read-only status latch fed
+by the display power well, independent of the `POWER_REQUEST` input). The guest's
+Intel driver samples `POWER_STATE` to decide which DBUF slices are already
+enabled, sees the stale "powered" bit, and never issues `POWER_REQUEST`; DBUF
+then powers down, the plane FIFO underruns, and scanout is corrupted (vertical
+stripes) until a full modeset (e.g. a display sleep/wake) re-requests power. The
+quirk traps the `DBUF_CTL` slice registers (as many as the generation exposes) in BAR0 and clears `POWER_STATE` on read whenever
+`POWER_REQUEST` is not set, presenting a consistent register — the same approach
+Intel's own GVT device model uses (`gen9_dbuf_ctl_mmio_write`). The guest then
+issues the power request and the real power well brings DBUF up. Native Linux
+i915 does not hit this because it force-drives `POWER_REQUEST` at load regardless
+of the readout; the Windows driver trusts the readout.
+
 ---
 
 ## What works and what does not
@@ -291,6 +308,36 @@ side-by-side comparison across host platforms (e.g. TGL vs RPL-P) when
 diagnosing GOP / connector init differences. Multiple dumps can be passed
 in one invocation; the decoder is read-only.
 
+### Debugging scanout corruption (iGPU MMIO register diff)
+
+Scanout corruption on a passed-through iGPU is usually a display-engine register
+left in a bad state. Because the device is bound to `vfio-pci` the host cannot
+read its BARs directly (the sysfs `resourceN` mmap is refused, and
+`/proc/<qemu>/mem` reads of the vfio BAR fault). Read the live MMIO through QEMU
+instead: `pmemsave` on the guest-physical BAR0 address dumps the register block
+to a file (QEMU maps the vfio BAR as a `ram_device` region). The helpers live in
+`tools/qemu/`:
+
+- `igpu-dump.py` — runs inside the `debug` container; snapshots the BAR0
+  display-register block (`0x40000..0x80000`) via QMP `pmemsave`.
+- `igpu-capture.sh` — from a workstation, captures two snapshots of the current
+  state and pulls them locally (set `NODE=root@<edge-node-ip>`).
+- `igpu-regdiff.py` — decodes and diffs two states, filtering volatile registers,
+  with a Gen12/RPL display-register name map.
+- `qmp.py` — minimal QMP/HMP helper (e.g. `info pci`, `xp`).
+
+Capture a corrupted state and a recovered state, then diff — the registers that
+differ are the prime suspects:
+
+```sh
+NODE=root@<edge-node-ip> tools/qemu/igpu-capture.sh bad     # while corrupted
+# ... recover (e.g. trigger a display sleep/wake) ...
+NODE=root@<edge-node-ip> tools/qemu/igpu-capture.sh good    # after recovery
+tools/qemu/igpu-regdiff.py --a igpu-dumps/bad*.bin --b igpu-dumps/good*.bin
+```
+
+This is how the DBUF_CTL `POWER_STATE` issue (patch 15) was found and verified.
+
 ---
 
 ## Supported Intel GPU generations

diff --git a/docs/QEMU-CRASH-DEBUGGING.md b/docs/QEMU-CRASH-DEBUGGING.md
@@ -0,0 +1,133 @@
+# QEMU / guest crash debugging
+
+EVE can automatically capture post-mortem state when a KVM guest or its qemu
+process crashes, so a rare, hard-to-reproduce fault can be root-caused from a
+single occurrence. This document is a usage guide: what the feature captures,
+the config knobs, and how to work with the results (including attaching `gdb`
+to a live, held VM over its UNIX socket).
+
+## What gets captured
+
+Two crash classes are handled, both **on by default**:
+
+| Crash | What EVE captures | When |
+|---|---|---|
+| **qemu process died** on a fatal signal (SIGBUS/SIGSEGV/SIGABRT) | a **qemu process core** (qemu's own address space) | `debug.qemu.process.core` (default on) |
+| **guest VM entered `internal-error`** (e.g. `KVM_RUN -EFAULT`) — qemu still alive | a **guest core** (guest physical RAM, ELF) | `debug.qemu.guest.core` (default on) |
+
+Both dumps are ELF (the universal, `gdb`-loadable format) and are compressed
+with zstd at rest.
+
+### Where dumps live
+
+All artifacts are written to the **encrypted vault**:
+
+```text
+/persist/vault/qemu-trace/<app-uuid>/<UTC-timestamp>.qemu-core.zst      # qemu process core
+/persist/vault/qemu-trace/<app-uuid>/<UTC-timestamp>.guestmem.elf.zst   # guest core
+/persist/vault/qemu-trace/<app-name>.<UTC-timestamp>.trace              # qemu trace (if enabled)
+```
+
+- Dumps are **encrypted at rest** (guest RAM can contain customer secrets).
+- They are **retained per-app as a small ring** (the newest few) and bounded by
+  a global cap and a free-space floor, so diagnostics can never fill `/persist`.
+- They are **collected by `collect-info`** (the developer/support bundle) so a
+  crash can be analyzed off-device.
+
+## Configuration (controller / `debug.qemu.*`)
+
+| Property | Default | Effect |
+|---|---|---|
+| `debug.qemu.process.core` | on | capture the qemu process core on a fatal signal |
+| `debug.qemu.guest.core` | on | capture the guest core on `internal-error` |
+| `debug.qemu.process.core.guest.ram` | off | also include guest RAM in the qemu process core (large; usually unnecessary — the guest core already has it) |
+| `debug.qemu.pause.on.crash` | off | on a guest crash, keep qemu **alive** and hold the domain for live inspection (see below) |
+| `debug.qemu.gdb` | off | expose a per-domain gdb stub UNIX socket |
+| `debug.qemu.trace.events` | "" | enable qemu tracing (see Tracing) |
+
+The controller reports a precise reason on the app instance, e.g.
+`QEMU process crashed, core dump saved` or
+`guest VM crashed, guest core saved`.
+
+## Scenario 1 — automatic capture (default)
+
+Nothing to configure. After a crash the dump appears under
+`/persist/vault/qemu-trace/<app-uuid>/`. Retrieve and analyze it on a dev host
+(the on-device `zstd` is minimal — always decompress off-device):
+
+```sh
+scp -i <key> root@<node>:/persist/vault/qemu-trace/<uuid>/<ts>.guestmem.elf.zst .
+zstd -d --long=31 <ts>.guestmem.elf.zst -o guest.elf
+gdb <guest-kernel-vmlinux> guest.elf        # or: crash <vmlinux> guest.elf
+```
+
+A qemu process core opens the same way (`gdb $(which qemu-system-x86_64)
+qemu.core`). `tools/qemu/analyse-guest-dump.sh` pulls per-vCPU RIPs from a
+guest core regardless of guest OS.
+
+## Scenario 2 — hold a crashed VM and attach gdb (live inspection)
+
+Set both knobs, then reproduce the crash:
+
+```text
+debug.qemu.pause.on.crash = true
+debug.qemu.gdb            = true
+```
+
+On a guest `internal-error`, EVE captures the guest core **and** leaves qemu
+alive/frozen (the app shows `BROKEN … held for inspection`) for ~30 minutes
+(then it auto-recovers). While held, attach `gdb` to the guest's stub over its
+UNIX socket — no port forwarding needed, pipe `gdb` through `ssh`+`socat`:
+
+```sh
+gdb
+(gdb) target remote | ssh -i <key> root@<node> socat - UNIX-CONNECT:/run/hypervisor/kvm/<domain>/gdb
+(gdb) info registers
+(gdb) x/16xg $rsp
+```
+
+`<domain>` is the qemu `-name` (`<uuid>.<version>.<appnum>`; `ls
+/run/hypervisor/kvm/` lists live ones). You can also bridge the socket to TCP
+if you prefer (`socat TCP-LISTEN:1234,reuseaddr,fork UNIX-CONNECT:<socket>` on
+the node + `ssh -L 1234:localhost:1234`, then `target remote :1234`).
+
+To release the hold early, restart the app instance from the controller.
+
+## Scenario 3 — qemu tracing
+
+Set `debug.qemu.trace.events` to a CSV of qemu trace-event names/globs and/or
+`@<preset>` macros, then reproduce:
+
+| Preset | Covers |
+|---|---|
+| `@iommu` | VFIO + intel-iommu DMA-mapping flux (IOTLB invalidations, unmap/replay) |
+| `@barmap` | PCI BAR-mapping / PM transitions (vfio-pci ↔ KVM EPT), mmap-fault |
+| `@vfio` | device lifecycle: INTx/MSI/MSI-X, reset (FLR/PM/hot-reset), display/EDID |
+
+Example: `debug.qemu.trace.events = "@barmap,@iommu,vfio_pci_write_config"`.
+The trace is a **binary simpletrace** log. Retrieve and decode on a host — pull
+both the trace and the matching `trace-events-all` (it must come from the same
+qemu-xen build, so copy it off the device; it lives in the xen-tools container),
+then decode with qemu's `simpletrace.py` from the qemu-xen source tree:
+
+```sh
+scp -i <key> root@<node>:/persist/vault/qemu-trace/<name>.<ts>.trace .
+scp -i <key> root@<node>:/containers/services/xen-tools/rootfs/usr/share/qemu-xen/qemu/trace-events-all .
+<qemu-xen-src>/scripts/simpletrace.py trace-events-all <name>.<ts>.trace
+```
+
+## Scenario 4 — fault injection (debug builds only)
+
+For validating the capture path on hardware, a **debug-only** qemu build
+(`CONFIG_EVE_CRASH_INJECTOR`, disabled in production) adds the
+`x-inject-internal-error` QMP command, which stops the VM into `internal-error`
+(emitting the same `STOP` a real crash does). `tools/qemu/inject-crash.sh`
+drives both classes on a node:
+
+```sh
+tools/qemu/inject-crash.sh guest <domain>   # x-inject-internal-error  -> guest core
+tools/qemu/inject-crash.sh qemu  <domain>   # SIGABRT the qemu process -> process core
+```
+
+A production build has no injector; a qemu process crash is still exercisable
+any time with `kill -ABRT <qemu-pid>`.