diff --git a/.github/workflows/yetus.yml b/.github/workflows/yetus.yml index 68e187742be..44dce5f607e 100644 --- a/.github/workflows/yetus.yml +++ b/.github/workflows/yetus.yml @@ -31,11 +31,20 @@ jobs: git diff upstream/${{ github.base_ref }}...HEAD > ${{ github.workspace }}/pr.patch # Get back to upstream master so patch can be applied git checkout upstream/master + git submodule update --init --recursive + + - name: Prepare gitconfig for container + run: | + cat > /tmp/gitconfig <<'EOF' + [safe] + directory = * + EOF - name: Yetus run: | docker run --rm \ -v ${{ github.workspace }}:/workspace \ + -v /tmp/gitconfig:/etc/gitconfig \ lfedge/eve-yetus:0.15.1-eve-2 \ test-patch \ --basedir=/workspace/src \ diff --git a/Makefile b/Makefile index 5f96622884b..2ecefe11b14 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ uniq = $(if $1,$(firstword $1) $(call uniq,$(filter-out $(firstword $1),$1))) # you are not supposed to tweak these variables -- they are effectively R/O HV_DEFAULT=kvm -GOVER ?= 1.24.1 +GOVER ?= 1.25.11 PKGBASE=github.com/lf-edge/eve GOMODULE=$(PKGBASE)/pkg/pillar GOTREE=$(CURDIR)/pkg/pillar diff --git a/README.md b/README.md index 294f3cc9ba3..e11aff63923 100644 --- a/README.md +++ b/README.md @@ -290,6 +290,15 @@ Once your Raspberry Pi is happily running an EVE image you can start using EVE c Notice that the support for Raspberry Pi 5 on U-boot is still limited. USB is currently not available during the boot stage. This is due to the new RP1 south-bridge chip, which requires a proprietary firmware and driver not yet supported by U-Boot. +#### How to install EVE to an NVMe on Raspberry Pi 5 + +Since USB is not supported on u-boot for Raspberry Pi 5, the installation should be performed from an SD Card: + +1. Make sure to enable `dtparam=pciex1` in your config.txt +2. Flash the EVE installer RAW image to an SD Card +3. Boot from the SD Card and let the installation process finish +4. Remove the SD Card + ### How to use on an Onlogic FR201 ARM device Onlogic Factor 201 (FR201) is a device based on the Raspberry Pi Compute Module 4 (CM4). There are two methods to install EVE on the FR201: flashing a live image directly using rpiboot, or using the USB installer. diff --git a/build-tools/src/scripts/Dockerfile b/build-tools/src/scripts/Dockerfile index ba9adfd0ceb..bcd4de60d2f 100644 --- a/build-tools/src/scripts/Dockerfile +++ b/build-tools/src/scripts/Dockerfile @@ -1,5 +1,8 @@ -ARG GOVER=1.20.1 -FROM golang:${GOVER}-alpine +ARG GOVER=1.25.11 +# Pin the Alpine minor version to match eve-alpine (3.22) so this builder tracks +# the same toolchain/libc as the real EVE build; the unpinned golang:*-alpine +# tag floats to whatever Alpine is newest. +FROM golang:${GOVER}-alpine3.22 ARG USER ARG GROUP ARG UID @@ -23,7 +26,7 @@ RUN echo "${USER} ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/${USER} # coreutils's uname -o breaks above url generation. # hadolint ignore=DL3018 RUN apk add --no-cache coreutils -ENV ZFS_VERSION=2.3.3 +ENV ZFS_VERSION=2.3.6 ENV ZFS_COMMIT=zfs-${ZFS_VERSION} ENV ZFS_REPO=https://github.com/openzfs/zfs @@ -64,3 +67,7 @@ RUN mv /go/bin/* /usr/bin ENV HOME /home/${USER} ENV GOFLAGS=-mod=vendor ENV GO111MODULE=on +# go-libzfs pulls in the OpenZFS libspl headers, which still call the glibc +# LFS64 symbols fstat64/stat64. musl (>=1.2.4) dropped those, so remap them to +# the plain 64-bit fstat/stat -- same as pkg/pillar/Dockerfile. +ENV CGO_CFLAGS="-Dfstat64=fstat -Dstat64=stat" diff --git a/docs/CONFIG-PROPERTIES.md b/docs/CONFIG-PROPERTIES.md index 87907dfa519..2a68883bdee 100644 --- a/docs/CONFIG-PROPERTIES.md +++ b/docs/CONFIG-PROPERTIES.md @@ -49,6 +49,7 @@ This document mirrors the key names, types, defaults, and ranges defined there. | storage.zfs.reserved.percent | integer percent | 20 | 1 | 99 | min. percent of persist partition reserved for zfs performance | | storage.longhorn.disk.reserved.gigabytes | integer GB | 2 | 0 | 1048576 | per-disk storage reserved by Longhorn on the local node; overrides Longhorn's default 25% reservation. 0 sets storageReserved to 0 bytes (no reservation). 1048576 disables EVE's override, leaving Longhorn's current value in place | | storage.longhorn.snapshot.cron | cron string | `0 0 * * *` | - | - | cron schedule for Longhorn recurring snapshots; empty string disables. Snapshots bound delta rebuilds after node power loss to writes since the last snapshot. Default daily at midnight UTC. Standard 5-field cron syntax. EVE-k only. | +| storage.longhorn.node-drain-policy | string | `block-for-eviction-if-contains-last-replica` | - | - | Longhorn cluster-wide node-drain-policy setting. Controls whether a node drain is permitted when the node holds Longhorn replicas. Valid values: `block-for-eviction`, `block-for-eviction-if-contains-last-replica`, `allow-if-replica-is-stopped`, `always-allow`. EVE-k only. | | storage.apps.ignore.disk.check | boolean | false | - | - | Ignore disk usage check for Apps. Allows apps to create images bigger than available disk | | timer.appcontainer.stats.interval | integer in seconds | 300 (5 minutes) | 1 | 4294967295 (max uint32) | collect application container stats | | timer.vault.ready.cutoff | integer in seconds | 300 (5 minutes) | 60 (1 minute) | 4294967295 (max uint32) | reboot after inaccessible vault | @@ -110,6 +111,13 @@ This document mirrors the key names, types, defaults, and ranges defined there. | dhcp.enable.vendorclassid | bool | true | - | - | Enables sending the DHCP Vendor Class Identifier (Option 60) to identify the device as EVE OS. This allows networks or DHCP servers to apply policies such as VLAN assignment or granting access to the EVE controller. Some badly configured DHCP servers may reject unknown vendor class IDs. Setting this to false disables sending the vendor class ID. | | igpu.gop | string | "" | - | - | Filename (basename only) of a proprietary Intel GOP Option ROM placed under `/persist/gop/`. Used for Intel iGPU passthrough to provide a pre-OS UEFI framebuffer. Empty (default) or a missing file falls back to the bundled open-source `igd.rom` (IgdAssignmentDxe only — OS display works but no pre-OS framebuffer). Path separators, `..`, and absolute paths are rejected to prevent directory traversal. See [INTEL-IGPU-PASSTHROUGH.md](INTEL-IGPU-PASSTHROUGH.md) for details. | | debug.enable.efi | boolean | false | - | - | When true, attaches an `isa-debugcon` device at I/O port 0x402 to each KVM guest, writing OVMF/EDK2 `DEBUG()` output to `/run/hypervisor/kvm//efi-debug.log`. Only produces useful output with a `TARGET=DEBUG` OVMF build (default is `TARGET=RELEASE` which compiles `DEBUG()` macros out). Primary diagnostic for iGPU passthrough GOP failures. | +| debug.qemu.process.core | boolean | true | - | - | When true (default), if a KVM guest's qemu process dies on a fatal signal (SIGBUS/SIGSEGV/SIGABRT) the host kernel writes a bounded process core; pillar compresses it (zstd) into the encrypted vault under `/persist/vault/qemu-trace//` and rotates it. See [QEMU-CRASH-DEBUGGING.md](QEMU-CRASH-DEBUGGING.md). | +| debug.qemu.guest.core | boolean | true | - | - | When true (default), if a KVM guest enters `internal-error` (e.g. `KVM_RUN -EFAULT`), pillar captures the guest's physical RAM as an ELF core (`dump-guest-memory`), compresses it (zstd) into the encrypted vault, and rotates it. | +| debug.qemu.process.core.guest.ram | boolean | false | - | - | When true, includes the guest's RAM in the qemu process core (`dump-guest-core = on`). Off by default: it makes the core as large as the VM's RAM and can hold guest secrets; the guest RAM is already captured far more cheaply by the guest core (`debug.qemu.guest.core`). Note the process core is capped at ~1 GiB (`RLIMIT_CORE`), so on a VM with more RAM this core is truncated — use the guest core for full guest RAM. | +| debug.qemu.pause.on.crash | boolean | false | - | - | When true, on a guest `internal-error` domainmgr keeps qemu alive and freezes the domain (BROKEN, held) for live inspection instead of tearing it down, until an operator releases it or a ~30 min timeout expires. An opt-in inspection aid on top of the always-on guest-core dump; pair with `debug.qemu.gdb`. | +| debug.qemu.gdb | boolean | false | - | - | When true, each KVM guest exposes a gdb stub on a per-domain UNIX socket at `/run/hypervisor/kvm//gdb` for live vCPU/memory inspection (especially with `debug.qemu.pause.on.crash`). Off by default. | +| debug.qemu.trace.events | string | "" | - | - | Comma-separated qemu trace-event names/globs and/or `@` macros (`@iommu`, `@barmap`, `@vfio`) to enable per-VM. Writes a binary simpletrace log to `/persist/vault/qemu-trace/..trace`. Empty (default) disables tracing. See [QEMU-CRASH-DEBUGGING.md](QEMU-CRASH-DEBUGGING.md). | +| debug.qemu.igpu.no.mmap | boolean | false | - | - | When true, adds `x-no-mmap=on` to the Intel iGPU vfio-pci device so every BAR access traps into qemu instead of being mmap'd direct to hardware. Debug aid: makes the guest's iGPU MMIO writes visible in the qemu trace (`debug.qemu.trace.events`). Large performance cost; off by default. Only affects the iGPU device. | ## Local Profile Server (LPS) intervals diff --git a/docs/INTEL-IGPU-PASSTHROUGH.md b/docs/INTEL-IGPU-PASSTHROUGH.md index f5d61662a25..38e1eb0819e 100644 --- a/docs/INTEL-IGPU-PASSTHROUGH.md +++ b/docs/INTEL-IGPU-PASSTHROUGH.md @@ -134,7 +134,7 @@ VfioIgdPkg builds `igd.rom`, an EFI Option ROM containing: ### Changes to QEMU's vfio-igd quirk -The QEMU patches in `pkg/xen-tools` (patches 08–11) rework `hw/vfio/igd.c`: +The QEMU patches in `pkg/xen-tools` (patches 08–11 and 15) rework `hw/vfio/igd.c`: **Patch 08 — igd_gen() backport**: upstream's `igd_gen()` returns correct generation numbers for Gen7 through Gen12 (Haswell through Raptor Lake). The old function returned @@ -172,6 +172,23 @@ Based on upstream QEMU commits: - [`f926baa0`](https://github.com/qemu/qemu/commit/f926baa03b7babb8291ea4c1cbeadaf224977dae) "vfio/igd: emulate BDSM in mmio bar0 for gen 6-10 devices" by Tomita Moeko +**Patch 15 — DBUF_CTL POWER_STATE sanitize** (Gen9+): on some hosts the firmware +POST modeset leaves the display data buffer (DBUF) powered, so the passed-through +`DBUF_CTL` slice registers (S1..S4) read back `POWER_STATE` (bit30) = 1 while +`POWER_REQUEST` (bit31) = 0 — a legitimate-but-inconsistent leftover (the device +is not display-reset on assignment; `POWER_STATE` is a read-only status latch fed +by the display power well, independent of the `POWER_REQUEST` input). The guest's +Intel driver samples `POWER_STATE` to decide which DBUF slices are already +enabled, sees the stale "powered" bit, and never issues `POWER_REQUEST`; DBUF +then powers down, the plane FIFO underruns, and scanout is corrupted (vertical +stripes) until a full modeset (e.g. a display sleep/wake) re-requests power. The +quirk traps the `DBUF_CTL` slice registers (as many as the generation exposes) in BAR0 and clears `POWER_STATE` on read whenever +`POWER_REQUEST` is not set, presenting a consistent register — the same approach +Intel's own GVT device model uses (`gen9_dbuf_ctl_mmio_write`). The guest then +issues the power request and the real power well brings DBUF up. Native Linux +i915 does not hit this because it force-drives `POWER_REQUEST` at load regardless +of the readout; the Windows driver trusts the readout. + --- ## What works and what does not @@ -291,6 +308,36 @@ side-by-side comparison across host platforms (e.g. TGL vs RPL-P) when diagnosing GOP / connector init differences. Multiple dumps can be passed in one invocation; the decoder is read-only. +### Debugging scanout corruption (iGPU MMIO register diff) + +Scanout corruption on a passed-through iGPU is usually a display-engine register +left in a bad state. Because the device is bound to `vfio-pci` the host cannot +read its BARs directly (the sysfs `resourceN` mmap is refused, and +`/proc//mem` reads of the vfio BAR fault). Read the live MMIO through QEMU +instead: `pmemsave` on the guest-physical BAR0 address dumps the register block +to a file (QEMU maps the vfio BAR as a `ram_device` region). The helpers live in +`tools/qemu/`: + +- `igpu-dump.py` — runs inside the `debug` container; snapshots the BAR0 + display-register block (`0x40000..0x80000`) via QMP `pmemsave`. +- `igpu-capture.sh` — from a workstation, captures two snapshots of the current + state and pulls them locally (set `NODE=root@`). +- `igpu-regdiff.py` — decodes and diffs two states, filtering volatile registers, + with a Gen12/RPL display-register name map. +- `qmp.py` — minimal QMP/HMP helper (e.g. `info pci`, `xp`). + +Capture a corrupted state and a recovered state, then diff — the registers that +differ are the prime suspects: + +```sh +NODE=root@ tools/qemu/igpu-capture.sh bad # while corrupted +# ... recover (e.g. trigger a display sleep/wake) ... +NODE=root@ tools/qemu/igpu-capture.sh good # after recovery +tools/qemu/igpu-regdiff.py --a igpu-dumps/bad*.bin --b igpu-dumps/good*.bin +``` + +This is how the DBUF_CTL `POWER_STATE` issue (patch 15) was found and verified. + --- ## Supported Intel GPU generations diff --git a/docs/QEMU-CRASH-DEBUGGING.md b/docs/QEMU-CRASH-DEBUGGING.md new file mode 100644 index 00000000000..dcf5e803923 --- /dev/null +++ b/docs/QEMU-CRASH-DEBUGGING.md @@ -0,0 +1,133 @@ +# QEMU / guest crash debugging + +EVE can automatically capture post-mortem state when a KVM guest or its qemu +process crashes, so a rare, hard-to-reproduce fault can be root-caused from a +single occurrence. This document is a usage guide: what the feature captures, +the config knobs, and how to work with the results (including attaching `gdb` +to a live, held VM over its UNIX socket). + +## What gets captured + +Two crash classes are handled, both **on by default**: + +| Crash | What EVE captures | When | +|---|---|---| +| **qemu process died** on a fatal signal (SIGBUS/SIGSEGV/SIGABRT) | a **qemu process core** (qemu's own address space) | `debug.qemu.process.core` (default on) | +| **guest VM entered `internal-error`** (e.g. `KVM_RUN -EFAULT`) — qemu still alive | a **guest core** (guest physical RAM, ELF) | `debug.qemu.guest.core` (default on) | + +Both dumps are ELF (the universal, `gdb`-loadable format) and are compressed +with zstd at rest. + +### Where dumps live + +All artifacts are written to the **encrypted vault**: + +```text +/persist/vault/qemu-trace//.qemu-core.zst # qemu process core +/persist/vault/qemu-trace//.guestmem.elf.zst # guest core +/persist/vault/qemu-trace/..trace # qemu trace (if enabled) +``` + +- Dumps are **encrypted at rest** (guest RAM can contain customer secrets). +- They are **retained per-app as a small ring** (the newest few) and bounded by + a global cap and a free-space floor, so diagnostics can never fill `/persist`. +- They are **collected by `collect-info`** (the developer/support bundle) so a + crash can be analyzed off-device. + +## Configuration (controller / `debug.qemu.*`) + +| Property | Default | Effect | +|---|---|---| +| `debug.qemu.process.core` | on | capture the qemu process core on a fatal signal | +| `debug.qemu.guest.core` | on | capture the guest core on `internal-error` | +| `debug.qemu.process.core.guest.ram` | off | also include guest RAM in the qemu process core (large; usually unnecessary — the guest core already has it) | +| `debug.qemu.pause.on.crash` | off | on a guest crash, keep qemu **alive** and hold the domain for live inspection (see below) | +| `debug.qemu.gdb` | off | expose a per-domain gdb stub UNIX socket | +| `debug.qemu.trace.events` | "" | enable qemu tracing (see Tracing) | + +The controller reports a precise reason on the app instance, e.g. +`QEMU process crashed, core dump saved` or +`guest VM crashed, guest core saved`. + +## Scenario 1 — automatic capture (default) + +Nothing to configure. After a crash the dump appears under +`/persist/vault/qemu-trace//`. Retrieve and analyze it on a dev host +(the on-device `zstd` is minimal — always decompress off-device): + +```sh +scp -i root@:/persist/vault/qemu-trace//.guestmem.elf.zst . +zstd -d --long=31 .guestmem.elf.zst -o guest.elf +gdb guest.elf # or: crash guest.elf +``` + +A qemu process core opens the same way (`gdb $(which qemu-system-x86_64) +qemu.core`). `tools/qemu/analyse-guest-dump.sh` pulls per-vCPU RIPs from a +guest core regardless of guest OS. + +## Scenario 2 — hold a crashed VM and attach gdb (live inspection) + +Set both knobs, then reproduce the crash: + +```text +debug.qemu.pause.on.crash = true +debug.qemu.gdb = true +``` + +On a guest `internal-error`, EVE captures the guest core **and** leaves qemu +alive/frozen (the app shows `BROKEN … held for inspection`) for ~30 minutes +(then it auto-recovers). While held, attach `gdb` to the guest's stub over its +UNIX socket — no port forwarding needed, pipe `gdb` through `ssh`+`socat`: + +```sh +gdb +(gdb) target remote | ssh -i root@ socat - UNIX-CONNECT:/run/hypervisor/kvm//gdb +(gdb) info registers +(gdb) x/16xg $rsp +``` + +`` is the qemu `-name` (`..`; `ls +/run/hypervisor/kvm/` lists live ones). You can also bridge the socket to TCP +if you prefer (`socat TCP-LISTEN:1234,reuseaddr,fork UNIX-CONNECT:` on +the node + `ssh -L 1234:localhost:1234`, then `target remote :1234`). + +To release the hold early, restart the app instance from the controller. + +## Scenario 3 — qemu tracing + +Set `debug.qemu.trace.events` to a CSV of qemu trace-event names/globs and/or +`@` macros, then reproduce: + +| Preset | Covers | +|---|---| +| `@iommu` | VFIO + intel-iommu DMA-mapping flux (IOTLB invalidations, unmap/replay) | +| `@barmap` | PCI BAR-mapping / PM transitions (vfio-pci ↔ KVM EPT), mmap-fault | +| `@vfio` | device lifecycle: INTx/MSI/MSI-X, reset (FLR/PM/hot-reset), display/EDID | + +Example: `debug.qemu.trace.events = "@barmap,@iommu,vfio_pci_write_config"`. +The trace is a **binary simpletrace** log. Retrieve and decode on a host — pull +both the trace and the matching `trace-events-all` (it must come from the same +qemu-xen build, so copy it off the device; it lives in the xen-tools container), +then decode with qemu's `simpletrace.py` from the qemu-xen source tree: + +```sh +scp -i root@:/persist/vault/qemu-trace/..trace . +scp -i root@:/containers/services/xen-tools/rootfs/usr/share/qemu-xen/qemu/trace-events-all . +/scripts/simpletrace.py trace-events-all ..trace +``` + +## Scenario 4 — fault injection (debug builds only) + +For validating the capture path on hardware, a **debug-only** qemu build +(`CONFIG_EVE_CRASH_INJECTOR`, disabled in production) adds the +`x-inject-internal-error` QMP command, which stops the VM into `internal-error` +(emitting the same `STOP` a real crash does). `tools/qemu/inject-crash.sh` +drives both classes on a node: + +```sh +tools/qemu/inject-crash.sh guest # x-inject-internal-error -> guest core +tools/qemu/inject-crash.sh qemu # SIGABRT the qemu process -> process core +``` + +A production build has no injector; a qemu process crash is still exercisable +any time with `kill -ABRT `. diff --git a/docs/ZFS.md b/docs/ZFS.md index 4777e0c2055..5f2efce33f7 100644 --- a/docs/ZFS.md +++ b/docs/ZFS.md @@ -2,9 +2,11 @@ ZFS provides a rich set of functionality but at a cost of extra resource usage. Currently ARC (Adaptive Replacement Cache) size is -limited to `min(256 MiB + 0.3% of total zpool size, 20% of system RAM)` but at least 384 -MiB. This RAM is included in the memory EVE reserves for its own operational needs. -Thus would this memory not be available for allocation for applications. +limited to +`min(256 MiB + 0.3% of total zpool size, 20% of system RAM)` +but at least 384 MiB. This RAM is included in the memory EVE reserves +for its own operational needs. Thus would this memory not be available +for allocation for applications. If system does not have enough memory to satisfy the limit mentioned above, severe performance degradation can occur on random access @@ -39,9 +41,10 @@ zfs_dirty_data_max = 50% of zfs_arc_max zfs_dirty_data_sync_percent = 15 ``` -The following tunables are hardcoded and are optimized values for SSD/NVMe based pools. -Please note that these tunables may not work optimally for HDD based pools. -WIP to dynamically adjust these parameters depending on the pool type. +The following tunables are hardcoded and are optimized values for +SSD/NVMe based pools. Please note that these tunables may not work +optimally for HDD based pools. WIP to dynamically adjust these +parameters depending on the pool type. ```bash zfs_vdev_sync_read_min_active = 35 @@ -56,5 +59,126 @@ zfs_vdev_async_write_max_active = 10 ### Minimum recommended system requirements -Minimum recommended system requirements to install ZFS storage is 32GB memory and 3 physical disks set in eve_persist_disk. -eve_install_skip_zfs_checks should be set in installation config to override the requirement check for experimental installs. +Minimum recommended system requirements to install ZFS storage is +32GB memory and 3 physical disks set in eve_persist_disk. +eve_install_skip_zfs_checks should be set in installation config to +override the requirement check for experimental installs. + +## Storage maintenance + +### Pool TRIM (EVE-k only) + +On EVE-k (kubevirt/Longhorn) nodes the persist pool is backed by NVMe +devices. ZFS does not automatically notify the NVMe controller of freed +block ranges; without periodic TRIM the device's garbage-collection +table grows and write amplification increases. + +EVE runs `zpool trim persist` at boot (to clear any backlog from the +previous run) and then on a recurring cron schedule. The schedule is +operator-configurable: + +| Config key | Default | Effect | +| --- | --- | --- | +| `timer.zfs.pool.trim.cron` | `0 3 * * 6,0` | Sat/Sun at 03:00 | + +Set to an empty string to disable the scheduled trim. The boot-time +trim always runs regardless of this setting. + +`zpool trim` returns immediately; the actual NVMe work runs in the +background. Progress is visible via `zpool status persist`. + +### Vault fstrim (EVE-k only) + +On EVE-k nodes `/persist/vault` is an ext4 filesystem on a ZFS zvol. +When files are deleted, ext4 marks the blocks as free but does not +notify ZFS. The zvol retains the blocks as allocated, inflating +`logicalused` on the persist pool and causing EVE to overestimate dom0 +disk usage. In severe cases this can incorrectly trigger maintenance +mode. + +EVE runs `fstrim /persist/vault` at boot (to drain any backlog) and +then on a recurring cron schedule: + +| Config key | Default | Effect | +| --- | --- | --- | +| `timer.vault.trim.cron` | `0 2 * * 6,0` | Sat/Sun at 02:00 | +| `timer.vault.trim.max.secs` | `1800` | Timeout in seconds; `0` = unlimited | + +Set `timer.vault.trim.cron` to an empty string to disable the +scheduled fstrim. The boot-time fstrim always runs regardless of this +setting. + +The two maintenance operations are staggered by default to avoid +overlap: + +| Time | Operation | +| --- | --- | +| 02:00 Sat/Sun | Vault fstrim | +| 03:00 Sat/Sun | Pool-level NVMe TRIM | + +### Observing trim activity + +#### Pubsub status (live and collect-info) + +Trim state is published to pubsub and captured in collect-info bundles, +making it available for post-mortem analysis without log grepping. + +**Vault fstrim** — `vaultmgr` publishes to `VaultStatus`: + +```sh +cat /run/vaultmgr/VaultStatus/vaultmgr.json \ + | grep -A4 TrimStatus +``` + +| Field | Meaning | +| --- | --- | +| `TrimStatus.LastStartTime` | fstrim start; zero if never run this boot | +| `TrimStatus.LastEndTime` | When it completed; zero while in progress | +| `TrimStatus.LastError` | Error string on failure; empty on success | + +**ZFS pool trim** — `zfsmanager` publishes to `ZFSPoolStatus`: + +```sh +cat /run/zfsmanager/ZFSPoolStatus/persist.json \ + | grep -A2 TrimStatus +``` + +| Field | Meaning | +| --- | --- | +| `TrimStatus.LastStartTime` | When the most recent `zpool trim` was invoked | + +Pool trim end time is not published — `zpool trim` returns immediately +and NVMe work continues in the background. Use `zpool status persist` +to check completion: + +```sh +zpool status persist | grep -A5 trim +``` + +#### Agent logs + +**Vault fstrim** (`vaultmgr` agent, Notice level): + +```sh +grep -i "TrimVault\|fstrim" /persist/newlog/agentlog/vaultmgr.log* +``` + +| Log message | Meaning | +| --- | --- | +| `TrimVault: starting fstrim /persist/vault (timeout Xs)` | Run started | +| `TrimVault: fstrim /persist/vault completed in Xs` | Success | +| `TrimVault: fstrim /persist/vault failed after Xs: ...` | Failure | +| `startVaultTrimSchedule: scheduled trim starting` | Cron tick fired | + +**ZFS pool trim** (`zfsmanager` agent, Notice level): + +```sh +grep -i "runPoolTrimSchedule" /persist/newlog/agentlog/zfsmanager.log* +``` + +| Log message | Meaning | +| --- | --- | +| `runPoolTrimSchedule: boot-time demand trim starting` | Boot trim started | +| `runPoolTrimSchedule: zpool trim persist initiated` | Command accepted | +| `runPoolTrimSchedule: scheduled trim starting` | Cron tick fired | +| `runPoolTrimSchedule: zpool trim persist: ` | Failure | diff --git a/evetest/tests/storage/vault_trim_test.go b/evetest/tests/storage/vault_trim_test.go new file mode 100644 index 00000000000..60bdd2695d5 --- /dev/null +++ b/evetest/tests/storage/vault_trim_test.go @@ -0,0 +1,155 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package storage_test + +import ( + "fmt" + "strconv" + "strings" + "testing" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + "github.com/lf-edge/eve/evetest" + pillartypes "github.com/lf-edge/eve/pkg/pillar/types" +) + +// TestVaultZvolTrimReclaimsBlocks verifies that fstrim on /persist/vault +// returns ghost blocks to ZFS, reducing logicalused on the persist/vault +// dataset on an EVE-k ZFS node. +// +// Ghost blocks accumulate when ext4 frees blocks (e.g. from Longhorn replica +// churn) that the underlying ZFS zvol never receives DISCARD for. Without +// periodic fstrim these blocks inflate logicalused, which inflates usedByDom0, +// which shrinks allowedDeviceDiskSize and can trigger false maintenance mode. +// +// The test writes 256 MiB of incompressible data (/dev/urandom bypasses ZFS +// zstd compression), deletes it to create ghost blocks, then verifies that +// fstrim causes logicalused to drop. Skipped on non-kubevirt or non-ZFS nodes. +func TestVaultZvolTrimReclaimsBlocks(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: evetest.HypervisorKubevirt, + WithFilesystem: evetest.FilesystemZFS, + DeviceReusePolicy: evetest.UseAsIs, + }, + ) + device := evetest.GetEdgeDevice(devName) + + // evetest.Setup returns once the device is onboarded and has fetched its + // config; it does NOT wait for the vault to be unlocked/mounted. Gate the + // test on vault readiness before touching /persist/vault, otherwise the + // write lands on the parent persist dataset's mountpoint directory (the + // ext4-on-zvol is not mounted yet) and logicalused on the zvol never moves. + + // Wait for vaultmgr to report the default vault ConversionComplete. Read + // the VaultStatus pubsub JSON on-device via a shell (not ReadPublication): + // ReadPublication/ReadFile Fatalf on a not-yet-published file, and the + // pubsub key "Application Data Store" contains spaces that break scp's + // remote path. A failed cat just fails the poll and we retry. + vaultStatusPath := `/run/vaultmgr/VaultStatus/` + pillartypes.DefaultVaultName + `.json` + t.Eventually(func() bool { + out, _, err := device.RunShellScript( + `eve exec pillar cat "`+vaultStatusPath+`"`, 15*time.Second, 0) + if err != nil { + return false // status not published yet + } + return strings.Contains(strings.ReplaceAll(out, " ", ""), + `"ConversionComplete":true`) + }, 5*time.Minute, 5*time.Second).Should(BeTrue(), + "vaultmgr must report the default vault ConversionComplete before writing") + + // The ext4-on-zvol must actually be mounted at /persist/vault; this is the + // decisive guard for where the write lands. + t.Eventually(func() error { + _, _, err := device.RunShellScript( + "eve exec pillar mountpoint -q /persist/vault", 15*time.Second, 0) + return err + }, 2*time.Minute, 5*time.Second).Should(Succeed(), + "/persist/vault ext4-on-zvol must be mounted before writing test data") + + const mib = 1024 * 1024 + + // Reclaim any pre-existing ghost blocks first so the baseline is + // deterministic and the write below inflates logicalused by the full + // amount — a write that merely reuses untrimmed ext4 free space would not. + // zfs/fstrim and the /persist/vault mount live in the pillar container, + // not the host SSH shell, so run everything via "eve exec pillar". + _, _, err := device.RunShellScript( + "eve exec pillar fstrim /persist/vault", 120*time.Second, 0) + t.Expect(err).To(BeNil(), "baseline cleanup fstrim failed") + _, _, err = device.RunShellScript("eve exec pillar sync", 30*time.Second, 0) + t.Expect(err).To(BeNil(), "sync after cleanup fstrim failed") + + baseline, err := vaultLogicalUsed(device) + t.Expect(err).To(BeNil(), "failed to read baseline logicalused") + + // Write 256 MiB of incompressible data. /dev/zero compresses to near-zero + // under zstd; /dev/urandom forces real ZFS block allocation. + _, _, err = device.RunShellScript( + `eve exec pillar dd if=/dev/urandom of=/persist/vault/trim_test `+ + `bs=1M count=256 conv=fsync`, + 120*time.Second, 0) + t.Expect(err).To(BeNil(), "failed to write trim_test file") + _, _, err = device.RunShellScript("eve exec pillar sync", 30*time.Second, 0) + t.Expect(err).To(BeNil(), "sync after write failed") + + // ZFS accounts zvol space per transaction group, so logicalused lags the + // write by a few seconds — poll until it reflects the 256 MiB. + t.Eventually(func() (int64, error) { + return vaultLogicalUsed(device) + }, 60*time.Second, 3*time.Second).Should(BeNumerically(">", baseline+200*mib), + "logicalused should rise ~256 MiB after the write (baseline=%d)", baseline) + + // Delete the file. ext4 frees the blocks but the underlying zvol never + // receives DISCARD, so they linger as ghost blocks (logicalused stays high). + _, _, err = device.RunShellScript( + "eve exec pillar rm /persist/vault/trim_test", 30*time.Second, 0) + t.Expect(err).To(BeNil(), "failed to remove trim_test file") + _, _, err = device.RunShellScript("eve exec pillar sync", 30*time.Second, 0) + t.Expect(err).To(BeNil(), "sync after rm failed") + + // fstrim issues DISCARD for the freed blocks; the zvol reclaims them. + // -v logs the trimmed byte count to aid diagnosis on failure. + trimOut, _, err := device.RunShellScript( + "eve exec pillar fstrim -v /persist/vault", 120*time.Second, 0) + t.Expect(err).To(BeNil(), "fstrim /persist/vault failed") + test.Logf("fstrim: %s", strings.TrimSpace(trimOut)) + _, _, err = device.RunShellScript("eve exec pillar sync", 30*time.Second, 0) + t.Expect(err).To(BeNil(), "sync after fstrim failed") + + // Poll until the reclaim is reflected: logicalused must fall back near + // baseline, proving fstrim returned the ghost blocks to ZFS. + t.Eventually(func() (int64, error) { + return vaultLogicalUsed(device) + }, 60*time.Second, 3*time.Second).Should(BeNumerically("<", baseline+64*mib), + "fstrim must reclaim ghost blocks; logicalused should return near baseline=%d", + baseline) +} + +// vaultLogicalUsed returns the current logicalused value for persist/vault in +// bytes, as reported by `zfs get -Hp logicalused`. zfs lives in the pillar +// container, so the command is run via "eve exec pillar". +func vaultLogicalUsed(device *evetest.EdgeDevice) (int64, error) { + stdout, _, err := device.RunShellScript( + "eve exec pillar zfs get -Hp logicalused persist/vault", + 30*time.Second, 0) + if err != nil { + return 0, err + } + // Output: "persist/vault\tlogicalused\t\t-\n" + fields := strings.Fields(strings.TrimSpace(stdout)) + if len(fields) < 3 { + return 0, fmt.Errorf("unexpected zfs get output: %q", stdout) + } + return strconv.ParseInt(fields[2], 10, 64) +} diff --git a/kernel-commits.mk b/kernel-commits.mk index b46eb412657..2239aaafbc3 100644 --- a/kernel-commits.mk +++ b/kernel-commits.mk @@ -3,5 +3,5 @@ KERNEL_COMMIT_amd64_v6.12.49_generic = dcdba3ddf871 KERNEL_COMMIT_arm64_v5.10.192_nvidia-jp5 = 2e0dcfd3260d KERNEL_COMMIT_arm64_v5.15.136_nvidia-jp6 = 4929f15eda41 KERNEL_COMMIT_arm64_v6.8.12_nvidia-jp7 = 452eaffef5ed -KERNEL_COMMIT_arm64_v6.1.155_generic = 88e6efd1067a +KERNEL_COMMIT_arm64_v6.1.155_generic = 9fa67514972d KERNEL_COMMIT_riscv64_v6.1.112_generic = 30aa75d58cdd diff --git a/pkg/debug/scripts/collect-info.sh b/pkg/debug/scripts/collect-info.sh index aaa506093bf..b2860c0cc4b 100755 --- a/pkg/debug/scripts/collect-info.sh +++ b/pkg/debug/scripts/collect-info.sh @@ -697,6 +697,8 @@ ln -s /persist/log "$DIR/persist-log" [ -d /persist/kubelog ] && ln -s /persist/kubelog "$DIR/persist-kubelog" ln -s /persist/netdump "$DIR/persist-netdump" ln -s /persist/kcrashes "$DIR/persist-kcrashes" +# qemu/guest crash dumps live in the (decrypted) vault; collect them for devs. +[ -d /persist/vault/qemu-trace ] && ln -s /persist/vault/qemu-trace "$DIR/persist-qemu-trace" [ -d /persist/memory-monitor/output ] && ln -s /persist/memory-monitor/output "$DIR/persist-memory-monitor-output" [ -f /persist/agentdebug/watcher/sigusr1 ] && cp /persist/agentdebug/watcher/sigusr1 "$DIR/goroutin-leak-detector-stacks-dump" ln -s /run "$DIR/root-run" diff --git a/pkg/installer/Dockerfile b/pkg/installer/Dockerfile index 876f29d423a..56541b5623c 100644 --- a/pkg/installer/Dockerfile +++ b/pkg/installer/Dockerfile @@ -28,7 +28,7 @@ RUN cargo sbom > sbom.spdx.json RUN cp "/usr/local/my-installer/target/$CARGO_BUILD_TARGET/release/installer" /usr/local/my-installer/target/installer -FROM lfedge/eve-debug:7a9363916694d3fcdb8e15f74624aceaaa7e0e9c AS debug +FROM lfedge/eve-debug:86d119744e5d405338a25f09fb08324efa52728c AS debug # Dockerfile to build installer img initrd FROM lfedge/eve-alpine:39f46094f640424c345164420ed789afd8a4088b AS build diff --git a/pkg/kube/cluster-init.sh b/pkg/kube/cluster-init.sh index c1d81296c33..9ba4304e994 100755 --- a/pkg/kube/cluster-init.sh +++ b/pkg/kube/cluster-init.sh @@ -1,5 +1,6 @@ #!/bin/sh # shellcheck disable=SC3043 # 'local' is non-POSIX but supported by busybox ash, EVE's /bin/sh +# shellcheck disable=SC1091 # sourced scripts exist only at device runtime, not in the repo tree # # Copyright (c) 2023-2024 Zededa, Inc. # SPDX-License-Identifier: Apache-2.0 @@ -175,7 +176,7 @@ mount_kube_root() { do sleep 1 done - mount "$KUBE_ROOT_ZFS" "$KUBE_ROOT_MOUNTPOINT" ## This is where we persist the cluster components (etcd) + mount "$KUBE_ROOT_ZFS" "$KUBE_ROOT_MOUNTPOINT" logmsg "persist/etcd-storage available" elif [ "$persistType" = "ext4" ]; then logmsg "Using EXT4 persistent storage" diff --git a/pkg/pillar/cmd/domainmgr/crash.go b/pkg/pillar/cmd/domainmgr/crash.go new file mode 100644 index 00000000000..81bc7feaa89 --- /dev/null +++ b/pkg/pillar/cmd/domainmgr/crash.go @@ -0,0 +1,326 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Crash handling for KVM/qemu domains. domainmgr owns the crash lifecycle +// end-to-end: it detects a mode-A crash (guest KVM_RUN -EFAULT -> +// RUN_STATE_INTERNAL_ERROR, surfaced as a DomainCrashEvent by the hypervisor), +// captures the guest core BEFORE any teardown, then applies policy (mark BROKEN +// and tear down, or hold for inspection). Mode-B (qemu process fatal signal) +// cores are written by the kernel via core_pattern and picked up on death. + +package domainmgr + +import ( + "bufio" + "math" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/lf-edge/eve/pkg/pillar/diskmetrics" + "github.com/lf-edge/eve/pkg/pillar/qemudump" + "github.com/lf-edge/eve/pkg/pillar/types" +) + +// cgroupUnlimited: memory.limit_in_bytes uses a near-max value for "no limit"; +// treat any absurdly large limit as unlimited so we fall back to system RAM. +const cgroupUnlimited = uint64(1) << 62 + +// cgroupMemBase is the v1 memory-cgroup root as seen in pillar's mounted host fs. +const cgroupMemBase = "/hostfs/sys/fs/cgroup/memory/" + +// compressorAvailMem returns the memory available to the dump compressor: the +// smallest headroom (limit-usage) across pillar's whole cgroup ancestry +// (/eve, /eve/services, /eve/services/pillar) and system MemAvailable. The zstd +// window is anonymous memory charged to pillar's cgroup, so it must fit the +// tightest binding cgroup — a parent can be loose while a child is nearly full +// (that OOM-killed zedbox). When a cgroup is unlimited, system RAM is the real +// bound. Returns 0 (=> minimum window) if nothing can be determined. +func compressorAvailMem() uint64 { + // This reads cgroup-v1 memory accounting. On a non-v1 (v2/unified) host the + // v1 files are absent, so we cannot learn pillar's cgroup headroom; sizing the + // window from system RAM could then exceed the cgroup limit and OOM-kill + // zedbox. Floor to the minimum window instead until cgroup-v2 is supported. + if _, err := os.Stat(cgroupMemBase); err != nil { + return 0 + } + avail := uint64(math.MaxUint64) + for _, g := range []string{"eve", "eve/services", "eve/services/pillar"} { + if h, ok := cgroupHeadroom(cgroupMemBase + g); ok && h < avail { + avail = h + } + } + if sys := sysMemAvailableBytes(); sys > 0 && sys < avail { + avail = sys + } + if avail == uint64(math.MaxUint64) { + return 0 + } + return avail +} + +// cgroupHeadroom returns limit-usage for a v1 memory cgroup dir. ok is false if +// the cgroup can't be read or is effectively unlimited (not a binding +// constraint); a full cgroup returns (0, true). +func cgroupHeadroom(dir string) (uint64, bool) { + limit, err1 := readUint64File(dir + "/memory.limit_in_bytes") + usage, err2 := readUint64File(dir + "/memory.usage_in_bytes") + if err1 != nil || err2 != nil || limit >= cgroupUnlimited { + return 0, false + } + if usage >= limit { + return 0, true + } + return limit - usage, true +} + +func readUint64File(path string) (uint64, error) { + b, err := os.ReadFile(path) + if err != nil { + return 0, err + } + return strconv.ParseUint(strings.TrimSpace(string(b)), 10, 64) +} + +// sysMemAvailableBytes returns /proc/meminfo MemAvailable in bytes, or 0. +func sysMemAvailableBytes() uint64 { + f, err := os.Open("/proc/meminfo") + if err != nil { + return 0 + } + defer f.Close() + sc := bufio.NewScanner(f) + for sc.Scan() { + fields := strings.Fields(sc.Text()) + if len(fields) >= 2 && fields[0] == "MemAvailable:" { + if kb, perr := strconv.ParseUint(fields[1], 10, 64); perr == nil { + return kb * 1024 + } + return 0 + } + } + return 0 +} + +const ( + // qemuDumpKeepPerDomain is the per-domain rotation ring size K. + qemuDumpKeepPerDomain = 3 + // pauseOnCrashTimeout auto-releases a held domain. + pauseOnCrashTimeout = 30 * time.Minute + // stagingOrphanMaxAge: raw cores older than this in the staging dir are + // orphans (real ones are picked up promptly) and get reaped opportunistically. + stagingOrphanMaxAge = time.Hour + + crashGiB = 1 << 30 +) + +// qemuDumpDir is the encrypted-vault directory holding qemu/guest dumps. +var qemuDumpDir = filepath.Join(types.SealedDirName, "qemu-trace") + +// captureResult is handed from the async guest-core capture goroutine back to +// the per-domain runHandler. +type captureResult struct { + dumpPath string + err error +} + +// setupDumpManager builds the vault-backed dump-storage manager and installs +// the host core_pattern so qemu process cores (mode B) land in the vault. Call +// after the vault is unlocked. On failure it logs and returns a usable manager +// anyway where possible; crash dumps are best-effort and must never be fatal to +// device management. +func setupDumpManager() *qemudump.Manager { + var total uint64 + if us, err := diskmetrics.PersistUsageStat(log); err == nil { + total = us.Total + } else { + log.Errorf("crash-dump: PersistUsageStat failed, using conservative caps: %v", err) + } + // Global cap min(20 GiB, 25% of /persist); floor max(4 GiB, 10%). + globalCap := uint64(20 * crashGiB) + if q := total / 4; total > 0 && q < globalCap { + globalCap = q + } + floor := uint64(4 * crashGiB) + if f := total / 10; f > floor { + floor = f + } + + mgr := qemudump.NewManager(qemudump.Config{ + Dir: qemuDumpDir, + KeepPerDomain: qemuDumpKeepPerDomain, + PerDomainQuota: globalCap, + GlobalCap: globalCap, + FreeSpaceFloor: floor, + Concurrency: 1, + Space: func() (free, total uint64, err error) { + us, err := diskmetrics.PersistUsageStat(log) + if err != nil { + return 0, 0, err + } + return us.Free, us.Total, nil + }, + // AvailMem bounds the zstd window, which is anonymous memory charged to + // pillar's /eve/services cgroup. Sizing it from system RAM alone + // OOM-kills zedbox on a tight cgroup; sizing it from the cgroup alone + // over-allocates when the cgroup is unlimited. So use the smaller of the + // cgroup headroom and system MemAvailable — whichever actually binds. + AvailMem: compressorAvailMem, + Log: log.Noticef, + }) + + if err := qemudump.InstallDefaultCorePattern(qemuDumpDir); err != nil { + log.Errorf("crash-dump: failed to install core_pattern: %v", err) + } else { + log.Noticef("crash-dump: core_pattern -> %s", qemudump.CorePattern(qemuDumpDir)) + } + // Drop any raw cores orphaned in the staging dir by a previous boot (their + // qemu pid is gone), so staging stays bounded. + if err := mgr.SweepStaging(); err != nil { + log.Errorf("crash-dump: sweep orphaned staging cores: %v", err) + } + return mgr +} + +// crashFrozen reports whether crash handling has frozen reconcile for the +// domain: while set, no teardown, restart, or config-driven change may proceed +// (a dump is in flight, or the domain is held for inspection). +func crashFrozen(status *types.DomainStatus) bool { + return status.CrashState == types.CrashCaptureInProgress || + status.CrashState == types.CrashHeld +} + +// beginCrashCapture starts capture-first handling of a mode-A crash: it freezes +// reconcile and launches a child goroutine that streams the guest core into the +// vault. The goroutine only reads qemu (never tears down and never touches +// createSema), so it cannot stall the watchdog or block other domains. Policy +// is applied by finishCrashCapture once the result arrives on captureDone. +func beginCrashCapture(ctx *domainContext, status *types.DomainStatus, + ev types.DomainCrashEvent, captureDone chan<- captureResult) { + + log.Warnf("crash-dump: domain %s crashed (runState=%s); capturing guest core before teardown", + status.Key(), ev.RunState) + status.CrashState = types.CrashCaptureInProgress + status.CrashRunState = ev.RunState + publishDomainStatus(ctx, status) + + if ctx.dumpMgr == nil || !ctx.qemuGuestCore { + captureDone <- captureResult{} + return + } + + key := status.Key() + snapshot := *status // immutable copy for the reader goroutine + mgr := ctx.dumpMgr + go func() { + w, err := mgr.NewDump(key, qemudump.KindGuestCore) + if err != nil { + captureDone <- captureResult{err: err} + return + } + dumpErr := hyper.Task(&snapshot).DumpGuestMemory(snapshot.DomainName, w) + closeErr := w.Close() + if dumpErr != nil { + captureDone <- captureResult{err: dumpErr} + return + } + captureDone <- captureResult{dumpPath: w.Path(), err: closeErr} + }() +} + +// finishCrashCapture records the capture outcome and applies crash policy. +func finishCrashCapture(ctx *domainContext, status *types.DomainStatus, res captureResult) { + if res.err != nil { + log.Errorf("crash-dump: guest core for %s failed: %v", status.Key(), res.err) + } else if res.dumpPath != "" { + status.GuestCoreDumpPath = res.dumpPath + status.LastDumpTaken = true + log.Warnf("crash-dump: guest core for %s written to %s", status.Key(), res.dumpPath) + } + status.CrashState = types.CrashCaptured + + // Report a precise, mode-A message to the controller. + msg := "guest VM crashed (guest core capture failed)" + if status.LastDumpTaken { + msg = "guest VM crashed, guest core saved" + } + + if ctx.qemuPauseOnCrash { + holdCrashedDomain(ctx, status, msg) + return + } + brokenAndTeardown(ctx, status, msg) +} + +// holdCrashedDomain keeps qemu alive and frozen for live inspection until an +// operator releases it or the timeout expires. +func holdCrashedDomain(ctx *domainContext, status *types.DomainStatus, msg string) { + status.CrashState = types.CrashHeld + status.HoldUntil = time.Now().Add(pauseOnCrashTimeout) + status.State = types.BROKEN + if ctx.qemuGdb { + // Surface the gdbstub for the operator (only exposed when debug.qemu.gdb + // was set at domain start). Mirrors gdbSocketPath in the hypervisor. + status.GdbSocket = filepath.Join("/run/hypervisor/kvm", status.DomainName, "gdb") + } + status.SetErrorNow(msg + "; held for inspection (debug.qemu.pause.on.crash), qemu left alive") + log.Warnf("crash-dump: holding domain %s for inspection until %s", + status.Key(), status.HoldUntil.Format(time.RFC3339)) + publishDomainStatus(ctx, status) +} + +// brokenAndTeardown clears the freeze, marks the domain BROKEN, and tears down +// qemu via the normal lifecycle. +func brokenAndTeardown(ctx *domainContext, status *types.DomainStatus, errMsg string) { + status.CrashState = types.CrashNone + status.Activated = false + status.State = types.BROKEN + status.SetErrorNow(errMsg) + if err := hyper.Task(status).Delete(status.DomainName); err != nil { + log.Errorf("crash-dump: delete domain %s: %v", status.DomainName, err) + } + if err := hyper.Task(status).Cleanup(status.DomainName); err != nil { + log.Errorf("crash-dump: cleanup domain %s: %v", status.DomainName, err) + } + status.DomainId = 0 + publishDomainStatus(ctx, status) +} + +// maybeReleaseHold releases a held (pause-on-crash) domain once its timeout +// expires, then tears it down. +func maybeReleaseHold(ctx *domainContext, status *types.DomainStatus) { + if status.CrashState != types.CrashHeld || time.Now().Before(status.HoldUntil) { + return + } + log.Warnf("crash-dump: pause-on-crash hold for %s expired; releasing", status.Key()) + brokenAndTeardown(ctx, status, "crash inspection hold expired; domain recovered") +} + +// pickupProcessCore compresses a kernel-written qemu process core (mode B) into +// the vault, if one is waiting for this domain's qemu pid, and returns the .zst +// path ("" if none / not a qemu domain / on error). Call while DomainId still +// holds the dead qemu's pid. +func pickupProcessCore(ctx *domainContext, status *types.DomainStatus) string { + if ctx.dumpMgr == nil || status.DomainId == 0 { + return "" + } + log.Noticef("crash-dump: checking for qemu process core of %s (pid %d)", + status.Key(), status.DomainId) + p, err := ctx.dumpMgr.PickupProcessCore(status.Key(), status.DomainId) + if err != nil { + log.Errorf("crash-dump: pickup qemu process core for %s: %v", status.Key(), err) + return "" + } + if p != "" { + status.GuestCoreDumpPath = p + log.Warnf("crash-dump: qemu process core for %s written to %s", status.Key(), p) + } + // A crash event is when staging can grow, so reap any orphaned raw cores now + // (never-picked-up cores would otherwise linger unbounded until reboot). + if err := ctx.dumpMgr.SweepStagingStale(stagingOrphanMaxAge); err != nil { + log.Errorf("crash-dump: sweep stale staging cores: %v", err) + } + return p +} diff --git a/pkg/pillar/cmd/domainmgr/domainmgr.go b/pkg/pillar/cmd/domainmgr/domainmgr.go index cd092b73b83..392c30cd8d2 100644 --- a/pkg/pillar/cmd/domainmgr/domainmgr.go +++ b/pkg/pillar/cmd/domainmgr/domainmgr.go @@ -42,6 +42,7 @@ import ( "github.com/lf-edge/eve/pkg/pillar/hypervisor" "github.com/lf-edge/eve/pkg/pillar/kubeapi" "github.com/lf-edge/eve/pkg/pillar/pubsub" + "github.com/lf-edge/eve/pkg/pillar/qemudump" "github.com/lf-edge/eve/pkg/pillar/sema" "github.com/lf-edge/eve/pkg/pillar/sriov" "github.com/lf-edge/eve/pkg/pillar/types" @@ -156,6 +157,15 @@ type domainContext struct { // Is it EVE 'k' hvTypeKube bool nodeName string + + // Crash-dump handling. dumpMgr owns the + // vault dump storage (compression/quota/rotation); nil if unavailable. + // The three flags mirror the debug.qemu.* global settings. + dumpMgr *qemudump.Manager + qemuProcessCore bool + qemuGuestCore bool + qemuPauseOnCrash bool + qemuGdb bool } // AddAgentSpecificCLIFlags adds CLI options @@ -644,6 +654,10 @@ func Run(ps *pubsub.PubSub, loggerArg *logrus.Logger, logArg *base.LogObject, ar log.Functionf("processed vault status") + // The vault is unlocked now, so the encrypted dump directory is reachable. + // Build the dump-storage manager and install the qemu core_pattern. + domainCtx.dumpMgr = setupDumpManager() + if err := containerd.StartUserContainerdInstance(); err != nil { log.Fatalf("StartUserContainerdInstance: failed %v", err) } @@ -1030,8 +1044,20 @@ func runHandler(ctx *domainContext, key string, configChannel <-chan Notify, cpu ticker := flextimer.NewRangeTicker(time.Duration(minInterval), time.Duration(maxInterval)) + // captureDone carries the result of an async guest-core capture (mode A) + // back into this goroutine so crash policy is always applied here, never in + // the capture goroutine. Buffered so that goroutine never blocks. + captureDone := make(chan captureResult, 1) + closed := false for !closed { + // Watch for a mode-A crash while the domain is activated and not already + // being handled. A nil channel (non-KVM, or already crash-frozen) never + // fires in the select. + var crashCh <-chan types.DomainCrashEvent + if st := lookupDomainStatus(ctx, key); st != nil && st.Activated && st.CrashState == types.CrashNone { + crashCh = hyper.Task(st).WatchCrash(st.DomainName) + } select { case _, ok := <-configChannel: if ok { @@ -1043,6 +1069,13 @@ func runHandler(ctx *domainContext, key string, configChannel <-chan Notify, cpu } config := c.(types.DomainConfig) status := lookupDomainStatus(ctx, key) + if status != nil && crashFrozen(status) { + // A crash dump is in flight or the domain is held for + // inspection; defer config changes so reconcile cannot + // restart or kill the frozen qemu. + log.Noticef("runHandler(%s): deferring config change while crash-frozen", key) + continue + } if status == nil { handleCreate(ctx, key, &config) } else { @@ -1073,18 +1106,39 @@ func runHandler(ctx *domainContext, key string, configChannel <-chan Notify, cpu log.Errorf("No Status for %s", config.DisplayName) continue } + if crashFrozen(status) { + continue + } if !config.VmConfig.CPUsPinned { if err = updateNonPinnedCPUs(ctx, &config, status); err != nil { log.Warnf("failed to redistribute CPUs in %s", config.DisplayName) } } } + case ev := <-crashCh: + // Mode-A crash (guest internal-error): capture the guest core + // before any teardown, then apply policy. + if status := lookupDomainStatus(ctx, key); status != nil { + beginCrashCapture(ctx, status, ev, captureDone) + } + case res := <-captureDone: + if status := lookupDomainStatus(ctx, key); status != nil { + finishCrashCapture(ctx, status, res) + } case <-ticker.C: log.Tracef("runHandler(%s) timer", key) status := lookupDomainStatus(ctx, key) if status != nil { - verifyStatus(ctx, status) - maybeRetry(ctx, status) + switch status.CrashState { + case types.CrashHeld: + maybeReleaseHold(ctx, status) + case types.CrashNone: + verifyStatus(ctx, status) + maybeRetry(ctx, status) + default: + // CrashCaptureInProgress / CrashCaptured: the capture + // goroutine will drive finishCrashCapture; do not reconcile. + } } } } @@ -1093,6 +1147,12 @@ func runHandler(ctx *domainContext, key string, configChannel <-chan Notify, cpu // Check if it is still running func verifyStatus(ctx *domainContext, status *types.DomainStatus) { + // Never reconcile a domain whose crash is being handled: a dump may be in + // flight, or the domain may be held for inspection. Tearing it down here + // would race the capture / kill the held qemu. + if crashFrozen(status) { + return + } // Check config.Active to avoid spurious errors when shutting down configActivate := false config := lookupDomainConfig(ctx, status.Key()) @@ -1124,7 +1184,16 @@ func verifyStatus(ctx *domainContext, status *types.DomainStatus) { if domainStatus == types.BROKEN { err := fmt.Errorf("one of the %s tasks has crashed (%v)", status.Key(), err) log.Error(err.Error()) - status.SetErrorNow("one of the application's tasks has crashed - please restart application instance") + // Mode B: qemu died on a fatal signal and the kernel wrote a + // process core via core_pattern. Pick it up (compress into the + // vault) while DomainId still holds the dead qemu's pid, before + // the teardown below zeroes it. A non-empty path means it was a + // qemu VM with a core; otherwise it's a plain container app. + if corePath := pickupProcessCore(ctx, status); corePath != "" { + status.SetErrorNow("QEMU process crashed, core dump saved") + } else { + status.SetErrorNow("one of the application's tasks has crashed - please restart application instance") + } status.State = types.BROKEN } else { //schedule for retry boot @@ -3119,6 +3188,10 @@ func handleGlobalConfigImpl(ctxArg interface{}, key string, ctx.metricInterval = metricInterval } ctx.processCloudInitMultiPart = gcp.GlobalValueBool(types.ProcessCloudInitMultiPart) + ctx.qemuProcessCore = gcp.GlobalValueBool(types.QemuProcessCore) + ctx.qemuGuestCore = gcp.GlobalValueBool(types.QemuGuestCore) + ctx.qemuPauseOnCrash = gcp.GlobalValueBool(types.QemuPauseOnCrash) + ctx.qemuGdb = gcp.GlobalValueBool(types.QemuGdb) ctx.GCInitialized = true } log.Functionf("handleGlobalConfigImpl done for %s. "+ diff --git a/pkg/pillar/cmd/domainmgr/verifystatus_test.go b/pkg/pillar/cmd/domainmgr/verifystatus_test.go index 0c824528777..b9f99d8f4d5 100644 --- a/pkg/pillar/cmd/domainmgr/verifystatus_test.go +++ b/pkg/pillar/cmd/domainmgr/verifystatus_test.go @@ -7,6 +7,7 @@ package domainmgr import ( "fmt" + "io" "os" "testing" @@ -97,10 +98,15 @@ func (m *mockKubeTask) OemWindowsLicenseKeySetup(_ *types.OemWindowsLicenseKeyIn func (m *mockKubeTask) Create(_ string, _ string, _ *types.DomainConfig) (int, error) { return 0, nil } -func (m *mockKubeTask) Start(_ string) error { return nil } -func (m *mockKubeTask) Stop(_ string, _ bool) error { return nil } -func (m *mockKubeTask) Delete(_ string) error { return nil } -func (m *mockKubeTask) Cleanup(_ string) error { return nil } +func (m *mockKubeTask) Start(_ string) error { return nil } +func (m *mockKubeTask) Stop(_ string, _ bool) error { return nil } +func (m *mockKubeTask) Delete(_ string) error { return nil } +func (m *mockKubeTask) Cleanup(_ string) error { return nil } +func (m *mockKubeTask) WatchCrash(_ string) <-chan types.DomainCrashEvent { return nil } +func (m *mockKubeTask) DumpGuestMemory(_ string, _ io.Writer) error { + return fmt.Errorf("DumpGuestMemory not supported") +} +func (m *mockKubeTask) GetDomainRunState(_ string) (string, error) { return "", nil } // --------------------------------------------------------------------------- // Minimal mock: hypervisor.Hypervisor — only Task() is called by verifyStatus diff --git a/pkg/pillar/cmd/vaultmgr/vaultmgr.go b/pkg/pillar/cmd/vaultmgr/vaultmgr.go index db74e62bd0b..056b2c777c6 100644 --- a/pkg/pillar/cmd/vaultmgr/vaultmgr.go +++ b/pkg/pillar/cmd/vaultmgr/vaultmgr.go @@ -28,6 +28,7 @@ import ( "flag" "fmt" "os" + "sync" "time" "github.com/lf-edge/eve-api/go/attest" @@ -58,10 +59,50 @@ type vaultMgrContext struct { vaultUCDone bool ps *pubsub.PubSub ucChan chan struct{} + globalConfig *types.ConfigItemValueMap + // trimMu guards trimStatus and the cached trim schedule config + // (trimCron/trimMaxSecs), all of which are read/written across the main + // goroutine and the trim goroutines. + trimMu sync.Mutex + trimStatus types.VaultTrimStatus + trimCron string + trimMaxSecs int + trimScheduleStarted bool // cli options args []string } +// refreshTrimConfig caches the current trim schedule settings from +// globalConfig under trimMu. Called from the main goroutine on every +// global config update so the trim goroutine picks up changes at runtime. +func (ctx *vaultMgrContext) refreshTrimConfig() { + ctx.trimMu.Lock() + ctx.trimCron = ctx.globalConfig.GlobalValueString(types.VaultTrimCron) + ctx.trimMaxSecs = int(ctx.globalConfig.GlobalValueInt(types.VaultTrimMaxSecs)) + ctx.trimMu.Unlock() +} + +// getTrimConfig returns the cached cron spec and per-run timeout under trimMu. +func (ctx *vaultMgrContext) getTrimConfig() (string, time.Duration) { + ctx.trimMu.Lock() + defer ctx.trimMu.Unlock() + return ctx.trimCron, time.Duration(ctx.trimMaxSecs) * time.Second +} + +// setTrimStatus stores the latest vault trim status under trimMu. +func (ctx *vaultMgrContext) setTrimStatus(s types.VaultTrimStatus) { + ctx.trimMu.Lock() + ctx.trimStatus = s + ctx.trimMu.Unlock() +} + +// getTrimStatus returns a copy of the latest vault trim status under trimMu. +func (ctx *vaultMgrContext) getTrimStatus() types.VaultTrimStatus { + ctx.trimMu.Lock() + defer ctx.trimMu.Unlock() + return ctx.trimStatus +} + // ProcessAgentSpecificCLIFlags process received CLI options func (ctxPtr *vaultMgrContext) ProcessAgentSpecificCLIFlags(flagSet *flag.FlagSet) { ctxPtr.args = flagSet.Args() @@ -178,8 +219,9 @@ func Run(ps *pubsub.PubSub, loggerArg *logrus.Logger, logArg *base.LogObject, ar // Context to pass around ctx := vaultMgrContext{ - ps: ps, - ucChan: make(chan struct{}), + ps: ps, + ucChan: make(chan struct{}), + globalConfig: types.DefaultConfigItemValueMap(), } // do we run a single command, or long-running service? @@ -293,12 +335,11 @@ func Run(ps *pubsub.PubSub, loggerArg *logrus.Logger, logArg *base.LogObject, ar ctx.defaultVaultUnlocked = true } if ctx.defaultVaultUnlocked || !tpmEnabled { - // Now that vault is unlocked, run any upgrade converter handler if needed - // In case of non-TPM platforms, we do this irrespective of - // defaultVaultUnlocked - log.Notice("Starting upgradeconverter(post-vault)") - go uc.RunPostVaultHandlers(agentName, ps, logger, log, - ctx.CLIParams().DebugOverride, ctx.ucChan) + // Now that vault is unlocked, run upgrade converter and schedule + // periodic vault trim. In case of non-TPM platforms, we do this + // irrespective of defaultVaultUnlocked. + startPostVaultReconcile(&ctx) + startVaultTrimSchedule(&ctx) } // publish vault key to Controller, if required @@ -308,6 +349,10 @@ func Run(ps *pubsub.PubSub, loggerArg *logrus.Logger, logArg *base.LogObject, ar for { select { + case change := <-subGlobalConfig.MsgChan(): + // Keep the cached trim schedule (and log level) current so the + // trim goroutine picks up config changes without a reboot. + subGlobalConfig.ProcessChange(change) case change := <-subVaultKeyFromController.MsgChan(): subVaultKeyFromController.ProcessChange(change) case <-stillRunning.C: @@ -363,6 +408,8 @@ func handleGlobalConfigImpl(ctxArg interface{}, key string, ctx.CLIParams().DebugOverride, logger) if gcp != nil { ctx.GCInitialized = true + ctx.globalConfig = gcp + ctx.refreshTrimConfig() } log.Functionf("handleGlobalConfigImpl done for %s\n", key) } @@ -378,6 +425,8 @@ func handleGlobalConfigDelete(ctxArg interface{}, key string, log.Functionf("handleGlobalConfigDelete for %s\n", key) agentlog.HandleGlobalConfig(log, ctx.subGlobalConfig, agentName, ctx.CLIParams().DebugOverride, logger) + ctx.globalConfig = types.DefaultConfigItemValueMap() + ctx.refreshTrimConfig() log.Functionf("handleGlobalConfigDelete done for %s\n", key) } @@ -515,12 +564,11 @@ func handleVaultKeyFromControllerImpl(ctxArg interface{}, key string, // Publish current status of vault getAndPublishAllVaultStatuses(ctx) - // Now that vault is unlocked, run any upgrade converter handler if needed - // The main select loop which is waiting on ucChan event, will publish - // the latest status of vault(s) once RunPostVaultHandlers is complete. - log.Notice("Starting upgradeconverter(post-vault)") - go uc.RunPostVaultHandlers(agentName, ctx.ps, logger, log, - ctx.CLIParams().DebugOverride, ctx.ucChan) + // Now that vault is unlocked, run upgrade converter and schedule periodic + // vault trim. The main select loop, waiting on ucChan, will publish the + // latest vault status once the reconcile is complete. + startPostVaultReconcile(ctx) + startVaultTrimSchedule(ctx) } func publishVaultKey(ctx *vaultMgrContext, vaultName string) error { @@ -569,11 +617,87 @@ func publishVaultKey(ctx *vaultMgrContext, vaultName string) error { return nil } +// startPostVaultReconcile runs post-unlock vault reconciliation off the main +// goroutine and signals ctx.ucChan when complete. That signal gates +// VaultStatus.ConversionComplete, which vaultmgr's waitUnsealed command — and +// therefore k3s startup in cluster-init.sh — blocks on. +// +// The vault fstrim runs first so that reclaiming a large backlog of stale +// blocks does not contend with k3s application I/O once the vault is reported +// ready. TrimVault is a no-op on non-EVE-k/non-ZFS handlers. Note that with +// VaultTrimMaxSecs set to 0 (unlimited) the trim runs to completion before +// RunPostVaultHandlers, so it can delay ConversionComplete — and therefore +// k3s startup — for as long as the trim takes; leave the timeout non-zero to +// bound this. +func startPostVaultReconcile(ctx *vaultMgrContext) { + _, trimTimeout := ctx.getTrimConfig() + log.Notice("Starting post-vault reconcile (vault trim + upgradeconverter)") + go func() { + ts := types.VaultTrimStatus{LastStartTime: time.Now()} + ctx.setTrimStatus(ts) + getAndPublishAllVaultStatuses(ctx) + if err := handler.TrimVault(trimTimeout); err != nil { + log.Errorf("TrimVault failed: %v", err) + ts.LastError = err.Error() + } + ts.LastEndTime = time.Now() + ctx.setTrimStatus(ts) + getAndPublishAllVaultStatuses(ctx) + uc.RunPostVaultHandlers(agentName, ctx.ps, logger, log, + ctx.CLIParams().DebugOverride, ctx.ucChan) + }() +} + +// startVaultTrimSchedule runs fstrim on a cron schedule for ongoing ghost +// block maintenance after the boot-time trim in startPostVaultReconcile. +// Runs fully async and does not gate ConversionComplete or k3s startup. +// TrimVault is a no-op on non-EVE-k/non-ZFS handlers. +// +// The cron spec and timeout are re-read from the cached global config on every +// tick, so a controller can retune or disable (empty spec) the schedule at +// runtime. The trimScheduleStarted guard ensures at most one ticker goroutine +// even if the unlock path runs more than once. +func startVaultTrimSchedule(ctx *vaultMgrContext) { + ctx.trimMu.Lock() + if ctx.trimScheduleStarted { + ctx.trimMu.Unlock() + return + } + ctx.trimScheduleStarted = true + ctx.trimMu.Unlock() + + go func() { + ticker := time.NewTicker(time.Minute) + defer ticker.Stop() + var lastFired time.Time + for t := range ticker.C { + cronSpec, timeout := ctx.getTrimConfig() + if cronSpec == "" { + continue + } + if types.CronShouldFire(cronSpec, t, &lastFired) { + log.Noticef("startVaultTrimSchedule: scheduled trim starting") + ts := types.VaultTrimStatus{LastStartTime: time.Now()} + ctx.setTrimStatus(ts) + getAndPublishAllVaultStatuses(ctx) + if err := handler.TrimVault(timeout); err != nil { + log.Errorf("startVaultTrimSchedule: scheduled trim: %v", err) + ts.LastError = err.Error() + } + ts.LastEndTime = time.Now() + ctx.setTrimStatus(ts) + getAndPublishAllVaultStatuses(ctx) + } + } + }() +} + func getAndPublishAllVaultStatuses(ctx *vaultMgrContext) { statuses := handler.GetVaultStatuses() + trimStatus := ctx.getTrimStatus() for _, status := range statuses { - // adjust ConversionComplete field with information from context status.ConversionComplete = ctx.vaultUCDone + status.TrimStatus = trimStatus publishVaultStatus(ctx, *status) } } diff --git a/pkg/pillar/cmd/zedkube/zedkube.go b/pkg/pillar/cmd/zedkube/zedkube.go index cdd23c078cd..3799c0251d0 100644 --- a/pkg/pillar/cmd/zedkube/zedkube.go +++ b/pkg/pillar/cmd/zedkube/zedkube.go @@ -134,6 +134,8 @@ type zedkube struct { longhornDiskReservedSet bool // longhornSnapshotSet is true once the desired recurring snapshot interval has been applied longhornSnapshotSet bool + // longhornDrainPolicySet is true once the desired node-drain-policy has been applied + longhornDrainPolicySet bool // Stuck-Pending-VMI detector state (keyed by app UUID string). // vmiPendingSince: first time we observed a Pending VMI with a Running @@ -761,6 +763,8 @@ func Run(ps *pubsub.PubSub, loggerArg *logrus.Logger, logArg *base.LogObject, ar zedkubeCtx.reconcileSRIOVDevicePlugin(&aa) } } + zedkubeCtx.applyLonghornNodeDrainPolicy() + kubeCfgTimer = time.NewTimer(kubeCfgInterval * time.Second) // Timer 5: leader-only safety-net re-evaluation of the stale-master @@ -948,12 +952,21 @@ func handleGlobalConfigImpl(ctxArg interface{}, key string, z.longhornSnapshotSet = false } + newDrainPolicy := newConfigItemValueMap.GlobalValueString(types.LonghornNodeDrainPolicy) + existingDrainPolicy := currentConfigItemValueMap.GlobalValueString(types.LonghornNodeDrainPolicy) + if newDrainPolicy != existingDrainPolicy { + log.Functionf("handleGlobalConfigImpl: LonghornNodeDrainPolicy changed %q -> %q", + existingDrainPolicy, newDrainPolicy) + z.longhornDrainPolicySet = false + } + z.globalConfig = newConfigItemValueMap z.applyLonghornDiskReserved() z.handleVmiDescheduleEventsOverride(newConfigItemValueMap) z.applyLonghornRecurringSnapshot() + z.applyLonghornNodeDrainPolicy() } log.Functionf("handleGlobalConfigImpl(%s): done", key) } @@ -996,6 +1009,21 @@ func (z *zedkube) applyLonghornRecurringSnapshot() { z.longhornSnapshotSet = applied } +// applyLonghornNodeDrainPolicy sets the cluster-wide node-drain-policy Longhorn setting. +// It is a no-op if already applied. Callers should retry until longhornDrainPolicySet is true. +func (z *zedkube) applyLonghornNodeDrainPolicy() { + if z.longhornDrainPolicySet { + return + } + policy := z.globalConfig.GlobalValueString(types.LonghornNodeDrainPolicy) + applied, err := kubeapi.SetLonghornNodeDrainPolicy(policy) + if err != nil { + log.Errorf("applyLonghornNodeDrainPolicy: %v", err) + return + } + z.longhornDrainPolicySet = applied +} + func handleK3sConfigOverrideChanged(currentGcp *types.ConfigItemValueMap, newGcp *types.ConfigItemValueMap) { oldVal := currentGcp.GlobalValueString(types.K3sConfigOverride) newVal := newGcp.GlobalValueString(types.K3sConfigOverride) diff --git a/pkg/pillar/cmd/zfsmanager/handlepooltrim.go b/pkg/pillar/cmd/zfsmanager/handlepooltrim.go new file mode 100644 index 00000000000..7f5a188afbc --- /dev/null +++ b/pkg/pillar/cmd/zfsmanager/handlepooltrim.go @@ -0,0 +1,88 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package zfsmanager + +import ( + "time" + + "github.com/lf-edge/eve/pkg/pillar/base" + "github.com/lf-edge/eve/pkg/pillar/types" +) + +// refreshTrimConfig caches the current ZFSPoolTrimCron from globalConfig under +// trimMu. Called from the main goroutine on every global config update so the +// pool trim goroutine picks up schedule changes at runtime. +func (ctx *zfsContext) refreshTrimConfig() { + ctx.trimMu.Lock() + ctx.trimCron = ctx.globalConfig.GlobalValueString(types.ZFSPoolTrimCron) + ctx.trimMu.Unlock() +} + +// getTrimCron returns the cached pool trim cron spec under trimMu. +func (ctx *zfsContext) getTrimCron() string { + ctx.trimMu.Lock() + defer ctx.trimMu.Unlock() + return ctx.trimCron +} + +// setTrimStart records the start time of the most recent pool trim under +// trimMu. +func (ctx *zfsContext) setTrimStart(t time.Time) { + ctx.trimMu.Lock() + ctx.trimStatus.LastStartTime = t + ctx.trimMu.Unlock() +} + +// getTrimStatus returns a copy of the latest pool trim status under trimMu. +func (ctx *zfsContext) getTrimStatus() types.PoolTrimStatus { + ctx.trimMu.Lock() + defer ctx.trimMu.Unlock() + return ctx.trimStatus +} + +// runPoolTrimSchedule issues a boot-time zpool trim immediately on start, then +// continues on the ZFSPoolTrimCron schedule. Both run fully async and do not +// gate any startup path. zpool trim initiates background NVMe I/O and returns +// immediately; progress is visible via `zpool status persist`. Only applies to +// EVE-k ZFS nodes; a no-op otherwise. +// +// The cron spec is re-read from the cached global config on every tick, so a +// controller can retune or disable (empty spec) the schedule at runtime; the +// ticker keeps running while the spec is empty so it can be re-enabled. +func runPoolTrimSchedule(ctx *zfsContext) { + if !base.IsHVTypeKube() { + return + } + go func() { + log.Noticef("runPoolTrimSchedule: boot-time demand trim starting") + runZpoolTrim(ctx) + + ticker := time.NewTicker(time.Minute) + defer ticker.Stop() + var lastFired time.Time + for t := range ticker.C { + cronSpec := ctx.getTrimCron() + if cronSpec == "" { + continue + } + if types.CronShouldFire(cronSpec, t, &lastFired) { + log.Noticef("runPoolTrimSchedule: scheduled trim starting") + runZpoolTrim(ctx) + } + } + }() +} + +func runZpoolTrim(ctx *zfsContext) { + ctx.setTrimStart(time.Now()) + collectAndPublishStorageStatus(ctx) + out, err := base.Exec(log, types.ZPoolBinary, "trim", types.PersistPool). + CombinedOutput() + if err != nil { + log.Errorf("runPoolTrimSchedule: zpool trim %s: %v (%s)", + types.PersistPool, err, out) + return + } + log.Noticef("runPoolTrimSchedule: zpool trim %s initiated", types.PersistPool) +} diff --git a/pkg/pillar/cmd/zfsmanager/zfsmanager.go b/pkg/pillar/cmd/zfsmanager/zfsmanager.go index 47e320d8db0..c701def05c3 100644 --- a/pkg/pillar/cmd/zfsmanager/zfsmanager.go +++ b/pkg/pillar/cmd/zfsmanager/zfsmanager.go @@ -59,6 +59,11 @@ type zfsContext struct { zfsIterLock sync.Mutex globalConfig *types.ConfigItemValueMap GCInitialized bool + // trimMu guards trimStatus and the cached trimCron, which are read from + // the pool trim goroutine and written from the main/publisher goroutines. + trimMu sync.Mutex + trimStatus types.PoolTrimStatus + trimCron string } // Run - an zfs run @@ -200,6 +205,8 @@ func Run(ps *pubsub.PubSub, loggerArg *logrus.Logger, logArg *base.LogObject, ar go processDisksTask(ctxPtr) + runPoolTrimSchedule(ctxPtr) + go deviceWatcher(ctxPtr) go storageStatusPublisher(ctxPtr) @@ -432,6 +439,7 @@ func handleGlobalConfigImpl(ctxArg interface{}, key string, maybeUpdateConfigItems(ctx, gcp) ctx.globalConfig = gcp ctx.GCInitialized = true + ctx.refreshTrimConfig() } log.Functionf("handleGlobalConfigImpl done for %s", key) } @@ -448,6 +456,7 @@ func handleGlobalConfigDelete(ctxArg interface{}, key string, agentlog.HandleGlobalConfig(log, ctx.subGlobalConfig, agentName, ctx.CLIParams().DebugOverride, logger) *ctx.globalConfig = *types.DefaultConfigItemValueMap() + ctx.refreshTrimConfig() log.Functionf("handleGlobalConfigDelete done for %s", key) } diff --git a/pkg/pillar/cmd/zfsmanager/zfsstoragestatus.go b/pkg/pillar/cmd/zfsmanager/zfsstoragestatus.go index 7938ec8a69f..b5d2ab96a64 100644 --- a/pkg/pillar/cmd/zfsmanager/zfsstoragestatus.go +++ b/pkg/pillar/cmd/zfsmanager/zfsstoragestatus.go @@ -184,6 +184,7 @@ func collectAndPublishStorageStatus(ctxPtr *zfsContext) { status.CompressionRatio = compressratio status.CountZvols = countZvolume status.StorageState = storageState + status.TrimStatus = ctxPtr.getTrimStatus() if err := ctxPtr.storageStatusPub.Publish(status.Key(), *status); err != nil { log.Errorf("error in publishing of storageStatus: %s", err) } diff --git a/pkg/pillar/go.mod b/pkg/pillar/go.mod index ae83d5d9a4a..a8b6292da31 100644 --- a/pkg/pillar/go.mod +++ b/pkg/pillar/go.mod @@ -134,7 +134,7 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/josharian/native v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/compress v1.18.0 github.com/kr/fs v0.1.0 // indirect github.com/leodido/go-urn v1.2.4 // indirect github.com/lithammer/shortuuid/v4 v4.2.0 // indirect diff --git a/pkg/pillar/hypervisor/containerd.go b/pkg/pillar/hypervisor/containerd.go index b910808e27e..3ba259fb4af 100644 --- a/pkg/pillar/hypervisor/containerd.go +++ b/pkg/pillar/hypervisor/containerd.go @@ -5,6 +5,7 @@ package hypervisor import ( "fmt" + "io" "math" "os" "path/filepath" @@ -226,6 +227,23 @@ func (ctx ctrdContext) Annotations(domainName string) (map[string]string, error) return ctx.ctrdClient.CtrGetAnnotations(ctrdCtx, domainName) } +// WatchCrash: containers have no VM-crash (mode A) concept; the default is no +// crash channel. KvmContext overrides this. Also the embedded default for the +// kubevirt and xen contexts. +func (ctx ctrdContext) WatchCrash(_ string) <-chan types.DomainCrashEvent { + return nil +} + +// DumpGuestMemory is a KVM/QMP-only primitive; unsupported for plain containers. +func (ctx ctrdContext) DumpGuestMemory(_ string, _ io.Writer) error { + return fmt.Errorf("DumpGuestMemory not supported by this hypervisor") +} + +// GetDomainRunState has no meaning for a container task. +func (ctx ctrdContext) GetDomainRunState(_ string) (string, error) { + return "", nil +} + func (ctx ctrdContext) Info(domainName string) (int, types.SwState, error) { ctrdCtx, done := ctx.ctrdClient.CtrNewUserServicesCtx() defer done() diff --git a/pkg/pillar/hypervisor/kvm.go b/pkg/pillar/hypervisor/kvm.go index f803f74ec01..339479a434d 100644 --- a/pkg/pillar/hypervisor/kvm.go +++ b/pkg/pillar/hypervisor/kvm.go @@ -14,6 +14,7 @@ import ( "os" "path/filepath" "runtime" + "sort" "strconv" "strings" "sync/atomic" @@ -28,6 +29,7 @@ import ( "github.com/lf-edge/eve/pkg/pillar/types" fileutils "github.com/lf-edge/eve/pkg/pillar/utils/file" procutils "github.com/lf-edge/eve/pkg/pillar/utils/proc" + "github.com/opencontainers/runtime-spec/specs-go" uuid "github.com/satori/go.uuid" "github.com/sirupsen/logrus" "golang.org/x/sys/unix" @@ -58,6 +60,18 @@ const ( // publish a UEFI GraphicsOutputProtocol, so there is no pre-OS // framebuffer (UEFI shell, GRUB, Windows boot logo go to nothing). igdRomPath = "/usr/lib/xen/boot/igd.rom" + + // qemuProcessCoreRlimitBytes caps the qemu process core the kernel writes + // on a fatal signal (debug.qemu.process.core). It bounds the raw, + // unquota'd on-disk file; 1 GiB is ample for a qemu core without guest RAM + // (the default) and small enough to bound a crash loop before pillar + // compresses and rotates it. + qemuProcessCoreRlimitBytes = 1 << 30 + + // qemuTraceDir holds per-VM simpletrace logs, in the encrypted vault + // alongside crash dumps (they can leak addresses / secrets). Available only + // after vault unlock, which is fine since domains run post-unlock. + qemuTraceDir = "/persist/vault/qemu-trace/" ) var ( @@ -117,7 +131,7 @@ const qemuGlobalConfTemplate = `# This file is automatically generated by domain [machine] type = "{{.Machine}}" - dump-guest-core = "off" + dump-guest-core = "{{if .DumpGuestCore}}on{{else}}off{{end}}" {{- if eq .Machine "virt" }} accel = "kvm:tcg" gic-version = "host" @@ -498,6 +512,9 @@ const qemuPCIPassthruTemplate = ` {{- if .Romfile }} romfile = "{{.Romfile}}" {{- end}} +{{- if .NoMmap }} + x-no-mmap = "on" +{{- end}} ` const qemuSerialTemplate = ` @@ -563,6 +580,7 @@ type tQemuGlobalConfContext struct { BootLoaderSettingsFile string BootOrder string // Boot order: "usb" (prioritize USB), "nousb" (deprioritize USB), or "" (default) EFIDebug bool // true when debug.enable.efi is set; wires isa-debugcon at 0x402 to capture OVMF DEBUG() output + DumpGuestCore bool // true when debug.qemu.dump.guest.core is set; includes guest RAM in a qemu process core (mode B) HasIntelIGPU bool // true when Intel iGPU is in the passthrough list; moves USB root port from slot 0x2 to 0x1b IgpuLpcDeviceID string // optional q35 ICH9-LPC device ID spoof for proprietary IntelGopDriver PCH whitelist (e.g. "0xa082"); empty = QEMU default 0x2918 types.DomainConfig @@ -583,6 +601,7 @@ type tQemuPCIPassthruContext struct { Romfile string // path to GOP option-ROM (proprietary or bundled), empty = no romfile= line Bus string Addr string + NoMmap bool // x-no-mmap=on: trap all BAR access instead of mmap (debug; iGPU only) } // Context for qemuPCIeRootPortTemplate. @@ -928,6 +947,19 @@ func (ctx KvmContext) Setup(status types.DomainStatus, config types.DomainConfig "-readconfig", file.Name(), "-pidfile", kvmStateDir+domainName+"/pid") + if globalConfig != nil { + if events := globalConfig.GlobalValueString(types.QemuTraceEvents); events != "" { + if traceArgs, err := setupQemuTrace(domainName, events); err != nil { + logrus.Warnf("qemu tracing requested for %s but setup failed: %v", domainName, err) + } else { + args = append(args, traceArgs...) + } + } + if globalConfig.GlobalValueBool(types.QemuGdb) { + args = append(args, "-gdb", "unix:"+gdbSocketPath(domainName)+",server=on,wait=off") + } + } + // Add CPUs affinity as a parameter to qemu. // It's not supported to be configured in the .ini file so we need to add it here. // The arguments are in the format of: -object thread-context,id=tc1,cpu-affinity=0-1,cpu-affinity=6-7 @@ -960,6 +992,27 @@ func (ctx KvmContext) Setup(status types.DomainStatus, config types.DomainConfig spec.Get().Process.Args = args logrus.Infof("Hypervisor args: %v", args) + // Enable a bounded kernel core dump of the qemu process on a fatal signal + // (SIGBUS/SIGSEGV/SIGABRT — the "mode B" class). The host + // core_pattern (installed by domainmgr) points at the encrypted vault; a + // non-pipe core_pattern only dumps a process whose RLIMIT_CORE is non-zero, + // so raising it here targets qemu alone. The cap bounds the raw file that + // the kernel writes with no quota of its own; pillar compresses and rotates + // it afterwards. Guest RAM is excluded from this core unless + // debug.qemu.process.core.guest.ram is set (the dump-guest-core machine + // prop), which keeps the typical process core small. Note the cap is fixed + // even when that debug knob is on: for a VM with more than ~1 GiB RAM the + // process core is truncated at the rlimit, so use the (full, streamed) guest + // core for guest RAM; the guest-RAM process core is a best-effort debug aid. + if globalConfig != nil && globalConfig.GlobalValueBool(types.QemuProcessCore) { + p := spec.Get().Process + p.Rlimits = append(p.Rlimits, specs.POSIXRlimit{ + Type: "RLIMIT_CORE", + Hard: qemuProcessCoreRlimitBytes, + Soft: qemuProcessCoreRlimitBytes, + }) + } + spec.GrantFullAccessToDevices() if err := spec.CreateContainer(true); err != nil { @@ -969,6 +1022,97 @@ func (ctx KvmContext) Setup(status types.DomainStatus, config types.DomainConfig return nil } +// gdbSocketPath is the per-domain gdbstub UNIX socket (debug.qemu.gdb, and the +// pause-on-crash hold). Under kvmStateDir so it is cleaned with the domain. +func gdbSocketPath(domainName string) string { + return kvmStateDir + domainName + "/gdb" +} + +// qemuTracePresets maps @ shorthand tokens to canonical qemu trace-event +// globs, so operators can request a curated set without memorizing event names. +// A debug.qemu.trace.events value may mix @presets and bare event names. +var qemuTracePresets = map[string][]string{ + // VFIO + intel-iommu DMA-mapping flux: IOTLB invalidations, the + // unmap-all-then-rebuild replay path (the unmap/rebuild race), per-DMA-region add/del. + "iommu": { + "vtd_inv_desc", + "vtd_iommu_replay", + "vtd_iotlb_*", + "vtd_iommu_translate", + "vfio_iommu_map", + "vfio_iommu_unmap", + }, + // PCI BAR-mapping flux + PM transitions: the vfio-pci <-> KVM EPT race + // behind KVM_RUN -EFAULT on the first MMIO after a PM gap, plus the + // vfio-mmap-fault angle (the qemu-SIGBUS angle). + "barmap": { + "vfio_listener_region_*", + "vfio_pci_write_config", + "vfio_pci_read_config", + "vfio_pci_cmd_*", + "vfio_region_*", + "vfio_quirk_*", + "memory_region_*", + "pci_update_mappings_*", + }, + // Device lifecycle distinct from DMA/BAR: interrupts (INTx/MSI/MSI-X), + // reset (FLR/PM/hot-reset), and display/EDID — for device-init / reset / + // IRQ / scanout problems (Code 43, missing IRQs, scanout corruption). + "vfio": { + "vfio_intx_*", + "vfio_msi_*", + "vfio_msix_*", + "vfio_pci_reset*", + "vfio_device_reset_handler", + "vfio_display_*", + }, +} + +// setupQemuTrace materializes the debug.qemu.trace.events list (a CSV of event +// names/globs and/or @ tokens) into a newline-separated events file and +// returns the qemu argv fragment enabling simpletrace to a per-VM binary log in +// the vault. Timestamped so a crash dump pairs with its trace by proximity. +func setupQemuTrace(domainName, traceEvents string) ([]string, error) { + if err := os.MkdirAll(qemuTraceDir, 0700); err != nil { + return nil, fmt.Errorf("mkdir %s: %w", qemuTraceDir, err) + } + eventsFile := kvmStateDir + domainName + "/qemu-trace-events" + var events []string + for _, tok := range strings.Split(traceEvents, ",") { + tok = strings.TrimSpace(tok) + if tok == "" { + continue + } + if strings.HasPrefix(tok, "@") { + preset, ok := qemuTracePresets[tok[1:]] + if !ok { + return nil, fmt.Errorf("unknown trace preset %q (known: %s)", + tok, strings.Join(qemuTracePresetNames(), ", ")) + } + events = append(events, preset...) + } else { + events = append(events, tok) + } + } + if len(events) == 0 { + return nil, fmt.Errorf("no trace events after expansion") + } + if err := os.WriteFile(eventsFile, []byte(strings.Join(events, "\n")+"\n"), 0644); err != nil { + return nil, fmt.Errorf("write %s: %w", eventsFile, err) + } + traceFile := fmt.Sprintf("%s%s.%s.trace", qemuTraceDir, domainName, time.Now().UTC().Format("20060102-150405")) + return []string{"-trace", "events=" + eventsFile + ",file=" + traceFile}, nil +} + +func qemuTracePresetNames() []string { + names := make([]string, 0, len(qemuTracePresets)) + for k := range qemuTracePresets { + names = append(names, "@"+k) + } + sort.Strings(names) + return names +} + // Coalesce per-app `EnableVncShimVM` flag and global `debug.enable.vnc.shim.vm` // debug flag, making sure we don't activate VNC for shim VM if VNC for // this application is disabled. @@ -1330,6 +1474,7 @@ type pciAssignmentsTemplateFiller struct { file io.Writer domainUUID string gopRomFilename string // value of igpu.gop config (basename) — empty = use bundled OSS ROM + igpuNoMmap bool // debug.qemu.igpu.no.mmap: add x-no-mmap=on to the iGPU vfio-pci device } func (f *pciAssignmentsTemplateFiller) pciEBridge(pciID int, pciWOFunction string) error { @@ -1387,6 +1532,14 @@ func (f *pciAssignmentsTemplateFiller) do(pciAssignments []pciDevice) error { // in qemuGlobalConfTemplate above. pciPTContext.Bus = "pcie.0" pciPTContext.Addr = "0x2" + // Debug knob: force all iGPU BAR access to trap (no mmap) so the + // guest's MMIO writes become visible in the qemu trace. + pciPTContext.NoMmap = f.igpuNoMmap + if f.igpuNoMmap { + logrus.Warnf("iGPU passthrough: domain %s x-no-mmap=on "+ + "(debug.qemu.igpu.no.mmap) — traps all iGPU BAR access, "+ + "large performance cost", f.domainUUID) + } logrus.Infof("iGPU passthrough: domain %s placing %s at guest BDF 00:02.0", f.domainUUID, pa.ioBundle.PciLong) if err := tQemuPCIPassthru.Execute(f.file, pciPTContext); err != nil { @@ -1595,11 +1748,13 @@ func (ctx KvmContext) CreateDomConfig(domainName string, bootOrder := bootOrderToFwCfgString(config.BootOrder) hasIntelIGPU := detectIntelIGPU(config.IoAdapterList, aa) - var efiDebug bool + var efiDebug, dumpGuestCore, igpuNoMmap bool var gopRomFilename string if globalConfig != nil { efiDebug = globalConfig.GlobalValueBool(types.EnableEFIDebug) + dumpGuestCore = globalConfig.GlobalValueBool(types.QemuProcessCoreGuestRAM) gopRomFilename = globalConfig.GlobalValueString(types.IGPUGOPFile) + igpuNoMmap = globalConfig.GlobalValueBool(types.QemuIgpuNoMmap) } // id.json (LPC device ID spoof) only applies on the proprietary GOP @@ -1626,6 +1781,7 @@ func (ctx KvmContext) CreateDomConfig(domainName string, BootLoaderSettingsFile: bootLoaderSettingsFile, BootOrder: bootOrder, EFIDebug: efiDebug, + DumpGuestCore: dumpGuestCore, HasIntelIGPU: hasIntelIGPU, IgpuLpcDeviceID: igpuLpcID, DomainConfig: config, @@ -1778,6 +1934,7 @@ func (ctx KvmContext) CreateDomConfig(domainName string, file: file, domainUUID: config.UUIDandVersion.UUID.String(), gopRomFilename: gopRomFilename, + igpuNoMmap: igpuNoMmap, } err = pciAssignmentsFiller.do(pciAssignments) if err != nil { @@ -1872,7 +2029,7 @@ func (ctx KvmContext) Start(domainName string) error { logrus.Debugf("starting qmpEventHandler") logrus.Infof("Creating %s at %s", "qmpEventHandler", agentlog.GetMyStack()) - go qmpEventHandler(getQmpListenerSocket(domainName), GetQmpExecutorSocket(domainName)) + go qmpEventHandler(getQmpListenerSocket(domainName), GetQmpExecutorSocket(domainName), domainName) annotations, err := ctx.ctrdContext.Annotations(domainName) if err != nil { @@ -2036,9 +2193,31 @@ func (ctx KvmContext) Cleanup(domainName string) error { return fmt.Errorf("failed to cleanup OVMF settings for domain %s: %v", domainName, err) } } + // Drop the crash channel so a restarted VM with the same name gets a fresh + // one (the qmpEventHandler goroutine for the old qemu has already exited on + // socket close). + crashRegistry.forget(domainName) return nil } +// WatchCrash returns the channel on which the domain's QMP monitor reports a +// mode-A crash (STOP -> internal-error). domainmgr consumes it to run +// capture-first-then-policy. +func (ctx KvmContext) WatchCrash(domainName string) <-chan types.DomainCrashEvent { + return crashRegistry.watch(domainName) +} + +// DumpGuestMemory streams the guest's physical RAM as an ELF core to w. +func (ctx KvmContext) DumpGuestMemory(domainName string, w io.Writer) error { + return execDumpGuestMemoryStream(GetQmpExecutorSocket(domainName), w) +} + +// GetDomainRunState returns the raw QMP run-state, the pollable backstop for a +// missed WatchCrash event. +func (ctx KvmContext) GetDomainRunState(domainName string) (string, error) { + return readQemuRunState(GetQmpExecutorSocket(domainName)) +} + // PCIReserve reserves a PCI device func (ctx KvmContext) PCIReserve(long string) error { return PCIReserveGeneric(long) diff --git a/pkg/pillar/hypervisor/kvm_test.go b/pkg/pillar/hypervisor/kvm_test.go index cbcf08e4276..cfd8d2cc678 100644 --- a/pkg/pillar/hypervisor/kvm_test.go +++ b/pkg/pillar/hypervisor/kvm_test.go @@ -7,6 +7,7 @@ import ( "os" "os/exec" "regexp" + "strings" "testing" "github.com/google/go-cmp/cmp" @@ -3381,3 +3382,43 @@ func TestDecideKvmState(t *testing.T) { }) } } + +// debug.qemu.process.core.guest.ram toggles the qemu `dump-guest-core` machine +// property, which decides whether a qemu process core includes guest RAM. +func TestCreateDomConfigProcessCoreGuestRAM(t *testing.T) { + id, err := uuid.NewV4() + if err != nil { + t.Fatalf("NewV4 failed: %v", err) + } + config := types.DomainConfig{ + UUIDandVersion: types.UUIDandVersion{UUID: id, Version: "1.0"}, + VmConfig: types.VmConfig{Kernel: "/boot/kernel", Memory: 1024 * 1024, VCpus: 1}, + } + aa := &types.AssignableAdapters{Initialized: true} + + render := func(t *testing.T, guestRAM bool) string { + conf, err := os.CreateTemp("/tmp", "config") + if err != nil { + t.Fatalf("CreateTemp: %v", err) + } + defer os.Remove(conf.Name()) + gc := types.DefaultConfigItemValueMap() + gc.SetGlobalValueBool(types.QemuProcessCoreGuestRAM, guestRAM) + if err := kvmIntel.CreateDomConfig(DefaultDomainName, config, types.DomainStatus{}, + nil, aa, gc, swtpmCtrlSock, conf); err != nil { + t.Fatalf("CreateDomConfig failed: %v", err) + } + out, err := os.ReadFile(conf.Name()) + if err != nil { + t.Fatalf("read: %v", err) + } + return string(out) + } + + if got := render(t, false); !strings.Contains(got, `dump-guest-core = "off"`) { + t.Fatalf("guest.ram off: expected dump-guest-core off, got:\n%s", got) + } + if got := render(t, true); !strings.Contains(got, `dump-guest-core = "on"`) { + t.Fatalf("guest.ram on: expected dump-guest-core on, got:\n%s", got) + } +} diff --git a/pkg/pillar/hypervisor/null.go b/pkg/pillar/hypervisor/null.go index 31ed451d97a..bf4bae69215 100644 --- a/pkg/pillar/hypervisor/null.go +++ b/pkg/pillar/hypervisor/null.go @@ -5,6 +5,7 @@ package hypervisor import ( "fmt" + "io" "os" "github.com/lf-edge/eve/pkg/pillar/types" @@ -132,6 +133,21 @@ func (ctx nullContext) Cleanup(_ string) error { return nil } +// WatchCrash: the null hypervisor has no VM-crash notion. +func (ctx nullContext) WatchCrash(_ string) <-chan types.DomainCrashEvent { + return nil +} + +// DumpGuestMemory is unsupported for the null hypervisor. +func (ctx nullContext) DumpGuestMemory(_ string, _ io.Writer) error { + return fmt.Errorf("DumpGuestMemory not supported by null hypervisor") +} + +// GetDomainRunState is unsupported for the null hypervisor. +func (ctx nullContext) GetDomainRunState(_ string) (string, error) { + return "", nil +} + func (ctx nullContext) Info(domainName string) (int, types.SwState, error) { if dom, found := ctx.doms[domainName]; found { logrus.Infof("Null Domain %s is %v and has the following config %s\n", domainName, dom.state, dom.config) diff --git a/pkg/pillar/hypervisor/qmp.go b/pkg/pillar/hypervisor/qmp.go index 50834529ebc..659c54dce12 100644 --- a/pkg/pillar/hypervisor/qmp.go +++ b/pkg/pillar/hypervisor/qmp.go @@ -206,7 +206,7 @@ func getQemuStatus(socket string) (types.SwState, error) { return state, errs } -func qmpEventHandler(listenerSocket, executorSocket string) { +func qmpEventHandler(listenerSocket, executorSocket, domainName string) { monitor, err := qmp.NewSocketMonitor("unix", listenerSocket, sockTimeout) if err != nil { logrus.Errorf("qmpEventHandler: Exception while getting monitor of listenerSocket: %s. %s", listenerSocket, err.Error()) @@ -240,8 +240,23 @@ func qmpEventHandler(listenerSocket, executorSocket string) { if err := execQuit(executorSocket); err != nil { logrus.Errorf("qmpEventHandler: Exception while quitting domain with socket: %s. %s", executorSocket, err.Error()) } + case "STOP": + // STOP carries no reason; query-status distinguishes an operator + // pause ("paused") from a KVM_RUN -EFAULT crash ("internal-error", + // mode A). On a crash, hand an abstract event to domainmgr, which + // owns capture-first-then-policy — this goroutine stays a + // translator and never dumps or tears down. + runState, err := readQemuRunState(executorSocket) + if err != nil { + logrus.Errorf("qmpEventHandler: STOP for %s but query-status failed: %v", domainName, err) + continue + } + logrus.Infof("qmpEventHandler: STOP event runState=%q for %s", runState, domainName) + if runState == "internal-error" { + crashRegistry.emit(domainName, runState) + } default: - //Not handling the following events: RESUME, NIC_RX_FILTER_CHANGED, RTC_CHANGE, POWERDOWN, STOP + //Not handling the following events: RESUME, NIC_RX_FILTER_CHANGED, RTC_CHANGE, POWERDOWN logrus.Warnf("qmpEventHandler: Unhandled event: %s from QMP socket: %s", event.Event, listenerSocket) } } diff --git a/pkg/pillar/hypervisor/qmp_crash.go b/pkg/pillar/hypervisor/qmp_crash.go new file mode 100644 index 00000000000..a5248435f12 --- /dev/null +++ b/pkg/pillar/hypervisor/qmp_crash.go @@ -0,0 +1,188 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package hypervisor + +import ( + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "sync" + "time" + + "github.com/lf-edge/eve/pkg/pillar/types" + "golang.org/x/sys/unix" +) + +// dumpStreamTimeout bounds how long we wait for qemu to stream a guest core +// through the FIFO. The capture runs in domainmgr's per-domain goroutine (never +// the main loop), so a hang here cannot stall the watchdog; the bound just +// stops a stuck dump from holding the domain frozen forever. +const dumpStreamTimeout = 15 * time.Minute + +// The crash-channel registry lets the per-domain QMP monitor goroutine (a +// translator, not an actor) hand mode-A crash notifications to domainmgr, which +// owns crash policy. Keyed by domain name; created lazily by WatchCrash and +// dropped by the KVM Cleanup. We never close the channels — a closed channel +// would busy-loop domainmgr's select; dropping the map entry is enough because +// emitCrash only sends to a still-registered channel. +var crashRegistry = newCrashRegistry() + +type crashRegistryT struct { + mu sync.Mutex + chans map[string]chan types.DomainCrashEvent +} + +func newCrashRegistry() *crashRegistryT { + return &crashRegistryT{chans: map[string]chan types.DomainCrashEvent{}} +} + +// watch returns the receive end of the domain's crash channel, creating it on +// first use. Buffered (1) so a crash notification is never lost if domainmgr is +// momentarily busy. +func (r *crashRegistryT) watch(domainName string) <-chan types.DomainCrashEvent { + r.mu.Lock() + defer r.mu.Unlock() + ch, ok := r.chans[domainName] + if !ok { + ch = make(chan types.DomainCrashEvent, 1) + r.chans[domainName] = ch + } + return ch +} + +// emit delivers a crash notification if domainmgr is watching this domain. The +// non-blocking send collapses repeated internal-error events into the single +// buffered slot. +func (r *crashRegistryT) emit(domainName, runState string) { + r.mu.Lock() + ch := r.chans[domainName] + r.mu.Unlock() + if ch == nil { + return + } + select { + case ch <- types.DomainCrashEvent{RunState: runState, When: time.Now()}: + default: + } +} + +// forget drops the domain's channel (on Cleanup / VM teardown) so a later +// WatchCrash for a restarted VM gets a fresh channel. +func (r *crashRegistryT) forget(domainName string) { + r.mu.Lock() + defer r.mu.Unlock() + delete(r.chans, domainName) +} + +// readQemuRunState returns the raw QMP run-state string ("running", +// "internal-error", "paused", …). Unlike getQemuStatus it does not map to +// SwState, so it can surface internal-error — the mode-A crash signal that has +// no SwState of its own. +func readQemuRunState(socket string) (string, error) { + raw, err := execRawCmd(socket, `{ "execute": "query-status" }`, false) + if err != nil { + return "", err + } + var resp struct { + Return struct { + Status string `json:"status"` + } `json:"return"` + } + if err := json.Unmarshal(raw, &resp); err != nil { + return "", fmt.Errorf("parse query-status: %w", err) + } + return resp.Return.Status, nil +} + +// execDumpGuestMemoryStream dumps the guest's physical RAM as an ELF core and +// streams it to w, so the caller can compress and quota it on the fly with no +// full-size intermediate on disk. It uses a FIFO plus +// dump-guest-memory protocol=file:,detach=true — the vendored go-qemu QMP +// client cannot pass an fd (no SCM_RIGHTS), and detach keeps the QMP call from +// blocking for the whole (possibly minutes-long) dump. If w returns an error +// mid-stream (e.g. quota exceeded), closing the read end makes qemu's dump +// thread see EPIPE and abort. +func execDumpGuestMemoryStream(socket string, w io.Writer) error { + fifo := filepath.Join(filepath.Dir(socket), fmt.Sprintf("guestmem-%d.fifo", time.Now().UnixNano())) + if err := unix.Mkfifo(fifo, 0600); err != nil { + return fmt.Errorf("mkfifo %s: %w", fifo, err) + } + defer os.Remove(fifo) + + // Start the reader BEFORE issuing the dump command. dump-guest-memory opens + // the FIFO for writing, which blocks until a reader is present; issuing the + // command first would wedge qemu opening the FIFO — the QMP call never + // returns and we never open the reader (a deadlock). Opening the read end + // here (also blocking) rendezvous with qemu's write open, in the handler or + // the detached thread. + // readerFile lets the timeout path close the read end so the reader + // goroutine's io.Copy unblocks and stops writing to w before we return + // (w is closed by the caller; concurrent Write/Close on the zstd encoder + // would be a data race). + var readerMu sync.Mutex + var readerFile *os.File + done := make(chan error, 1) + go func() { + f, err := os.Open(fifo) // blocks until a writer (qemu) opens the FIFO + if err != nil { + done <- fmt.Errorf("open guest-core FIFO: %w", err) + return + } + readerMu.Lock() + readerFile = f + readerMu.Unlock() + defer f.Close() + _, err = io.Copy(w, f) + done <- err + }() + + cmd := fmt.Sprintf(`{ "execute": "dump-guest-memory", "arguments": { "paging": false, "detach": true, "protocol": "file:%s" } }`, fifo) + if _, err := execRawCmd(socket, cmd, true); err != nil { + // The command failed, so qemu will never open the write end; unblock the + // waiting reader by opening (and closing) the write end ourselves. + if wf, oerr := os.OpenFile(fifo, os.O_WRONLY, 0); oerr == nil { + _ = wf.Close() + } + <-done + return fmt.Errorf("dump-guest-memory: %w", err) + } + + select { + case err := <-done: + if err != nil { + return fmt.Errorf("streaming guest core: %w", err) + } + return nil + case <-time.After(dumpStreamTimeout): + // If the goroutine actually finished right at the deadline, honor its + // result instead of forcing anything. + select { + case err := <-done: + if err != nil { + return fmt.Errorf("streaming guest core: %w", err) + } + return nil + default: + } + // Still blocked — stop it before returning so it can no longer write to + // w (the caller closes w; concurrent Write/Close on the zstd encoder is a + // data race). It is either in io.Copy (readerFile set — close the read + // end) or still in os.Open waiting for qemu to open the write end + // (readerFile nil — rendezvous by opening the write end ourselves, as the + // command-error path above does, so the read open returns and the + // goroutine can exit). Then drain it. + readerMu.Lock() + rf := readerFile + readerMu.Unlock() + if rf != nil { + _ = rf.Close() + } else if wf, oerr := os.OpenFile(fifo, os.O_WRONLY, 0); oerr == nil { + _ = wf.Close() + } + <-done + return fmt.Errorf("timed out after %s streaming guest core", dumpStreamTimeout) + } +} diff --git a/pkg/pillar/kubeapi/longhornconfig.go b/pkg/pillar/kubeapi/longhornconfig.go new file mode 100644 index 00000000000..91ed1a47667 --- /dev/null +++ b/pkg/pillar/kubeapi/longhornconfig.go @@ -0,0 +1,56 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +//go:build k + +package kubeapi + +import ( + "context" + "fmt" + + "github.com/longhorn/longhorn-manager/k8s/pkg/client/clientset/versioned" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// longhornNodeDrainPolicySettingName is the name of the Longhorn Setting object for node drain policy. +const longhornNodeDrainPolicySettingName = "node-drain-policy" + +// SetLonghornNodeDrainPolicy sets the Longhorn cluster-wide node-drain-policy setting. +// Returns (true, nil) when successfully applied; (false, nil) when Longhorn is not yet available. +func SetLonghornNodeDrainPolicy(policy string) (bool, error) { + apiExists, err := longhornAPIExists() + if !apiExists && err == nil { + return false, nil + } + if err != nil { + return false, err + } + config, err := GetKubeConfig() + if err != nil { + return false, fmt.Errorf("SetLonghornNodeDrainPolicy: kubeconfig: %v", err) + } + lhClient, err := versioned.NewForConfig(config) + if err != nil { + return false, fmt.Errorf("SetLonghornNodeDrainPolicy: versioned client: %v", err) + } + lhCtx, lhCancel := context.WithTimeout(context.Background(), kubeAPITimeout) + defer lhCancel() + settings := lhClient.LonghornV1beta2().Settings(longhornNamespace) + existing, err := settings.Get(lhCtx, longhornNodeDrainPolicySettingName, metav1.GetOptions{}) + if err != nil { + if k8serrors.IsNotFound(err) { + return false, nil + } + return false, fmt.Errorf("SetLonghornNodeDrainPolicy: get: %v", err) + } + if existing.Value == policy { + return true, nil + } + existing.Value = policy + if _, err := settings.Update(lhCtx, existing, metav1.UpdateOptions{}); err != nil { + return false, fmt.Errorf("SetLonghornNodeDrainPolicy: update: %v", err) + } + return true, nil +} diff --git a/pkg/pillar/qemudump/budget.go b/pkg/pillar/qemudump/budget.go new file mode 100644 index 00000000000..469bd957b57 --- /dev/null +++ b/pkg/pillar/qemudump/budget.go @@ -0,0 +1,34 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package qemudump + +const ( + // budgetHeadroomPercent is the fraction of the compressor's available + // memory (pillar's cgroup headroom) that the zstd window(s) may claim. Kept + // small so the crash-time compressor leaves most of the headroom untouched + // — the window is anonymous, non-reclaimable memory, and blowing pillar's + // cgroup limit OOM-kills zedbox and reboots the device. + budgetHeadroomPercent = 25 + // budgetHardCapBytes is the absolute ceiling on the compressor memory + // budget, independent of headroom — a bigger window is a marginal ratio + // gain per the design. On a tight cgroup the headroom fraction (not this + // cap) is what binds; on a device with real headroom the window can grow up + // to this. + budgetHardCapBytes uint64 = 512 << 20 +) + +// ComputeBudget returns the compressor memory budget: budgetHeadroomPercent of +// availBytes, hard-capped at budgetHardCapBytes. availBytes MUST be the memory +// actually available to pillar (its cgroup headroom = limit - usage), NOT +// system-wide free RAM — the window is charged to pillar's cgroup and must fit +// its headroom or the kernel OOM-kills zedbox. Feed the result to +// ChooseWindowLog. A small or zero budget is fine — ChooseWindowLog floors the +// window at a negligible-RAM size. +func ComputeBudget(availBytes uint64) uint64 { + budget := availBytes * budgetHeadroomPercent / 100 + if budget > budgetHardCapBytes { + budget = budgetHardCapBytes + } + return budget +} diff --git a/pkg/pillar/qemudump/budget_test.go b/pkg/pillar/qemudump/budget_test.go new file mode 100644 index 00000000000..1e7307ae0d1 --- /dev/null +++ b/pkg/pillar/qemudump/budget_test.go @@ -0,0 +1,32 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package qemudump + +import "testing" + +// ComputeBudget takes a bounded fraction (25%) of the compressor's available +// memory (the smaller of pillar's cgroup headroom and system RAM), hard-capped +// at 512 MiB, so the crash-time compressor can never claim more than a safe +// slice of pillar's headroom and OOM-kill zedbox. +func TestComputeBudget(t *testing.T) { + cases := []struct { + name string + avail uint64 + want uint64 + }{ + {"lots of headroom caps at 512MiB", 8 * gib, 512 * mib}, + {"1GiB headroom uses 25%", 1 * gib, 256 * mib}, + {"512MiB headroom uses 25%", 512 * mib, 128 * mib}, + {"128MiB headroom uses 25%", 128 * mib, 32 * mib}, + {"tight cgroup, tiny budget", 40 * mib, 10 * mib}, + {"no headroom", 0, 0}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := ComputeBudget(tc.avail); got != tc.want { + t.Fatalf("ComputeBudget(%d) = %d, want %d", tc.avail, got, tc.want) + } + }) + } +} diff --git a/pkg/pillar/qemudump/compressfile_test.go b/pkg/pillar/qemudump/compressfile_test.go new file mode 100644 index 00000000000..562f9365a05 --- /dev/null +++ b/pkg/pillar/qemudump/compressfile_test.go @@ -0,0 +1,94 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package qemudump + +import ( + "bytes" + "errors" + "os" + "path/filepath" + "testing" +) + +// CompressFile turns a kernel-written raw core into a rotated, quota-enforced +// .zst under the domain's dir and removes the raw source (design doc §4.2, §7). +func TestCompressFileHappyPath(t *testing.T) { + dir := t.TempDir() + m := NewManager(generousConfig(dir)) + + raw := filepath.Join(t.TempDir(), "core.1234") + content := bytes.Repeat([]byte("QEMUCORE"), 4096) // compressible + if err := os.WriteFile(raw, content, 0600); err != nil { + t.Fatalf("seed raw: %v", err) + } + + dst, err := m.CompressFile("dom1", KindProcessCore, raw) + if err != nil { + t.Fatalf("CompressFile: %v", err) + } + if _, err := os.Stat(raw); !os.IsNotExist(err) { + t.Fatalf("raw source not removed (err=%v)", err) + } + if got := decompress(t, dst); !bytes.Equal(got, content) { + t.Fatalf("round-trip mismatch: %d vs %d bytes", len(got), len(content)) + } +} + +// If compression can't fit the quota, both the partial .zst and the raw source +// are removed — an uncompressed core must never be left in the vault. +func TestCompressFileQuotaAbortRemovesBoth(t *testing.T) { + dir := t.TempDir() + cfg := generousConfig(dir) + cfg.PerDomainQuota = 4 * kib + m := NewManager(cfg) + + raw := filepath.Join(t.TempDir(), "core.1234") + if err := os.WriteFile(raw, incompressible(1*mib), 0600); err != nil { + t.Fatalf("seed raw: %v", err) + } + + _, err := m.CompressFile("dom1", KindProcessCore, raw) + if !errors.Is(err, ErrQuotaExceeded) { + t.Fatalf("err = %v, want ErrQuotaExceeded", err) + } + if _, err := os.Stat(raw); !os.IsNotExist(err) { + t.Fatalf("raw source not removed after abort (err=%v)", err) + } + assertNoDumps(t, filepath.Join(dir, "dom1")) +} + +// PickupProcessCore finds a kernel-written raw core for a pid in staging and +// compresses it into the domain's ring, removing the raw. +func TestPickupProcessCore(t *testing.T) { + dir := t.TempDir() + m := NewManager(generousConfig(dir)) + staging := StagingDir(dir) + if err := os.MkdirAll(staging, 0700); err != nil { + t.Fatalf("mkdir: %v", err) + } + raw := filepath.Join(staging, "core-4242-1720000000") + content := bytes.Repeat([]byte("QEMU"), 4096) + if err := os.WriteFile(raw, content, 0600); err != nil { + t.Fatalf("seed: %v", err) + } + + dst, err := m.PickupProcessCore("dom1", 4242) + if err != nil { + t.Fatalf("PickupProcessCore: %v", err) + } + if dst == "" { + t.Fatalf("expected a dump path") + } + if _, err := os.Stat(raw); !os.IsNotExist(err) { + t.Fatalf("raw not removed") + } + if got := decompress(t, dst); !bytes.Equal(got, content) { + t.Fatalf("round-trip mismatch") + } + + // No core waiting for an unknown pid -> ("", nil). + if dst, err := m.PickupProcessCore("dom1", 9999); err != nil || dst != "" { + t.Fatalf("PickupProcessCore(absent) = %q, %v; want \"\", nil", dst, err) + } +} diff --git a/pkg/pillar/qemudump/corepattern.go b/pkg/pillar/qemudump/corepattern.go new file mode 100644 index 00000000000..ae153597df4 --- /dev/null +++ b/pkg/pillar/qemudump/corepattern.go @@ -0,0 +1,127 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package qemudump + +import ( + "fmt" + "os" + "path/filepath" + "strconv" + "time" +) + +// procCorePattern is the kernel knob that names where process cores are written. +const procCorePattern = "/proc/sys/kernel/core_pattern" + +// StagingDir is where the kernel drops raw qemu process cores before pillar +// correlates each to its domain, compresses it, and moves it into the domain's +// ring. It is a hidden subdir of the dump root so it never looks like a +// finished per-domain dump. +func StagingDir(dumpDir string) string { + return filepath.Join(dumpDir, ".incoming") +} + +// CorePattern is the absolute kernel core_pattern that writes raw qemu process +// cores into StagingDir, keyed by pid (%p) and time (%t) so FindCoreForPID can +// correlate a core to the domain whose qemu had that pid. It must be absolute +// so the kernel knows where to write regardless of the crashing process's CWD. +// No %e (comm): only qemu is given a non-zero RLIMIT_CORE, so every core here is +// already a qemu core. +func CorePattern(dumpDir string) string { + return filepath.Join(StagingDir(dumpDir), "core-%p-%t") +} + +// InstallCorePattern creates the staging directory and writes the core_pattern +// to procPath (pass procCorePattern in production, a temp file in tests). Call +// it only after the vault is mounted, since the staging dir lives in the vault. +func InstallCorePattern(procPath, dumpDir string) error { + if err := os.MkdirAll(StagingDir(dumpDir), 0700); err != nil { + return fmt.Errorf("qemudump: mkdir staging %s: %w", StagingDir(dumpDir), err) + } + if err := os.WriteFile(procPath, []byte(CorePattern(dumpDir)), 0644); err != nil { + return fmt.Errorf("qemudump: write core_pattern %s: %w", procPath, err) + } + return nil +} + +// InstallDefaultCorePattern installs the core_pattern at the real kernel path. +func InstallDefaultCorePattern(dumpDir string) error { + return InstallCorePattern(procCorePattern, dumpDir) +} + +// SweepStaging removes any raw cores left in the staging dir. Call it once at +// pillar startup: anything there is an orphan from a previous boot (its qemu +// pid is gone, so it can't be attributed to a domain and would otherwise linger +// unbounded). Files are only picked up promptly when a live domain dies, so a +// startup sweep cannot race an in-progress capture. A missing dir is not an +// error. +func (m *Manager) SweepStaging() error { + staging := StagingDir(m.cfg.Dir) + entries, err := os.ReadDir(staging) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return fmt.Errorf("qemudump: readdir staging %s: %w", staging, err) + } + var firstErr error + for _, e := range entries { + if e.IsDir() { + continue + } + if err := os.Remove(filepath.Join(staging, e.Name())); err != nil && firstErr == nil { + firstErr = fmt.Errorf("qemudump: sweep %s: %w", e.Name(), err) + } + } + return firstErr +} + +// SweepStagingStale removes raw cores in the staging dir whose mtime is older +// than maxAge. Unlike SweepStaging (boot-only, removes everything) this is safe +// to call at runtime: a real core is picked up promptly when its domain dies, so +// anything older than maxAge is an orphan — qemu pid already gone, a failed +// pickup, or (core_pattern is host-global) a core from a non-qemu process — that +// would otherwise linger unbounded until the next reboot. A missing dir is not +// an error. +func (m *Manager) SweepStagingStale(maxAge time.Duration) error { + staging := StagingDir(m.cfg.Dir) + entries, err := os.ReadDir(staging) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return fmt.Errorf("qemudump: readdir staging %s: %w", staging, err) + } + cutoff := time.Now().Add(-maxAge) + var firstErr error + for _, e := range entries { + if e.IsDir() { + continue + } + info, err := e.Info() + if err != nil { + continue // vanished between ReadDir and Info; skip + } + if info.ModTime().After(cutoff) { + continue // too recent: may be in-progress or awaiting prompt pickup + } + if err := os.Remove(filepath.Join(staging, e.Name())); err != nil && firstErr == nil { + firstErr = fmt.Errorf("qemudump: sweep stale %s: %w", e.Name(), err) + } + } + return firstErr +} + +// FindCoreForPID returns the path of a raw core in the staging dir written for +// pid, or "" if none. The core_pattern names files core--