From b3f0bbda8a46cc913cf89590ffd22c86e638b7ca Mon Sep 17 00:00:00 2001 From: eriknordmark Date: Tue, 28 Jul 2026 07:00:38 -0700 Subject: [PATCH 1/6] kubeapi: ignore stray longhorn-system daemonsets An EVE-k node reports cluster storage as unready, and every app volume stays in CREATING_VOLUME, whenever any DaemonSet in the longhorn-system namespace lacks a Running-and-Ready pod on this node -- including DaemonSets that Longhorn does not own and that are never expected to become ready. The readiness check iterated over every DaemonSet in the namespace and required each one to be healthy, consulting its list of expected DaemonSets only afterwards to confirm those three exist. In practice this is reached through EVE's own collect-info, which leaves a SupportBundle agent DaemonSet behind; the node then refuses to serve volumes for as long as that object exists, with no way for an operator to tell why. Restrict the per-node health requirement to the DaemonSets Longhorn is expected to run, and skip anything else sharing the namespace. Restricting the loop also removes a second false failure: a DaemonSet whose node selector legitimately excludes this node reported zero pods here and was treated as missing. Signed-off-by: eriknordmark Co-Authored-By: Claude Opus 5 --- pkg/pillar/kubeapi/kubeapi.go | 10 ++- pkg/pillar/kubeapi/longhornready_test.go | 102 +++++++++++++++++++++++ 2 files changed, 111 insertions(+), 1 deletion(-) create mode 100644 pkg/pillar/kubeapi/longhornready_test.go diff --git a/pkg/pillar/kubeapi/kubeapi.go b/pkg/pillar/kubeapi/kubeapi.go index 5e2fbfcab9e..e8488d33a8f 100644 --- a/pkg/pillar/kubeapi/kubeapi.go +++ b/pkg/pillar/kubeapi/kubeapi.go @@ -288,14 +288,22 @@ func checkLonghornReady(client kubernetes.Interface, nodeName string) error { "longhorn-csi-plugin": false, "engine-image": false, } - // Check if each daemonset is running and ready on this node + // Check if each daemonset is running and ready on this node. Only the + // expected daemonsets above gate readiness: anything else sharing the + // namespace is not ours to judge, and a permanently unready stray would + // otherwise block every volume on this node for as long as it exists. for _, lhDaemonset := range lhDaemonsets.Items { lhDsName := lhDaemonset.GetName() + expected := false for dsPrefix := range lhExpectedDaemonsets { if strings.HasPrefix(lhDsName, dsPrefix) { lhExpectedDaemonsets[dsPrefix] = true + expected = true } } + if !expected { + continue + } var labelSelectors []string for dsLabelK, dsLabelV := range lhDaemonset.Spec.Template.Labels { diff --git a/pkg/pillar/kubeapi/longhornready_test.go b/pkg/pillar/kubeapi/longhornready_test.go new file mode 100644 index 00000000000..04667444602 --- /dev/null +++ b/pkg/pillar/kubeapi/longhornready_test.go @@ -0,0 +1,102 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +//go:build k + +package kubeapi + +import ( + "testing" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" +) + +const lhTestNode = "test-node" + +func lhDaemonset(name string) *appsv1.DaemonSet { + return &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "longhorn-system"}, + Spec: appsv1.DaemonSetSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": name}}, + }, + }, + } +} + +func lhPod(dsName string, ready bool) *corev1.Pod { + phase := corev1.PodRunning + if !ready { + phase = corev1.PodPending + } + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: dsName + "-pod", + Namespace: "longhorn-system", + Labels: map[string]string{"app": dsName}, + }, + Spec: corev1.PodSpec{NodeName: lhTestNode}, + Status: corev1.PodStatus{ + Phase: phase, + ContainerStatuses: []corev1.ContainerStatus{{Ready: ready}}, + }, + } +} + +// healthyLonghornObjects returns the three daemonsets checkLonghornReady +// expects, each with a Running-and-Ready pod on lhTestNode. +func healthyLonghornObjects() []runtime.Object { + names := []string{"longhorn-manager", "longhorn-csi-plugin", "engine-image-ei-abcdef12"} + objs := make([]runtime.Object, 0, len(names)*2) + for _, n := range names { + objs = append(objs, lhDaemonset(n), lhPod(n, true)) + } + return objs +} + +func TestCheckLonghornReadyHealthy(t *testing.T) { + client := fake.NewSimpleClientset(healthyLonghornObjects()...) + if err := checkLonghornReady(client, lhTestNode); err != nil { + t.Fatalf("expected ready, got %v", err) + } +} + +// A daemonset that is not one of longhorn's own must not gate storage. EVE's +// collect-info leaves a SupportBundle agent daemonset behind in this namespace +// which never becomes ready, and it used to block every volume on the node. +func TestCheckLonghornReadyIgnoresStrayDaemonset(t *testing.T) { + objs := healthyLonghornObjects() + objs = append(objs, lhDaemonset("longhorn-support-bundle-agent")) + client := fake.NewSimpleClientset(objs...) + if err := checkLonghornReady(client, lhTestNode); err != nil { + t.Fatalf("stray daemonset must not gate readiness, got %v", err) + } +} + +func TestCheckLonghornReadyMissingExpectedDaemonset(t *testing.T) { + objs := []runtime.Object{ + lhDaemonset("longhorn-manager"), lhPod("longhorn-manager", true), + lhDaemonset("longhorn-csi-plugin"), lhPod("longhorn-csi-plugin", true), + } + client := fake.NewSimpleClientset(objs...) + err := checkLonghornReady(client, lhTestNode) + if err == nil { + t.Fatal("expected an error when engine-image is absent") + } +} + +func TestCheckLonghornReadyExpectedDaemonsetNotReady(t *testing.T) { + names := []string{"longhorn-manager", "longhorn-csi-plugin", "engine-image-ei-abcdef12"} + var objs []runtime.Object + for _, n := range names { + objs = append(objs, lhDaemonset(n), lhPod(n, n != "longhorn-manager")) + } + client := fake.NewSimpleClientset(objs...) + if err := checkLonghornReady(client, lhTestNode); err == nil { + t.Fatal("expected an error when an expected daemonset pod is not ready") + } +} From 4c7b0200f92e655f954f2bd45b1f07e93a55ba72 Mon Sep 17 00:00:00 2001 From: eriknordmark Date: Tue, 28 Jul 2026 09:11:59 -0700 Subject: [PATCH 2/6] volumemgr: report storage readiness truthfully On an EVE-k node volumemgr waits up to 40 minutes for the cluster to be able to serve a volume, and then reports Initialized regardless of how that wait ended. A node whose Longhorn or CDI never came up is therefore indistinguishable, from the outside, from a healthy one -- the only difference is a line in volumemgr's own log. Anything consuming the status, an operator inspecting it, or a test asserting on it is misled in precisely the case that matters. Report the outcome instead: Initialized now reflects whether cluster storage became usable, and a new UnmetCondition carries the gate that was still outstanding, reusing the sub-condition the kubernetes wait already computes ("longhorn not ready: ...", "kubevirt not ready: ..."). Nodes that are not EVE-k have no such gate and are Initialized from the start, as before. Volumes are unaffected either way: they are gated separately and defer and retry until storage appears. Signed-off-by: eriknordmark Co-Authored-By: Claude Opus 5 --- pkg/pillar/cmd/volumemgr/handlediskmetrics.go | 3 ++- pkg/pillar/cmd/volumemgr/volumemgr.go | 16 ++++++++++++++++ pkg/pillar/types/volumetypes.go | 10 ++++++++-- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/pkg/pillar/cmd/volumemgr/handlediskmetrics.go b/pkg/pillar/cmd/volumemgr/handlediskmetrics.go index ff71532b3ab..1c869cdc998 100644 --- a/pkg/pillar/cmd/volumemgr/handlediskmetrics.go +++ b/pkg/pillar/cmd/volumemgr/handlediskmetrics.go @@ -142,7 +142,8 @@ func generateAndPublishVolumeMgrStatus(ctx *volumemgrContext) { } st := types.VolumeMgrStatus{ Name: agentName, - Initialized: true, + Initialized: ctx.storageReady, + UnmetCondition: ctx.storageUnmet, RemainingSpace: remaining, } ctx.pubVolumeMgrStatus.Publish(st.Key(), st) diff --git a/pkg/pillar/cmd/volumemgr/volumemgr.go b/pkg/pillar/cmd/volumemgr/volumemgr.go index 496ed38dbc5..492dbd3c2c7 100644 --- a/pkg/pillar/cmd/volumemgr/volumemgr.go +++ b/pkg/pillar/cmd/volumemgr/volumemgr.go @@ -89,6 +89,13 @@ type volumemgrContext struct { gcRunning bool initGced bool // Will be marked true after initObjects are garbage collected + // storageReady is false only on an EVE-k node whose cluster storage never + // came up; storageUnmet then carries the outstanding gate. Both are + // reported through VolumeMgrStatus. Written once during startup, before + // the disk-metrics task that reads them is launched. + storageReady bool + storageUnmet string + globalConfig *types.ConfigItemValueMap GCInitialized bool vdiskGCTime uint32 // In seconds; XXX delete when OldVolumeStatus is deleted @@ -169,6 +176,9 @@ func Run(ps *pubsub.PubSub, loggerArg *logrus.Logger, logArg *base.LogObject, ar globalConfig: types.DefaultConfigItemValueMap(), persistType: persist.ReadPersistType(), hvTypeKube: base.IsHVTypeKube(), + // Only an EVE-k node has cluster storage to wait for; everywhere else + // storage is usable as soon as volumemgr is up. + storageReady: !base.IsHVTypeKube(), } agentbase.Init(&ctx, logger, log, agentName, agentbase.WithPidFile(), @@ -341,6 +351,7 @@ func Run(ps *pubsub.PubSub, loggerArg *logrus.Logger, logArg *base.LogObject, ar }) if err != nil { log.Errorf("volumemgr run: wait for kubernetes error %v", err) + ctx.storageUnmet = err.Error() } else { log.Noticef("volumemgr run: kubernetes node ready, longhorn ready") } @@ -371,9 +382,14 @@ func Run(ps *pubsub.PubSub, loggerArg *logrus.Logger, logArg *base.LogObject, ar } } storageDeadline.Stop() + ctx.storageReady = storageReady if storageReady { + ctx.storageUnmet = "" log.Noticef("volumemgr run: cluster storage (longhorn+CDI) ready") } else { + if ctx.storageUnmet == "" { + ctx.storageUnmet = "cluster storage (longhorn+CDI) not ready" + } log.Warnf("volumemgr run: timeout waiting for cluster storage; " + "volumes will defer and retry") } diff --git a/pkg/pillar/types/volumetypes.go b/pkg/pillar/types/volumetypes.go index e6750f97810..a2c74400971 100644 --- a/pkg/pillar/types/volumetypes.go +++ b/pkg/pillar/types/volumetypes.go @@ -619,8 +619,14 @@ func (status VolumeCreatePending) LogDelete(logBase *base.LogObject) { // VolumeMgrStatus : type VolumeMgrStatus struct { - Name string - Initialized bool + Name string + Initialized bool + // UnmetCondition names the readiness gate still outstanding when + // Initialized is false, e.g. "longhorn not ready: longhorn missing + // daemonset:engine-image". Empty when Initialized is true. Informational: + // it exists so an operator can tell a converging cluster from a stuck one + // without correlating agent logs. + UnmetCondition string RemainingSpace uint64 // In bytes. Takes into account "reserved" for dom0 } From c87cf0ebcc1c37307e9a7b699447b7b48afb6f4c Mon Sep 17 00:00:00 2001 From: eriknordmark Date: Sun, 2 Aug 2026 11:13:50 +0200 Subject: [PATCH 3/6] diag: show unmet cluster-storage condition An EVE-k node whose cluster storage never converged looks healthy from the console: applications that need a volume simply sit waiting, and nothing in the diag summary says why. Now that volumemgr reports that outcome, diag subscribes to VolumeMgrStatus and prints a warning naming the gate that was still outstanding when storage failed to become usable at startup. A healthy node, and any device that is not EVE-k, prints nothing extra. Signed-off-by: eriknordmark Co-Authored-By: Claude Opus 5 --- pkg/pillar/cmd/diag/diag.go | 50 ++++++++++++++++++++++++++++++++++++ pkg/pillar/docs/volumemgr.md | 2 ++ 2 files changed, 52 insertions(+) diff --git a/pkg/pillar/cmd/diag/diag.go b/pkg/pillar/cmd/diag/diag.go index 93a005e67c9..f6c91e6cb5f 100644 --- a/pkg/pillar/cmd/diag/diag.go +++ b/pkg/pillar/cmd/diag/diag.go @@ -68,6 +68,8 @@ type diagContext struct { zedagentStatus types.ZedAgentStatus subVaultStatus pubsub.Subscription vaultStatus types.VaultStatus + subVolumeMgrStatus pubsub.Subscription + volumeMgrStatus types.VolumeMgrStatus subAppInstanceSummary pubsub.Subscription appInstanceSummary types.AppInstanceSummary subAppInstanceStatus pubsub.Subscription @@ -349,6 +351,25 @@ func Run(ps *pubsub.PubSub, loggerArg *logrus.Logger, logArg *base.LogObject, ar ctx.subVaultStatus = subVaultStatus subVaultStatus.Activate() + // Look for VolumeMgrStatus from volumemgr, to show whether cluster storage + // became usable on an EVE-k node. + subVolumeMgrStatus, err := ps.NewSubscription(pubsub.SubscriptionOptions{ + AgentName: "volumemgr", + MyAgentName: agentName, + TopicImpl: types.VolumeMgrStatus{}, + Activate: false, + Ctx: &ctx, + CreateHandler: handleVolumeMgrStatusCreate, + ModifyHandler: handleVolumeMgrStatusModify, + WarningTime: warningTime, + ErrorTime: errorTime, + }) + if err != nil { + log.Fatal(err) + } + ctx.subVolumeMgrStatus = subVolumeMgrStatus + subVolumeMgrStatus.Activate() + // Look for AppInstanceSummary from zedmanager subAppInstanceSummary, err := ps.NewSubscription(pubsub.SubscriptionOptions{ AgentName: "zedmanager", @@ -449,6 +470,9 @@ func Run(ps *pubsub.PubSub, loggerArg *logrus.Logger, logArg *base.LogObject, ar case change := <-subVaultStatus.MsgChan(): subVaultStatus.ProcessChange(change) + case change := <-subVolumeMgrStatus.MsgChan(): + subVolumeMgrStatus.ProcessChange(change) + case change := <-subAppInstanceSummary.MsgChan(): subAppInstanceSummary.ProcessChange(change) @@ -571,6 +595,24 @@ func handleVaultStatusImpl(ctxArg interface{}, key string, triggerPrintOutput(ctx, "VaultStatus") } +func handleVolumeMgrStatusCreate(ctxArg interface{}, key string, + statusArg interface{}) { + handleVolumeMgrStatusImpl(ctxArg, key, statusArg) +} + +func handleVolumeMgrStatusModify(ctxArg interface{}, key string, + statusArg interface{}, oldStatusArg interface{}) { + handleVolumeMgrStatusImpl(ctxArg, key, statusArg) +} + +func handleVolumeMgrStatusImpl(ctxArg interface{}, key string, + statusArg interface{}) { + + ctx := ctxArg.(*diagContext) + ctx.volumeMgrStatus = statusArg.(types.VolumeMgrStatus) + triggerPrintOutput(ctx, "VolumeMgrStatus") +} + func handleAppInstanceSummaryCreate(ctxArg interface{}, key string, statusArg interface{}) { handleAppInstanceSummaryImpl(ctxArg, key, statusArg) @@ -873,6 +915,14 @@ func printOutput(ctx *diagContext, caller string) { } } + // volumemgr decides once, at startup, whether cluster storage became + // usable; a node where it never did cannot create any application volume. + // An empty Name means volumemgr has not published a status yet. + if ctx.volumeMgrStatus.Name != "" && !ctx.volumeMgrStatus.Initialized { + ctx.ph.Print("WARNING: cluster storage did not become ready at startup: %s\n", + ctx.volumeMgrStatus.UnmetCondition) + } + // Determine what we print for app summary summary := ctx.appInstanceSummary if ctx.appInstanceSummary.TotalError > 0 { diff --git a/pkg/pillar/docs/volumemgr.md b/pkg/pillar/docs/volumemgr.md index 81ece2f6474..37635d29661 100644 --- a/pkg/pillar/docs/volumemgr.md +++ b/pkg/pillar/docs/volumemgr.md @@ -47,6 +47,8 @@ Volume Manager interacts with the Cloud controller (e.g. zedcontrol) indirectly Both VolumeStatus and VolumeConfig use the agentScope mechanism to keep the baseosmgr and zedmanager use separately. +Volume Manager also publishes a single VolumeMgrStatus describing itself rather than any one volume. `RemainingSpace` drives nodeagent's decision to enter MaintenanceMode when the device is out of disk space. `Initialized` says whether storage is usable at all: on an EVE-k node it is the outcome of waiting for cluster storage (Longhorn and CDI) to come up, and when it is false `UnmetCondition` names the gate that was still outstanding, which diag prints on the console. + Volume Manager in turn requests work from downloader and verifier. This consists of a set of objects (all using the agentScope mechanism): - DownloaderConfig is published by volumemgr and subscribed to by downloader. This specifies the desire to find a downloaded blob for a particular object. From ef79fba9afa51b4bf0ba2f84616cf21ba19376c3 Mon Sep 17 00:00:00 2001 From: eriknordmark Date: Sun, 2 Aug 2026 13:38:55 +0200 Subject: [PATCH 4/6] evetest: cover the diag summary diag is the only operator-facing summary of device health -- controller connectivity, attestation and vault state, applications, cluster storage -- and none of it reaches the EVE API, so nothing noticed when it lost a section, went silent, or filled with errors on a healthy device. Add a test that deploys an application and reads the summary the way a consumer does, through the metadata server at GET /eve/v1/diag, asserting that a healthy onboarded device reports itself online and connected, lists its management port as up, and lists the deployed application as running. It also cross-checks the storage state against volumemgr's own publication. This is the first coverage of msrv's diag handler as well. A device that reaches the controller takes the short path through the port section, so the per-port detail and the "all management ports passed" verdict are absent from a healthy summary; the assertions match the lines that path actually emits. Signed-off-by: eriknordmark Co-Authored-By: Claude Opus 5 --- evetest/tests/diag/diag_test.go | 261 +++++++++++++++++++++++++++ evetest/tests/diag/testsuite_test.go | 39 ++++ 2 files changed, 300 insertions(+) create mode 100644 evetest/tests/diag/diag_test.go create mode 100644 evetest/tests/diag/testsuite_test.go diff --git a/evetest/tests/diag/diag_test.go b/evetest/tests/diag/diag_test.go new file mode 100644 index 00000000000..3df53293436 --- /dev/null +++ b/evetest/tests/diag/diag_test.go @@ -0,0 +1,261 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Tests for diag, the on-device diagnostic summary EVE prints for an operator. + +package diag_test + +import ( + "encoding/json" + "fmt" + "regexp" + "strings" + "testing" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + eveconfig "github.com/lf-edge/eve-api/go/config" + "github.com/lf-edge/eve-api/go/evecommon" + "github.com/lf-edge/eve/evetest" + "github.com/lf-edge/eve/evetest/netmodels" + pillartypes "github.com/lf-edge/eve/pkg/pillar/types" +) + +// Structural anchors of a complete diag summary. diag prints the sections in a +// fixed order and only reaches the port sections once it has the blink counter, +// the device network status and the port config list, so a dump taken early in +// boot legitimately stops after the application line. The assertions below are +// therefore polled until one dump carries all of them. +// +// A device that reaches the controller takes the short path through the port +// section: diag prints one line per port and skips the per-port DNS, routing +// and ping detail along with the "PASS: All management ports passed test" +// verdict, all of which appear only once connectivity is already broken. +var ( + diagHeaderRE = regexp.MustCompile( + `(?m)^INFO: updated diag information at \S+ due to \S+$`) + deviceLineRE = regexp.MustCompile( + `(?m)^(INFO|WARNING|ERROR): device: online attest: .+ vault: .+ pcr: .+$`) + appsLineRE = regexp.MustCompile( + `(?m)^INFO: applications: \d+ starting, \d+ running$`) + portsLineRE = regexp.MustCompile( + `(?m)^INFO: Have \d+ total ports\. \d+ ports should be connected to EV controller$`) + mgmtPortLineRE = regexp.MustCompile( + `(?m)^INFO: Port \S+: .*link: up use: mgmt .+$`) +) + +// TestDiagOutput verifies that diag produces its full diagnostic summary on a +// healthy, onboarded device, and that an application can retrieve it through +// the metadata server. +// +// Why this matters +// ---------------- +// diag is EVE's only operator-facing summary of device health: connectivity to +// the controller, attestation and vault state, application status and the +// readiness of cluster storage. None of it is reported through the EVE API, so +// nothing else in the test suites would notice diag going silent, losing a +// section, or filling with errors on an otherwise healthy device. +// +// The output has three sinks, all fed by the same content: /dev/tty1 for a +// physically attached console, /run/diag.out for a shell on the device, and +// GET /eve/v1/diag on the metadata server for applications. This test asserts +// through the metadata endpoint, which is the one documented for consumers and +// exercises msrv's handler along the way. +// +// Network model +// ------------- +// - netmodels.SingleEthWithDHCP -- one mgmt+app port with the SDN DNS server +// and a static entry for the controller. diag reports per-port +// connectivity, so the port must genuinely reach the controller for the +// healthy-path assertions to mean anything. +// +// Device configuration +// -------------------- +// - ethernet0 (eth0, mgmt+app): DHCP. +// - A Local NI ("local-ni") with a container application attached, which is +// what makes the metadata server reachable at 169.254.169.254 from inside +// the application. diag itself does not depend on either. +// +// Phases +// ------ +// 1. Bring up the port, the Local NI and the application, and wait until the +// application answers over SSH through the 2222->22 port-forward. +// 2. Fetch GET /eve/v1/diag from inside the application until one response +// carries a complete summary, and assert on its structure: the update +// header, the device/attest/vault/pcr line reporting the device online, +// the application counts, the summary line naming the device onboarded +// and connected, the port count, the management port up, and the +// deployed application listed as running. +// 3. Cross-check the storage state diag reports against volumemgr's own +// VolumeMgrStatus publication: on a non-EVE-k node storage is usable from +// the start, so the status must say so and diag must not print the +// cluster-storage warning. +// +// Parameters +// ---------- +// - HYPERVISOR: kvm or xen. Kubevirt is skipped -- it is reserved for the +// cluster tests, and this test deploys an application. +func TestDiagOutput(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + evetest.SkipIfHypervisorKubevirt() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + log := evetest.Logger() + + devConfig := evetest.NewEdgeDeviceConfig(devName) + dhcpNet := devConfig.AddNetwork( + evetest.DHCPNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + }) + devConfig.AddNetworkAdapter( + evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + device.ApplyConfig(devConfig, true, true) + + niUUID := devConfig.AddNetworkInstance(evetest.LocalNetworkInstanceConfig{ + DisplayName: "local-ni", + Port: "ethernet0", + Subnet: evetest.IPSubnet("10.11.12.0/24"), + DHCPRange: pillartypes.IPRange{ + Start: evetest.IPAddress("10.11.12.2"), + End: evetest.IPAddress("10.11.12.254"), + }, + Gateway: evetest.IPAddress("10.11.12.1"), + EnableFlowlog: false, + MTU: 1500, + ForwardLLDP: false, + }) + appUUID := devConfig.AddApplication(evetest.ApplicationInstanceConfig{ + DisplayName: "diag-reader", + Activate: true, + Image: evetest.DockerContainer{ + ImageName: "lfedge/evetest-ubuntu-ctr", + Tag: "1.0", + }, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 500 * evetest.MiB, + NetworkAdapters: []evetest.AppNetworkAdapter{ + evetest.VirtualNetworkAdapter{ + LogicalLabel: "vif0", + NetworkInstanceUUID: niUUID, + MAC: evetest.MACAddress("02:16:3e:00:00:01"), + PortFwdRules: []evetest.PortFwdRule{ + { + Protocol: evetest.NetworkProtocolTCP, + EdgeNodePort: 2222, + AppPort: 22, + }, + }, + ACLAllowRules: []evetest.ACLAllowRule{ + { + Protocol: evetest.NetworkProtocolAny, + RemoteSubnet: evetest.IPSubnet("0.0.0.0/0"), + }, + }, + }, + }, + }) + device.ApplyConfig(devConfig, false, false) + + timeoutExcludingDownload := 5 * time.Minute + device.WaitUntilAppIsRunning(appUUID, timeoutExcludingDownload) + evetest.Checkpoint("app-running") + + appAuth := evetest.UsernamePasswordAuth{ + Username: "root", + Password: "testpassword", + } + sshTimeout := 20 * time.Second + timeout := 3 * time.Minute + polling := 3 * time.Second + + log.Infof("Waiting for the application to become reachable over SSH") + t.Eventually(func(t Gomega) { + output, _, err := device.RunShellScriptInsideApp(appUUID, appAuth, + "hostname", sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + t.Expect(output).To(ContainSubstring(appUUID.String())) + }, timeout, polling).Should(Succeed()) + + // The status code is appended on its own line so that a non-200 response + // is distinguishable from an empty body. + const fetchDiag = `curl -sS -w '\nHTTP_STATUS:%{http_code}\n' ` + + `http://169.254.169.254/eve/v1/diag` + + log.Infof("Fetching the diag summary from inside the application") + var summary string + t.Eventually(func(t Gomega) { + output, _, err := device.RunShellScriptInsideApp(appUUID, appAuth, + fetchDiag, sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + t.Expect(output).To(ContainSubstring("HTTP_STATUS:200")) + summary = strings.SplitN(output, "HTTP_STATUS:", 2)[0] + + t.Expect(summary).To(MatchRegexp(diagHeaderRE.String()), + "diag must state when it last updated") + t.Expect(summary).To(MatchRegexp(deviceLineRE.String()), + "diag must report device, attestation, vault and PCR state") + t.Expect(summary).To(MatchRegexp(appsLineRE.String()), + "diag must report the application counts") + t.Expect(summary).To(ContainSubstring( + "INFO: Summary: Connected to EV Controller and onboarded")) + t.Expect(summary).To(MatchRegexp(portsLineRE.String()), + "diag must report how many ports should reach the controller") + t.Expect(summary).To(MatchRegexp(mgmtPortLineRE.String()), + "diag must report the management port as up") + t.Expect(summary).To(ContainSubstring(fmt.Sprintf( + "INFO: App diag-reader uuid %s state RUNNING", appUUID))) + }, timeout, polling).Should(Succeed()) + evetest.Checkpoint("diag-summary-fetched") + + log.Infof("diag summary:\n%s", summary) + t.Expect(summary).ToNot(ContainSubstring("WARNING: state "), + "the device network state must be SUCCESS") + t.Expect(summary).ToNot(ContainSubstring( + "ERROR: No management ports passed test")) + + // volumemgr publishes its status only once its startup waits are over, + // hence the poll rather than a single read. Its pubsub state lives in the + // pillar container, not in the namespace an SSH session lands in. + var volumeMgrStatus pillartypes.VolumeMgrStatus + t.Eventually(func(t Gomega) { + output, _, err := device.RunShellScript( + `eve exec pillar cat /run/volumemgr/VolumeMgrStatus/volumemgr.json`, + sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + t.Expect(json.Unmarshal([]byte(output), &volumeMgrStatus)).To(Succeed()) + }, timeout, polling).Should(Succeed()) + + t.Expect(volumeMgrStatus.Initialized).To(BeTrue(), + "storage is usable from the start on a node without cluster storage") + t.Expect(summary).ToNot(ContainSubstring( + "cluster storage did not become ready at startup")) +} diff --git a/evetest/tests/diag/testsuite_test.go b/evetest/tests/diag/testsuite_test.go new file mode 100644 index 00000000000..77743d8fa5d --- /dev/null +++ b/evetest/tests/diag/testsuite_test.go @@ -0,0 +1,39 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package diag_test + +import ( + "testing" + + "github.com/lf-edge/eve/evetest" +) + +// TestDiagSuite drives the scenarios covering diag, EVE's on-device diagnostic +// summary. diag reports nothing through the EVE API, so these are the only +// tests that notice it losing a section, going silent, or reporting errors on a +// healthy device. +// +// The suite declares the HYPERVISOR parameter once; every subtest deploys an +// application to read the summary from the metadata server and reads the value +// via evetest.GetHypervisorParameterValue(). +// +// Subtests +// -------- +// - TestDiagOutput -- the full summary on a healthy onboarded device, read +// through GET /eve/v1/diag, plus a cross-check of the reported cluster +// storage state against volumemgr's own publication. +func TestDiagSuite(test *testing.T) { + evetest.Init(test) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + + evetest.RunTestSuite( + evetest.TestCase{ + Test: TestDiagOutput, + }, + ) +} From 9ceabc4aad16d632cf015b082158258f329b4ea7 Mon Sep 17 00:00:00 2001 From: eriknordmark Date: Sun, 2 Aug 2026 14:31:38 +0200 Subject: [PATCH 5/6] diag: unit-test the storage warning The evetest coverage of the diag summary asserts the healthy path, where the cluster-storage warning is absent -- which is equally true of the code before this warning existed, so it cannot tell the two apart. Drive printOutput directly instead. Leaving the network state unset stops it right after the storage line, so no subscription is needed, and the three cases that matter can be asserted: a node whose storage never became usable is named along with the outstanding gate, a node with usable storage is not warned about, and neither is one whose volumemgr has not published yet. The first of these fails without the warning in place. Signed-off-by: eriknordmark Co-Authored-By: Claude Opus 5 --- pkg/pillar/cmd/diag/printoutput_test.go | 81 +++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 pkg/pillar/cmd/diag/printoutput_test.go diff --git a/pkg/pillar/cmd/diag/printoutput_test.go b/pkg/pillar/cmd/diag/printoutput_test.go new file mode 100644 index 00000000000..e98247e9133 --- /dev/null +++ b/pkg/pillar/cmd/diag/printoutput_test.go @@ -0,0 +1,81 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package diag + +import ( + "os" + "strings" + "testing" + + "github.com/lf-edge/eve/pkg/pillar/base" + "github.com/lf-edge/eve/pkg/pillar/types" + "github.com/sirupsen/logrus" +) + +const storageWarning = "WARNING: cluster storage did not become ready at startup" + +// printSummary renders one diag summary for the given volumemgr status and +// returns what landed in the state file. Leaving gotDNS, gotBC and gotDPCList +// unset stops printOutput right after the storage and application lines, which +// is as far as these tests need it to go and keeps them free of any pubsub +// subscription. +func printSummary(t *testing.T, status types.VolumeMgrStatus) string { + t.Helper() + log = base.NewSourceLogObject(logrus.StandardLogger(), "diag", 0) + + devNull, err := os.OpenFile(os.DevNull, os.O_WRONLY, 0) + if err != nil { + t.Fatal(err) + } + defer devNull.Close() + stateFile, err := os.CreateTemp(t.TempDir(), "diag.out") + if err != nil { + t.Fatal(err) + } + defer stateFile.Close() + + ctx := &diagContext{ + DeviceNetworkStatus: &types.DeviceNetworkStatus{}, + DevicePortConfigList: &types.DevicePortConfigList{}, + volumeMgrStatus: status, + } + ctx.ph = PrintIfSpaceInit(devNull, stateFile.Name(), 100, 200) + printOutput(ctx, "test") + + content, err := os.ReadFile(stateFile.Name()) + if err != nil { + t.Fatal(err) + } + return string(content) +} + +func TestDiagReportsUnmetStorageCondition(t *testing.T) { + const condition = "longhorn not ready: daemonset:longhorn-manager not running on node" + out := printSummary(t, types.VolumeMgrStatus{ + Name: "volumemgr", + UnmetCondition: condition, + }) + if !strings.Contains(out, storageWarning+": "+condition) { + t.Fatalf("summary must name the outstanding storage gate, got:\n%s", out) + } +} + +func TestDiagQuietWhenStorageIsReady(t *testing.T) { + out := printSummary(t, types.VolumeMgrStatus{ + Name: "volumemgr", + Initialized: true, + }) + if strings.Contains(out, storageWarning) { + t.Fatalf("a node with usable storage must not be warned about, got:\n%s", out) + } +} + +// volumemgr publishes nothing while its startup waits are still running, and an +// absent status must not be reported as a storage failure. +func TestDiagQuietBeforeVolumeMgrPublishes(t *testing.T) { + out := printSummary(t, types.VolumeMgrStatus{}) + if strings.Contains(out, storageWarning) { + t.Fatalf("no warning is due before volumemgr publishes, got:\n%s", out) + } +} From dc8490cab8147de1b889c971e41c193910ec4013 Mon Sep 17 00:00:00 2001 From: eriknordmark Date: Sun, 2 Aug 2026 15:32:20 +0200 Subject: [PATCH 6/6] volumemgr: unit-test the reported readiness Nothing failed when volumemgr reported storage as usable regardless of the outcome, which is what let the original bug stand. A test has to see the published status, and the publishing path first computes the remaining disk space from /persist -- absent in any test environment, so the function returns before publishing anything. Split the status out of the publish, and assert on it directly: a node whose storage never came up reports that, along with the gate it was waiting on, and a node with usable storage reports success and no gate. The first fails if the field goes back to being a constant. Signed-off-by: eriknordmark Co-Authored-By: Claude Opus 5 --- pkg/pillar/cmd/volumemgr/handlediskmetrics.go | 15 +++++--- .../cmd/volumemgr/storagereadiness_test.go | 34 +++++++++++++++++++ 2 files changed, 45 insertions(+), 4 deletions(-) create mode 100644 pkg/pillar/cmd/volumemgr/storagereadiness_test.go diff --git a/pkg/pillar/cmd/volumemgr/handlediskmetrics.go b/pkg/pillar/cmd/volumemgr/handlediskmetrics.go index 1c869cdc998..e31a80f3c0f 100644 --- a/pkg/pillar/cmd/volumemgr/handlediskmetrics.go +++ b/pkg/pillar/cmd/volumemgr/handlediskmetrics.go @@ -140,13 +140,20 @@ func generateAndPublishVolumeMgrStatus(ctx *volumemgrContext) { log.Error(err) return } - st := types.VolumeMgrStatus{ + st := ctx.volumeMgrStatus(remaining) + ctx.pubVolumeMgrStatus.Publish(st.Key(), st) +} + +// volumeMgrStatus describes volumemgr itself: whether storage ever became +// usable, the gate still outstanding when it did not, and the space left for +// volumes after everything reserved for EVE has been subtracted. +func (ctxPtr *volumemgrContext) volumeMgrStatus(remaining uint64) types.VolumeMgrStatus { + return types.VolumeMgrStatus{ Name: agentName, - Initialized: ctx.storageReady, - UnmetCondition: ctx.storageUnmet, + Initialized: ctxPtr.storageReady, + UnmetCondition: ctxPtr.storageUnmet, RemainingSpace: remaining, } - ctx.pubVolumeMgrStatus.Publish(st.Key(), st) } // createOrUpdateDiskMetrics creates or updates metrics for all disks, mountpaths and volumeStatuses diff --git a/pkg/pillar/cmd/volumemgr/storagereadiness_test.go b/pkg/pillar/cmd/volumemgr/storagereadiness_test.go new file mode 100644 index 00000000000..79e917f0c29 --- /dev/null +++ b/pkg/pillar/cmd/volumemgr/storagereadiness_test.go @@ -0,0 +1,34 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package volumemgr + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +const testUnmetCondition = "timed out waiting for the condition (last unmet " + + "condition: longhorn not ready: daemonset:longhorn-manager not running on node)" + +// A node whose cluster storage never came up must say so, and name the gate it +// was still waiting on. Reporting success here is indistinguishable, to +// everything outside volumemgr, from a node that is genuinely healthy. +func TestVolumeMgrStatusReportsUnmetCondition(t *testing.T) { + ctx := volumemgrContext{ + storageReady: false, + storageUnmet: testUnmetCondition, + } + status := ctx.volumeMgrStatus(1024) + assert.False(t, status.Initialized) + assert.Equal(t, testUnmetCondition, status.UnmetCondition) + assert.Equal(t, uint64(1024), status.RemainingSpace) +} + +func TestVolumeMgrStatusReportsStorageReady(t *testing.T) { + ctx := volumemgrContext{storageReady: true} + status := ctx.volumeMgrStatus(1024) + assert.True(t, status.Initialized) + assert.Empty(t, status.UnmetCondition) +}