diff --git a/evetest/tests/diag/diag_test.go b/evetest/tests/diag/diag_test.go new file mode 100644 index 00000000000..3df53293436 --- /dev/null +++ b/evetest/tests/diag/diag_test.go @@ -0,0 +1,261 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Tests for diag, the on-device diagnostic summary EVE prints for an operator. + +package diag_test + +import ( + "encoding/json" + "fmt" + "regexp" + "strings" + "testing" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + eveconfig "github.com/lf-edge/eve-api/go/config" + "github.com/lf-edge/eve-api/go/evecommon" + "github.com/lf-edge/eve/evetest" + "github.com/lf-edge/eve/evetest/netmodels" + pillartypes "github.com/lf-edge/eve/pkg/pillar/types" +) + +// Structural anchors of a complete diag summary. diag prints the sections in a +// fixed order and only reaches the port sections once it has the blink counter, +// the device network status and the port config list, so a dump taken early in +// boot legitimately stops after the application line. The assertions below are +// therefore polled until one dump carries all of them. +// +// A device that reaches the controller takes the short path through the port +// section: diag prints one line per port and skips the per-port DNS, routing +// and ping detail along with the "PASS: All management ports passed test" +// verdict, all of which appear only once connectivity is already broken. +var ( + diagHeaderRE = regexp.MustCompile( + `(?m)^INFO: updated diag information at \S+ due to \S+$`) + deviceLineRE = regexp.MustCompile( + `(?m)^(INFO|WARNING|ERROR): device: online attest: .+ vault: .+ pcr: .+$`) + appsLineRE = regexp.MustCompile( + `(?m)^INFO: applications: \d+ starting, \d+ running$`) + portsLineRE = regexp.MustCompile( + `(?m)^INFO: Have \d+ total ports\. \d+ ports should be connected to EV controller$`) + mgmtPortLineRE = regexp.MustCompile( + `(?m)^INFO: Port \S+: .*link: up use: mgmt .+$`) +) + +// TestDiagOutput verifies that diag produces its full diagnostic summary on a +// healthy, onboarded device, and that an application can retrieve it through +// the metadata server. +// +// Why this matters +// ---------------- +// diag is EVE's only operator-facing summary of device health: connectivity to +// the controller, attestation and vault state, application status and the +// readiness of cluster storage. None of it is reported through the EVE API, so +// nothing else in the test suites would notice diag going silent, losing a +// section, or filling with errors on an otherwise healthy device. +// +// The output has three sinks, all fed by the same content: /dev/tty1 for a +// physically attached console, /run/diag.out for a shell on the device, and +// GET /eve/v1/diag on the metadata server for applications. This test asserts +// through the metadata endpoint, which is the one documented for consumers and +// exercises msrv's handler along the way. +// +// Network model +// ------------- +// - netmodels.SingleEthWithDHCP -- one mgmt+app port with the SDN DNS server +// and a static entry for the controller. diag reports per-port +// connectivity, so the port must genuinely reach the controller for the +// healthy-path assertions to mean anything. +// +// Device configuration +// -------------------- +// - ethernet0 (eth0, mgmt+app): DHCP. +// - A Local NI ("local-ni") with a container application attached, which is +// what makes the metadata server reachable at 169.254.169.254 from inside +// the application. diag itself does not depend on either. +// +// Phases +// ------ +// 1. Bring up the port, the Local NI and the application, and wait until the +// application answers over SSH through the 2222->22 port-forward. +// 2. Fetch GET /eve/v1/diag from inside the application until one response +// carries a complete summary, and assert on its structure: the update +// header, the device/attest/vault/pcr line reporting the device online, +// the application counts, the summary line naming the device onboarded +// and connected, the port count, the management port up, and the +// deployed application listed as running. +// 3. Cross-check the storage state diag reports against volumemgr's own +// VolumeMgrStatus publication: on a non-EVE-k node storage is usable from +// the start, so the status must say so and diag must not print the +// cluster-storage warning. +// +// Parameters +// ---------- +// - HYPERVISOR: kvm or xen. Kubevirt is skipped -- it is reserved for the +// cluster tests, and this test deploys an application. +func TestDiagOutput(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + evetest.SkipIfHypervisorKubevirt() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + log := evetest.Logger() + + devConfig := evetest.NewEdgeDeviceConfig(devName) + dhcpNet := devConfig.AddNetwork( + evetest.DHCPNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + }) + devConfig.AddNetworkAdapter( + evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + device.ApplyConfig(devConfig, true, true) + + niUUID := devConfig.AddNetworkInstance(evetest.LocalNetworkInstanceConfig{ + DisplayName: "local-ni", + Port: "ethernet0", + Subnet: evetest.IPSubnet("10.11.12.0/24"), + DHCPRange: pillartypes.IPRange{ + Start: evetest.IPAddress("10.11.12.2"), + End: evetest.IPAddress("10.11.12.254"), + }, + Gateway: evetest.IPAddress("10.11.12.1"), + EnableFlowlog: false, + MTU: 1500, + ForwardLLDP: false, + }) + appUUID := devConfig.AddApplication(evetest.ApplicationInstanceConfig{ + DisplayName: "diag-reader", + Activate: true, + Image: evetest.DockerContainer{ + ImageName: "lfedge/evetest-ubuntu-ctr", + Tag: "1.0", + }, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 500 * evetest.MiB, + NetworkAdapters: []evetest.AppNetworkAdapter{ + evetest.VirtualNetworkAdapter{ + LogicalLabel: "vif0", + NetworkInstanceUUID: niUUID, + MAC: evetest.MACAddress("02:16:3e:00:00:01"), + PortFwdRules: []evetest.PortFwdRule{ + { + Protocol: evetest.NetworkProtocolTCP, + EdgeNodePort: 2222, + AppPort: 22, + }, + }, + ACLAllowRules: []evetest.ACLAllowRule{ + { + Protocol: evetest.NetworkProtocolAny, + RemoteSubnet: evetest.IPSubnet("0.0.0.0/0"), + }, + }, + }, + }, + }) + device.ApplyConfig(devConfig, false, false) + + timeoutExcludingDownload := 5 * time.Minute + device.WaitUntilAppIsRunning(appUUID, timeoutExcludingDownload) + evetest.Checkpoint("app-running") + + appAuth := evetest.UsernamePasswordAuth{ + Username: "root", + Password: "testpassword", + } + sshTimeout := 20 * time.Second + timeout := 3 * time.Minute + polling := 3 * time.Second + + log.Infof("Waiting for the application to become reachable over SSH") + t.Eventually(func(t Gomega) { + output, _, err := device.RunShellScriptInsideApp(appUUID, appAuth, + "hostname", sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + t.Expect(output).To(ContainSubstring(appUUID.String())) + }, timeout, polling).Should(Succeed()) + + // The status code is appended on its own line so that a non-200 response + // is distinguishable from an empty body. + const fetchDiag = `curl -sS -w '\nHTTP_STATUS:%{http_code}\n' ` + + `http://169.254.169.254/eve/v1/diag` + + log.Infof("Fetching the diag summary from inside the application") + var summary string + t.Eventually(func(t Gomega) { + output, _, err := device.RunShellScriptInsideApp(appUUID, appAuth, + fetchDiag, sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + t.Expect(output).To(ContainSubstring("HTTP_STATUS:200")) + summary = strings.SplitN(output, "HTTP_STATUS:", 2)[0] + + t.Expect(summary).To(MatchRegexp(diagHeaderRE.String()), + "diag must state when it last updated") + t.Expect(summary).To(MatchRegexp(deviceLineRE.String()), + "diag must report device, attestation, vault and PCR state") + t.Expect(summary).To(MatchRegexp(appsLineRE.String()), + "diag must report the application counts") + t.Expect(summary).To(ContainSubstring( + "INFO: Summary: Connected to EV Controller and onboarded")) + t.Expect(summary).To(MatchRegexp(portsLineRE.String()), + "diag must report how many ports should reach the controller") + t.Expect(summary).To(MatchRegexp(mgmtPortLineRE.String()), + "diag must report the management port as up") + t.Expect(summary).To(ContainSubstring(fmt.Sprintf( + "INFO: App diag-reader uuid %s state RUNNING", appUUID))) + }, timeout, polling).Should(Succeed()) + evetest.Checkpoint("diag-summary-fetched") + + log.Infof("diag summary:\n%s", summary) + t.Expect(summary).ToNot(ContainSubstring("WARNING: state "), + "the device network state must be SUCCESS") + t.Expect(summary).ToNot(ContainSubstring( + "ERROR: No management ports passed test")) + + // volumemgr publishes its status only once its startup waits are over, + // hence the poll rather than a single read. Its pubsub state lives in the + // pillar container, not in the namespace an SSH session lands in. + var volumeMgrStatus pillartypes.VolumeMgrStatus + t.Eventually(func(t Gomega) { + output, _, err := device.RunShellScript( + `eve exec pillar cat /run/volumemgr/VolumeMgrStatus/volumemgr.json`, + sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + t.Expect(json.Unmarshal([]byte(output), &volumeMgrStatus)).To(Succeed()) + }, timeout, polling).Should(Succeed()) + + t.Expect(volumeMgrStatus.Initialized).To(BeTrue(), + "storage is usable from the start on a node without cluster storage") + t.Expect(summary).ToNot(ContainSubstring( + "cluster storage did not become ready at startup")) +} diff --git a/evetest/tests/diag/testsuite_test.go b/evetest/tests/diag/testsuite_test.go new file mode 100644 index 00000000000..77743d8fa5d --- /dev/null +++ b/evetest/tests/diag/testsuite_test.go @@ -0,0 +1,39 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package diag_test + +import ( + "testing" + + "github.com/lf-edge/eve/evetest" +) + +// TestDiagSuite drives the scenarios covering diag, EVE's on-device diagnostic +// summary. diag reports nothing through the EVE API, so these are the only +// tests that notice it losing a section, going silent, or reporting errors on a +// healthy device. +// +// The suite declares the HYPERVISOR parameter once; every subtest deploys an +// application to read the summary from the metadata server and reads the value +// via evetest.GetHypervisorParameterValue(). +// +// Subtests +// -------- +// - TestDiagOutput -- the full summary on a healthy onboarded device, read +// through GET /eve/v1/diag, plus a cross-check of the reported cluster +// storage state against volumemgr's own publication. +func TestDiagSuite(test *testing.T) { + evetest.Init(test) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + + evetest.RunTestSuite( + evetest.TestCase{ + Test: TestDiagOutput, + }, + ) +} diff --git a/pkg/pillar/cmd/diag/diag.go b/pkg/pillar/cmd/diag/diag.go index 93a005e67c9..f6c91e6cb5f 100644 --- a/pkg/pillar/cmd/diag/diag.go +++ b/pkg/pillar/cmd/diag/diag.go @@ -68,6 +68,8 @@ type diagContext struct { zedagentStatus types.ZedAgentStatus subVaultStatus pubsub.Subscription vaultStatus types.VaultStatus + subVolumeMgrStatus pubsub.Subscription + volumeMgrStatus types.VolumeMgrStatus subAppInstanceSummary pubsub.Subscription appInstanceSummary types.AppInstanceSummary subAppInstanceStatus pubsub.Subscription @@ -349,6 +351,25 @@ func Run(ps *pubsub.PubSub, loggerArg *logrus.Logger, logArg *base.LogObject, ar ctx.subVaultStatus = subVaultStatus subVaultStatus.Activate() + // Look for VolumeMgrStatus from volumemgr, to show whether cluster storage + // became usable on an EVE-k node. + subVolumeMgrStatus, err := ps.NewSubscription(pubsub.SubscriptionOptions{ + AgentName: "volumemgr", + MyAgentName: agentName, + TopicImpl: types.VolumeMgrStatus{}, + Activate: false, + Ctx: &ctx, + CreateHandler: handleVolumeMgrStatusCreate, + ModifyHandler: handleVolumeMgrStatusModify, + WarningTime: warningTime, + ErrorTime: errorTime, + }) + if err != nil { + log.Fatal(err) + } + ctx.subVolumeMgrStatus = subVolumeMgrStatus + subVolumeMgrStatus.Activate() + // Look for AppInstanceSummary from zedmanager subAppInstanceSummary, err := ps.NewSubscription(pubsub.SubscriptionOptions{ AgentName: "zedmanager", @@ -449,6 +470,9 @@ func Run(ps *pubsub.PubSub, loggerArg *logrus.Logger, logArg *base.LogObject, ar case change := <-subVaultStatus.MsgChan(): subVaultStatus.ProcessChange(change) + case change := <-subVolumeMgrStatus.MsgChan(): + subVolumeMgrStatus.ProcessChange(change) + case change := <-subAppInstanceSummary.MsgChan(): subAppInstanceSummary.ProcessChange(change) @@ -571,6 +595,24 @@ func handleVaultStatusImpl(ctxArg interface{}, key string, triggerPrintOutput(ctx, "VaultStatus") } +func handleVolumeMgrStatusCreate(ctxArg interface{}, key string, + statusArg interface{}) { + handleVolumeMgrStatusImpl(ctxArg, key, statusArg) +} + +func handleVolumeMgrStatusModify(ctxArg interface{}, key string, + statusArg interface{}, oldStatusArg interface{}) { + handleVolumeMgrStatusImpl(ctxArg, key, statusArg) +} + +func handleVolumeMgrStatusImpl(ctxArg interface{}, key string, + statusArg interface{}) { + + ctx := ctxArg.(*diagContext) + ctx.volumeMgrStatus = statusArg.(types.VolumeMgrStatus) + triggerPrintOutput(ctx, "VolumeMgrStatus") +} + func handleAppInstanceSummaryCreate(ctxArg interface{}, key string, statusArg interface{}) { handleAppInstanceSummaryImpl(ctxArg, key, statusArg) @@ -873,6 +915,14 @@ func printOutput(ctx *diagContext, caller string) { } } + // volumemgr decides once, at startup, whether cluster storage became + // usable; a node where it never did cannot create any application volume. + // An empty Name means volumemgr has not published a status yet. + if ctx.volumeMgrStatus.Name != "" && !ctx.volumeMgrStatus.Initialized { + ctx.ph.Print("WARNING: cluster storage did not become ready at startup: %s\n", + ctx.volumeMgrStatus.UnmetCondition) + } + // Determine what we print for app summary summary := ctx.appInstanceSummary if ctx.appInstanceSummary.TotalError > 0 { diff --git a/pkg/pillar/cmd/diag/printoutput_test.go b/pkg/pillar/cmd/diag/printoutput_test.go new file mode 100644 index 00000000000..e98247e9133 --- /dev/null +++ b/pkg/pillar/cmd/diag/printoutput_test.go @@ -0,0 +1,81 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package diag + +import ( + "os" + "strings" + "testing" + + "github.com/lf-edge/eve/pkg/pillar/base" + "github.com/lf-edge/eve/pkg/pillar/types" + "github.com/sirupsen/logrus" +) + +const storageWarning = "WARNING: cluster storage did not become ready at startup" + +// printSummary renders one diag summary for the given volumemgr status and +// returns what landed in the state file. Leaving gotDNS, gotBC and gotDPCList +// unset stops printOutput right after the storage and application lines, which +// is as far as these tests need it to go and keeps them free of any pubsub +// subscription. +func printSummary(t *testing.T, status types.VolumeMgrStatus) string { + t.Helper() + log = base.NewSourceLogObject(logrus.StandardLogger(), "diag", 0) + + devNull, err := os.OpenFile(os.DevNull, os.O_WRONLY, 0) + if err != nil { + t.Fatal(err) + } + defer devNull.Close() + stateFile, err := os.CreateTemp(t.TempDir(), "diag.out") + if err != nil { + t.Fatal(err) + } + defer stateFile.Close() + + ctx := &diagContext{ + DeviceNetworkStatus: &types.DeviceNetworkStatus{}, + DevicePortConfigList: &types.DevicePortConfigList{}, + volumeMgrStatus: status, + } + ctx.ph = PrintIfSpaceInit(devNull, stateFile.Name(), 100, 200) + printOutput(ctx, "test") + + content, err := os.ReadFile(stateFile.Name()) + if err != nil { + t.Fatal(err) + } + return string(content) +} + +func TestDiagReportsUnmetStorageCondition(t *testing.T) { + const condition = "longhorn not ready: daemonset:longhorn-manager not running on node" + out := printSummary(t, types.VolumeMgrStatus{ + Name: "volumemgr", + UnmetCondition: condition, + }) + if !strings.Contains(out, storageWarning+": "+condition) { + t.Fatalf("summary must name the outstanding storage gate, got:\n%s", out) + } +} + +func TestDiagQuietWhenStorageIsReady(t *testing.T) { + out := printSummary(t, types.VolumeMgrStatus{ + Name: "volumemgr", + Initialized: true, + }) + if strings.Contains(out, storageWarning) { + t.Fatalf("a node with usable storage must not be warned about, got:\n%s", out) + } +} + +// volumemgr publishes nothing while its startup waits are still running, and an +// absent status must not be reported as a storage failure. +func TestDiagQuietBeforeVolumeMgrPublishes(t *testing.T) { + out := printSummary(t, types.VolumeMgrStatus{}) + if strings.Contains(out, storageWarning) { + t.Fatalf("no warning is due before volumemgr publishes, got:\n%s", out) + } +} diff --git a/pkg/pillar/cmd/volumemgr/handlediskmetrics.go b/pkg/pillar/cmd/volumemgr/handlediskmetrics.go index ff71532b3ab..e31a80f3c0f 100644 --- a/pkg/pillar/cmd/volumemgr/handlediskmetrics.go +++ b/pkg/pillar/cmd/volumemgr/handlediskmetrics.go @@ -140,12 +140,20 @@ func generateAndPublishVolumeMgrStatus(ctx *volumemgrContext) { log.Error(err) return } - st := types.VolumeMgrStatus{ + st := ctx.volumeMgrStatus(remaining) + ctx.pubVolumeMgrStatus.Publish(st.Key(), st) +} + +// volumeMgrStatus describes volumemgr itself: whether storage ever became +// usable, the gate still outstanding when it did not, and the space left for +// volumes after everything reserved for EVE has been subtracted. +func (ctxPtr *volumemgrContext) volumeMgrStatus(remaining uint64) types.VolumeMgrStatus { + return types.VolumeMgrStatus{ Name: agentName, - Initialized: true, + Initialized: ctxPtr.storageReady, + UnmetCondition: ctxPtr.storageUnmet, RemainingSpace: remaining, } - ctx.pubVolumeMgrStatus.Publish(st.Key(), st) } // createOrUpdateDiskMetrics creates or updates metrics for all disks, mountpaths and volumeStatuses diff --git a/pkg/pillar/cmd/volumemgr/storagereadiness_test.go b/pkg/pillar/cmd/volumemgr/storagereadiness_test.go new file mode 100644 index 00000000000..79e917f0c29 --- /dev/null +++ b/pkg/pillar/cmd/volumemgr/storagereadiness_test.go @@ -0,0 +1,34 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package volumemgr + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +const testUnmetCondition = "timed out waiting for the condition (last unmet " + + "condition: longhorn not ready: daemonset:longhorn-manager not running on node)" + +// A node whose cluster storage never came up must say so, and name the gate it +// was still waiting on. Reporting success here is indistinguishable, to +// everything outside volumemgr, from a node that is genuinely healthy. +func TestVolumeMgrStatusReportsUnmetCondition(t *testing.T) { + ctx := volumemgrContext{ + storageReady: false, + storageUnmet: testUnmetCondition, + } + status := ctx.volumeMgrStatus(1024) + assert.False(t, status.Initialized) + assert.Equal(t, testUnmetCondition, status.UnmetCondition) + assert.Equal(t, uint64(1024), status.RemainingSpace) +} + +func TestVolumeMgrStatusReportsStorageReady(t *testing.T) { + ctx := volumemgrContext{storageReady: true} + status := ctx.volumeMgrStatus(1024) + assert.True(t, status.Initialized) + assert.Empty(t, status.UnmetCondition) +} diff --git a/pkg/pillar/cmd/volumemgr/volumemgr.go b/pkg/pillar/cmd/volumemgr/volumemgr.go index 496ed38dbc5..492dbd3c2c7 100644 --- a/pkg/pillar/cmd/volumemgr/volumemgr.go +++ b/pkg/pillar/cmd/volumemgr/volumemgr.go @@ -89,6 +89,13 @@ type volumemgrContext struct { gcRunning bool initGced bool // Will be marked true after initObjects are garbage collected + // storageReady is false only on an EVE-k node whose cluster storage never + // came up; storageUnmet then carries the outstanding gate. Both are + // reported through VolumeMgrStatus. Written once during startup, before + // the disk-metrics task that reads them is launched. + storageReady bool + storageUnmet string + globalConfig *types.ConfigItemValueMap GCInitialized bool vdiskGCTime uint32 // In seconds; XXX delete when OldVolumeStatus is deleted @@ -169,6 +176,9 @@ func Run(ps *pubsub.PubSub, loggerArg *logrus.Logger, logArg *base.LogObject, ar globalConfig: types.DefaultConfigItemValueMap(), persistType: persist.ReadPersistType(), hvTypeKube: base.IsHVTypeKube(), + // Only an EVE-k node has cluster storage to wait for; everywhere else + // storage is usable as soon as volumemgr is up. + storageReady: !base.IsHVTypeKube(), } agentbase.Init(&ctx, logger, log, agentName, agentbase.WithPidFile(), @@ -341,6 +351,7 @@ func Run(ps *pubsub.PubSub, loggerArg *logrus.Logger, logArg *base.LogObject, ar }) if err != nil { log.Errorf("volumemgr run: wait for kubernetes error %v", err) + ctx.storageUnmet = err.Error() } else { log.Noticef("volumemgr run: kubernetes node ready, longhorn ready") } @@ -371,9 +382,14 @@ func Run(ps *pubsub.PubSub, loggerArg *logrus.Logger, logArg *base.LogObject, ar } } storageDeadline.Stop() + ctx.storageReady = storageReady if storageReady { + ctx.storageUnmet = "" log.Noticef("volumemgr run: cluster storage (longhorn+CDI) ready") } else { + if ctx.storageUnmet == "" { + ctx.storageUnmet = "cluster storage (longhorn+CDI) not ready" + } log.Warnf("volumemgr run: timeout waiting for cluster storage; " + "volumes will defer and retry") } diff --git a/pkg/pillar/docs/volumemgr.md b/pkg/pillar/docs/volumemgr.md index 81ece2f6474..37635d29661 100644 --- a/pkg/pillar/docs/volumemgr.md +++ b/pkg/pillar/docs/volumemgr.md @@ -47,6 +47,8 @@ Volume Manager interacts with the Cloud controller (e.g. zedcontrol) indirectly Both VolumeStatus and VolumeConfig use the agentScope mechanism to keep the baseosmgr and zedmanager use separately. +Volume Manager also publishes a single VolumeMgrStatus describing itself rather than any one volume. `RemainingSpace` drives nodeagent's decision to enter MaintenanceMode when the device is out of disk space. `Initialized` says whether storage is usable at all: on an EVE-k node it is the outcome of waiting for cluster storage (Longhorn and CDI) to come up, and when it is false `UnmetCondition` names the gate that was still outstanding, which diag prints on the console. + Volume Manager in turn requests work from downloader and verifier. This consists of a set of objects (all using the agentScope mechanism): - DownloaderConfig is published by volumemgr and subscribed to by downloader. This specifies the desire to find a downloaded blob for a particular object. diff --git a/pkg/pillar/kubeapi/kubeapi.go b/pkg/pillar/kubeapi/kubeapi.go index 5e2fbfcab9e..e8488d33a8f 100644 --- a/pkg/pillar/kubeapi/kubeapi.go +++ b/pkg/pillar/kubeapi/kubeapi.go @@ -288,14 +288,22 @@ func checkLonghornReady(client kubernetes.Interface, nodeName string) error { "longhorn-csi-plugin": false, "engine-image": false, } - // Check if each daemonset is running and ready on this node + // Check if each daemonset is running and ready on this node. Only the + // expected daemonsets above gate readiness: anything else sharing the + // namespace is not ours to judge, and a permanently unready stray would + // otherwise block every volume on this node for as long as it exists. for _, lhDaemonset := range lhDaemonsets.Items { lhDsName := lhDaemonset.GetName() + expected := false for dsPrefix := range lhExpectedDaemonsets { if strings.HasPrefix(lhDsName, dsPrefix) { lhExpectedDaemonsets[dsPrefix] = true + expected = true } } + if !expected { + continue + } var labelSelectors []string for dsLabelK, dsLabelV := range lhDaemonset.Spec.Template.Labels { diff --git a/pkg/pillar/kubeapi/longhornready_test.go b/pkg/pillar/kubeapi/longhornready_test.go new file mode 100644 index 00000000000..04667444602 --- /dev/null +++ b/pkg/pillar/kubeapi/longhornready_test.go @@ -0,0 +1,102 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +//go:build k + +package kubeapi + +import ( + "testing" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" +) + +const lhTestNode = "test-node" + +func lhDaemonset(name string) *appsv1.DaemonSet { + return &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "longhorn-system"}, + Spec: appsv1.DaemonSetSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": name}}, + }, + }, + } +} + +func lhPod(dsName string, ready bool) *corev1.Pod { + phase := corev1.PodRunning + if !ready { + phase = corev1.PodPending + } + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: dsName + "-pod", + Namespace: "longhorn-system", + Labels: map[string]string{"app": dsName}, + }, + Spec: corev1.PodSpec{NodeName: lhTestNode}, + Status: corev1.PodStatus{ + Phase: phase, + ContainerStatuses: []corev1.ContainerStatus{{Ready: ready}}, + }, + } +} + +// healthyLonghornObjects returns the three daemonsets checkLonghornReady +// expects, each with a Running-and-Ready pod on lhTestNode. +func healthyLonghornObjects() []runtime.Object { + names := []string{"longhorn-manager", "longhorn-csi-plugin", "engine-image-ei-abcdef12"} + objs := make([]runtime.Object, 0, len(names)*2) + for _, n := range names { + objs = append(objs, lhDaemonset(n), lhPod(n, true)) + } + return objs +} + +func TestCheckLonghornReadyHealthy(t *testing.T) { + client := fake.NewSimpleClientset(healthyLonghornObjects()...) + if err := checkLonghornReady(client, lhTestNode); err != nil { + t.Fatalf("expected ready, got %v", err) + } +} + +// A daemonset that is not one of longhorn's own must not gate storage. EVE's +// collect-info leaves a SupportBundle agent daemonset behind in this namespace +// which never becomes ready, and it used to block every volume on the node. +func TestCheckLonghornReadyIgnoresStrayDaemonset(t *testing.T) { + objs := healthyLonghornObjects() + objs = append(objs, lhDaemonset("longhorn-support-bundle-agent")) + client := fake.NewSimpleClientset(objs...) + if err := checkLonghornReady(client, lhTestNode); err != nil { + t.Fatalf("stray daemonset must not gate readiness, got %v", err) + } +} + +func TestCheckLonghornReadyMissingExpectedDaemonset(t *testing.T) { + objs := []runtime.Object{ + lhDaemonset("longhorn-manager"), lhPod("longhorn-manager", true), + lhDaemonset("longhorn-csi-plugin"), lhPod("longhorn-csi-plugin", true), + } + client := fake.NewSimpleClientset(objs...) + err := checkLonghornReady(client, lhTestNode) + if err == nil { + t.Fatal("expected an error when engine-image is absent") + } +} + +func TestCheckLonghornReadyExpectedDaemonsetNotReady(t *testing.T) { + names := []string{"longhorn-manager", "longhorn-csi-plugin", "engine-image-ei-abcdef12"} + var objs []runtime.Object + for _, n := range names { + objs = append(objs, lhDaemonset(n), lhPod(n, n != "longhorn-manager")) + } + client := fake.NewSimpleClientset(objs...) + if err := checkLonghornReady(client, lhTestNode); err == nil { + t.Fatal("expected an error when an expected daemonset pod is not ready") + } +} diff --git a/pkg/pillar/types/volumetypes.go b/pkg/pillar/types/volumetypes.go index e6750f97810..a2c74400971 100644 --- a/pkg/pillar/types/volumetypes.go +++ b/pkg/pillar/types/volumetypes.go @@ -619,8 +619,14 @@ func (status VolumeCreatePending) LogDelete(logBase *base.LogObject) { // VolumeMgrStatus : type VolumeMgrStatus struct { - Name string - Initialized bool + Name string + Initialized bool + // UnmetCondition names the readiness gate still outstanding when + // Initialized is false, e.g. "longhorn not ready: longhorn missing + // daemonset:engine-image". Empty when Initialized is true. Informational: + // it exists so an operator can tell a converging cluster from a stuck one + // without correlating agent logs. + UnmetCondition string RemainingSpace uint64 // In bytes. Takes into account "reserved" for dom0 }