From a59d5dbf05e6b92827bbfaa0810769be3464cfcf Mon Sep 17 00:00:00 2001 From: Andrew Durbin Date: Thu, 13 Aug 2026 15:26:54 -0600 Subject: [PATCH 1/2] evetest: add cluster tie-breaker support ClusterNode gets a TieBreaker field. NewEdgeClusterConfig finds the onboarded UUID of that node and writes it to EdgeNodeCluster.TieBreakerNodeId. It writes the UUID to the configuration of all three nodes, because each node compares that UUID with its own to know if the role applies to itself. A new TestHarness.deviceUUID helper does the name-to-UUID lookup. A maximum of one node can be the tie-breaker. TestTieBreakerCluster starts a three-node cluster with a tie-breaker. It first makes sure that all three nodes report the same designation. It then waits for the tie-breaker-config-applied label, which pkg/kube writes last. The test then examines the node labels, the cordon, the KubeVirt and Longhorn replica counts, the CDI and longhorn-manager node selectors, and the drained node. It last starts an application, and makes sure that no VMI and no Longhorn replica goes to the tie-breaker, and that the application volume uses the two-replica storage class. The test warns, and does not fail, about the Longhorn node specification, the KubeVirt CR infra replicas, and the node selectors of virt-handler and longhorn-csi-plugin. pkg/kube applies these one time at cluster creation and does not reconcile them, thus other controllers set them back. The warnings show if the values hold at first, or never apply. The test runs in TestNodeClusterSuite after TestThreeNodesCluster. The comment about device reuse in that suite was not correct, because clusterDeviceRequirements prevents reuse. The comment is now correct. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Andrew Durbin --- evetest/clusterconfig.go | 39 +- evetest/setup.go | 13 + evetest/tests/cluster/testsuite_test.go | 12 +- evetest/tests/cluster/tiebreaker_test.go | 469 +++++++++++++++++++++++ 4 files changed, 526 insertions(+), 7 deletions(-) create mode 100644 evetest/tests/cluster/tiebreaker_test.go diff --git a/evetest/clusterconfig.go b/evetest/clusterconfig.go index f89329a6e84..3ca0c4a4287 100644 --- a/evetest/clusterconfig.go +++ b/evetest/clusterconfig.go @@ -20,11 +20,21 @@ import ( // physical port used for intra-cluster communication on this node. // Exactly one node should have BootstrapNode set to true — its ClusterIP // is used as the join server IP for all nodes. +// At most one node may have TieBreaker set to true. type ClusterNode struct { DevName string ClusterIP *net.IPNet ClusterInterface string BootstrapNode bool + + // TieBreaker marks this node as the cluster tie-breaker: the third + // node of an HA cluster, which exists only to give etcd a quorum + // vote. EVE keeps it cordoned, runs no workloads on it and places no + // Longhorn replicas there, which also lowers the replica count of + // the storage class used for new volumes. + // + // Leave it false on every node to configure no tie-breaker at all. + TieBreaker bool } // EdgeClusterConfig manages device configurations for a cluster of edge nodes. @@ -52,9 +62,11 @@ func NewEdgeClusterConfig( th.t.Fatalf("Edge Cluster requires at least one node") } - // Find the bootstrap node to derive the join server IP. - var joinServerIP string + // Find the bootstrap node to derive the join server IP, and the + // tie-breaker node (if any) to derive its UUID below. + var joinServerIP, tieBreakerDevName string bootstrapCount := 0 + tieBreakerCount := 0 for _, node := range nodes { if node.BootstrapNode { bootstrapCount++ @@ -62,11 +74,19 @@ func NewEdgeClusterConfig( joinServerIP = node.ClusterIP.IP.String() } } + if node.TieBreaker { + tieBreakerCount++ + tieBreakerDevName = node.DevName + } } if bootstrapCount != 1 { th.t.Fatalf("Edge Cluster requires exactly one node marked "+ "as BootstrapNode (found %d)", bootstrapCount) } + if tieBreakerCount > 1 { + th.t.Fatalf("Edge Cluster allows at most one node marked "+ + "as TieBreaker (found %d)", tieBreakerCount) + } cc := &EdgeClusterConfig{ th: th, @@ -88,6 +108,20 @@ func NewEdgeClusterConfig( } cc.Token = base64.StdEncoding.EncodeToString(tokenBytes) + // Resolve the tie-breaker device name to the UUID that EVE knows it + // by. Every node is told which node is the tie-breaker, because each + // one compares the configured UUID against its own to decide whether + // it must apply the tie-breaker role to itself. + var tieBreakerNodeID string + if tieBreakerDevName != "" { + id, onboarded := th.deviceUUID(tieBreakerDevName) + if !onboarded { + th.t.Fatalf("Device %q must be onboarded to be the cluster "+ + "tie-breaker (cannot resolve UUID)", tieBreakerDevName) + } + tieBreakerNodeID = id.String() + } + // Apply cluster config to each device with its own ClusterIP // and individually encrypted token. for _, node := range nodes { @@ -112,6 +146,7 @@ func NewEdgeClusterConfig( ClusterType: clusterType, JoinServerIp: joinServerIP, EncryptedClusterToken: cipherData, + TieBreakerNodeId: tieBreakerNodeID, } if node.ClusterIP != nil { dc.Cluster.ClusterIpPrefix = node.ClusterIP.String() diff --git a/evetest/setup.go b/evetest/setup.go index 89f428cd7c9..eb4de850efc 100644 --- a/evetest/setup.go +++ b/evetest/setup.go @@ -1046,6 +1046,19 @@ func (th *TestHarness) isDeviceOnboarded(devName string) bool { return found && devState.ID != uuid.Nil } +// deviceUUID returns the UUID assigned to the device during onboarding. +// The second return value is false if the device is unknown or is not +// onboarded yet, in which case it has no UUID to report. +func (th *TestHarness) deviceUUID(devName string) (uuid.UUID, bool) { + th.devicesM.Lock() + defer th.devicesM.Unlock() + devState, found := th.devices[devName] + if !found || devState.ID == uuid.Nil { + return uuid.Nil, false + } + return devState.ID, true +} + // Wait for the device to publish its ECDH certificate to the controller. func (th *TestHarness) waitForDeviceECDHCert( devName string, devUUID uuid.UUID) (*x509.Certificate, error) { diff --git a/evetest/tests/cluster/testsuite_test.go b/evetest/tests/cluster/testsuite_test.go index 5a7b2ef8887..d6f0ef5b656 100644 --- a/evetest/tests/cluster/testsuite_test.go +++ b/evetest/tests/cluster/testsuite_test.go @@ -18,11 +18,10 @@ import ( // purge runs before the fault-injecting VMIRS test, so a failure in the // ordinary app lifecycle is not masked by chaos. // -// TestClusterToSingleConversion runs last and shares the three-device, -// SeparateClusterPort requirements of TestThreeNodesCluster so the VMs -// are reused. It is ordered after it deliberately: it converts a node -// out of the cluster, which is a destructive change to the topology the -// preceding test relies on. +// Every cluster subtest re-creates its devices, because +// clusterDeviceRequirements sets CreateFromScratchWithLiveImage, which +// maybeReuseDevices always rejects. No subtest inherits cluster state from +// the one before it. // // Test parameters // --------------- @@ -50,6 +49,9 @@ func TestNodeClusterSuite(test *testing.T) { evetest.TestCase{ Test: TestThreeNodesCluster, }, + evetest.TestCase{ + Test: TestTieBreakerCluster, + }, evetest.TestCase{ Test: TestClusterToSingleConversion, }, diff --git a/evetest/tests/cluster/tiebreaker_test.go b/evetest/tests/cluster/tiebreaker_test.go new file mode 100644 index 00000000000..65c0d41ea1e --- /dev/null +++ b/evetest/tests/cluster/tiebreaker_test.go @@ -0,0 +1,469 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package cluster_test + +import ( + "fmt" + "strings" + "testing" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + eveconfig "github.com/lf-edge/eve-api/go/config" + "github.com/lf-edge/eve-api/go/evecommon" + "github.com/lf-edge/eve/evetest" + "github.com/lf-edge/eve/evetest/netmodels" + "github.com/lf-edge/eve/pkg/pillar/types" +) + +const ( + // tieBreakerApplyTimeout bounds how long the kube-init FSM may take to + // finish the tie-breaker phase. The phase drains the node and asks + // Longhorn to evict its replicas, so it is generous. + tieBreakerApplyTimeout = 20 * time.Minute + + // tieBreakerSettleTimeout bounds the individual state checks made once + // the phase reports completion. They are already true by then, so this + // only absorbs the latency of reading them back. + tieBreakerSettleTimeout = 2 * time.Minute + + tieBreakerPollInterval = 15 * time.Second + + // tieBreakerWarnTimeout bounds the checks that only warn. They are quick + // because the values either hold straight after the phase or never do. + tieBreakerWarnTimeout = 60 * time.Second + + // tieBreakerKubectlTimeout bounds a single kubectl invocation. + tieBreakerKubectlTimeout = 2 * time.Minute + + // tieBreakerNodeLabel and its values mirror pkg/kube/kube-init/ + // tiebreaker: the tie-breaker is labelled "true", every other node + // "false", and workloads select on "false" to stay off it. + tieBreakerNodeLabel = "tie-breaker-node" + tieBreakerLabelSet = "true" + tieBreakerLabelUnst = "false" + + // tieBreakerStatusLabel is stamped on every node once the phase + // succeeds, so it is the gate the assertions below wait on. + tieBreakerStatusLabel = "tie-breaker-config-applied=1" + + // Replica count the phase leaves the KubeVirt and Longhorn control + // planes at: one per non-tie-breaker node. + tieBreakerReplicas = "2" +) + +// runKubectl goes through "eve exec kube" because kubectl exists only in the +// kube container, not in the host shell. +func runKubectl(device *evetest.EdgeDevice, args string) (string, error) { + stdout, stderr, err := device.RunShellScript( + "eve exec kube kubectl "+args, tieBreakerKubectlTimeout, 0) + if err != nil { + return "", fmt.Errorf("kubectl %s: %w (stderr: %s)", + args, err, strings.TrimSpace(stderr)) + } + return strings.TrimSpace(stdout), nil +} + +func expectKubectl(t *WithT, device *evetest.EdgeDevice, + what, args, want string, timeout time.Duration) { + t.Eventually(func() (string, error) { + return runKubectl(device, args) + }, timeout, tieBreakerPollInterval).Should(Equal(want), what) +} + +// warnKubectl polls for a state that pkg/kube applies once at cluster +// creation and never reconciles. It warns instead of failing, because the +// owning controller sets these values back. The warning records what the +// state was, which shows whether the value held at first and drifted, or +// never applied. +func warnKubectl(device *evetest.EdgeDevice, what, args, want string) { + log := evetest.Logger() + deadline := time.Now().Add(tieBreakerWarnTimeout) + var last string + for { + out, err := runKubectl(device, args) + switch { + case err != nil: + last = err.Error() + case out == want: + return + default: + last = out + } + if time.Now().After(deadline) { + log.Warnf("tie-breaker: %s: got %q, want %q", what, last, want) + return + } + time.Sleep(tieBreakerPollInterval) + } +} + +// expectKubectlNoField polls until no whitespace-separated field of the query +// output equals unwanted. An empty result passes, so callers must make sure +// the query has something to return. +func expectKubectlNoField(t *WithT, device *evetest.EdgeDevice, + what, args, unwanted string, timeout time.Duration) { + t.Eventually(func() error { + out, err := runKubectl(device, args) + if err != nil { + return err + } + for _, f := range strings.Fields(out) { + if f == unwanted { + return fmt.Errorf("%s: found %q in %q", what, unwanted, out) + } + } + return nil + }, timeout, tieBreakerPollInterval).Should(Succeed()) +} + +// expectKubectlFields takes a wantCount of -1 as "any non-zero number of +// fields", for sets whose size is an implementation detail. +func expectKubectlFields(t *WithT, device *evetest.EdgeDevice, + what, args string, wantCount int, wantEach string, timeout time.Duration) { + t.Eventually(func() error { + out, err := runKubectl(device, args) + if err != nil { + return err + } + fields := strings.Fields(out) + if len(fields) == 0 { + return fmt.Errorf("%s: query returned nothing", what) + } + if wantCount >= 0 && len(fields) != wantCount { + return fmt.Errorf("%s: got %d fields (%q), want %d", + what, len(fields), out, wantCount) + } + for _, f := range fields { + if f != wantEach { + return fmt.Errorf("%s: got %q in %q, want every field to be %q", + what, f, out, wantEach) + } + } + return nil + }, timeout, tieBreakerPollInterval).Should(Succeed()) +} + +// TestTieBreakerCluster verifies the tie-breaker node of a three-node EVE-K +// cluster: the node that holds a quorum vote only, runs no workloads and +// carries no Longhorn replicas. +// +// The designation travels from the EVE API through zedagent's +// EdgeNodeClusterConfig into pkg/kube, which labels, cordons and drains the +// node and configures KubeVirt, CDI and Longhorn to keep off it. This test +// covers that chain, ending with an app deployment that proves no VMI or +// Longhorn replica lands on the tie-breaker. +// +// Network model +// ------------- +// - netmodels.SeparateClusterPort -- the same model TestThreeNodesCluster +// uses: eth0 for mgmt and apps, eth1 for the cluster interconnect. +// +// Device configuration +// -------------------- +// - clusterDeviceRequirements (top of cluster_test.go), three devices. +// edge-dev3 is the tie-breaker; edge-dev1 is the bootstrap node. +// +// Test params +// ----------- +// - TPM (bool), FILESYSTEM. +// +// Suite placement +// --------------- +// - TestNodeClusterSuite, after TestThreeNodesCluster, so a plain +// three-node failure shows in the simpler test first. +func TestTieBreakerCluster(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + // Define configurable parameters available for the test. + evetest.DefineTestParameters( + evetest.TPMParameter(), + evetest.FilesystemParameter(), + ) + + // Get parameter values set for this test execution. + withTPM := evetest.GetTPMParameterValue() + filesystem := evetest.GetFilesystemParameterValue() + + // Set up the test harness and specify the test prerequisites. + var requiredDevices [3]evetest.Requirement + var devName [3]string + for i := 0; i < 3; i++ { + devName[i] = fmt.Sprintf("edge-dev%d", i+1) + requiredDevices[i] = clusterDeviceRequirements(devName[i], withTPM, filesystem) + } + + requiredNetModel := evetest.RequireNetworkModel{ + NetworkModel: netmodels.SeparateClusterPort, + } + var requirements []evetest.Requirement + requirements = append(requirements, requiredDevices[:]...) + requirements = append(requirements, requiredNetModel) + evetest.Setup(requirements...) + evetest.Checkpoint("setup-done") + + // Build the cluster configuration. The last node is the tie-breaker. + const tieBreakerIdx = 2 + var nodes [3]evetest.ClusterNode + for i := 0; i < 3; i++ { + clusterIP := evetest.IPAddressWithPrefix(fmt.Sprintf("10.244.244.%d/24", i+2)) + nodes[i] = evetest.ClusterNode{ + DevName: devName[i], + ClusterIP: clusterIP, + ClusterInterface: "ethernet1", + BootstrapNode: i == 0, + TieBreaker: i == tieBreakerIdx, + } + } + clusterConfig := evetest.NewEdgeClusterConfig( + eveconfig.ClusterType_CLUSTER_TYPE_REPLICATED_STORAGE, + nodes[:]..., + ) + + // Configure network adapters and networks (applied to all devices). + dhcpNet := clusterConfig.AddNetwork( + evetest.DHCPNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + }) + noIPNet := clusterConfig.AddNetwork(evetest.NoIPNetworkConfig{}) + clusterConfig.AddNetworkAdapter( + evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + clusterConfig.AddNetworkAdapter( + evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet1", + PhysicalLabel: "eth1", + InterfaceName: "eth1", + NetworkUUID: noIPNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageShared, + }) + + // Apply the initial configuration to each device in parallel. + cluster := evetest.NewEdgeCluster("test-cluster") + cluster.ApplyConfig(clusterConfig, true, true) + evetest.Checkpoint("initial-config-applied") + + cluster.WaitUntilNodesAreReady(30 * time.Minute) + evetest.Checkpoint("nodes-are-ready") + + log := evetest.Logger() + + // Every node must learn which node is the tie-breaker, because each one + // decides locally whether the role applies to itself. + var tieBreakerNodeID string + for i := 0; i < 3; i++ { + device := evetest.GetEdgeDevice(devName[i]) + var clusterCfg types.EdgeNodeClusterConfig + t.Eventually(func() error { + cfgs, err := evetest.ReadAllPublications[types.EdgeNodeClusterConfig]( + device, "zedagent", false) + if err != nil { + return err + } + if len(cfgs) != 1 { + return fmt.Errorf("%s publishes %d EdgeNodeClusterConfig, want 1", + devName[i], len(cfgs)) + } + clusterCfg = cfgs[0] + return nil + }, 5*time.Minute, tieBreakerPollInterval).Should(Succeed()) + + nodeID := clusterCfg.TieBreakerNodeID.UUID.String() + t.Expect(clusterCfg.TieBreakerNodeID.UUID).NotTo(Equal(evetest.NilUUID), + "%s was given no tie-breaker designation", devName[i]) + if tieBreakerNodeID == "" { + tieBreakerNodeID = nodeID + } + t.Expect(nodeID).To(Equal(tieBreakerNodeID), + "%s disagrees about which node is the tie-breaker", devName[i]) + log.Infof("%s reports tie-breaker node UUID=%s", devName[i], nodeID) + } + evetest.Checkpoint("tie-breaker-designation-agreed") + + // Query Kubernetes through a node that stays schedulable, so the + // tie-breaker's own cordon and drain cannot interfere. + device := evetest.GetEdgeDevice(devName[0]) + + // The phase stamps the status label on every node only after all of its + // steps and the drain succeed, so this is the gate for everything below. + expectKubectlFields(t, device, "tie-breaker phase completed on every node", + "get nodes -l "+tieBreakerStatusLabel+ + ` -o jsonpath="{.items[*].metadata.labels['tie-breaker-config-applied']}"`, + 3, "1", tieBreakerApplyTimeout) + evetest.Checkpoint("tie-breaker-phase-applied") + + // node-uuid is the key the phase itself maps the configured UUID through, + // so the test and the code agree on which node is which. + var tieName string + t.Eventually(func() (string, error) { + out, err := runKubectl(device, "get nodes -l node-uuid="+tieBreakerNodeID+ + ` -o jsonpath="{.items[*].metadata.name}"`) + if err != nil { + return "", err + } + tieName = out + return out, nil + }, tieBreakerSettleTimeout, tieBreakerPollInterval).ShouldNot(BeEmpty(), + "no Kubernetes node carries node-uuid="+tieBreakerNodeID) + t.Expect(strings.Fields(tieName)).To(HaveLen(1), + "expected exactly one node with node-uuid=%s, got %q", + tieBreakerNodeID, tieName) + log.Infof("tie-breaker node UUID=%s is Kubernetes node %q", + tieBreakerNodeID, tieName) + + // Node labels and cordon: the tie-breaker is labelled true and + // cordoned, the other two are labelled false and left schedulable. + // Matching the whole node list against the one designated name proves + // both that the label is on the right node and that no other node + // carries it. + expectKubectl(t, device, "exactly one node is the tie-breaker", + "get nodes -l "+tieBreakerNodeLabel+"="+tieBreakerLabelSet+ + ` -o jsonpath="{.items[*].metadata.name}"`, + tieName, tieBreakerSettleTimeout) + expectKubectl(t, device, "tie-breaker node is cordoned", + fmt.Sprintf(`get node %s -o jsonpath="{.spec.unschedulable}"`, tieName), + "true", tieBreakerSettleTimeout) + expectKubectlFields(t, device, "the other two nodes are labelled as workers", + "get nodes -l "+tieBreakerNodeLabel+"="+tieBreakerLabelUnst+ + ` -o jsonpath="{.items[*].metadata.labels['`+tieBreakerNodeLabel+`']}"`, + 2, tieBreakerLabelUnst, tieBreakerSettleTimeout) + // Matched on the absence of "true" rather than an exact value, because + // spec.unschedulable is omitempty: an uncordoned node may report false + // or omit the field altogether. + expectKubectlNoField(t, device, "the other two nodes stay schedulable", + "get nodes -l "+tieBreakerNodeLabel+"="+tieBreakerLabelUnst+ + ` -o jsonpath="{.items[*].spec.unschedulable}"`, + "true", tieBreakerSettleTimeout) + evetest.Checkpoint("tie-breaker-node-state-verified") + + // KubeVirt: control plane scaled to one replica per worker. + expectKubectl(t, device, "virt-operator is scaled to the worker count", + `get deploy virt-operator -n kubevirt -o jsonpath="{.spec.replicas}"`, + tieBreakerReplicas, tieBreakerSettleTimeout) + warnKubectl(device, "KubeVirt CR infra replicas", + `get kubevirt kubevirt -n kubevirt -o jsonpath="{.spec.infra.replicas}"`, + tieBreakerReplicas) + warnKubectl(device, "virt-handler node selector", + `get ds virt-handler -n kubevirt`+ + ` -o jsonpath="{.spec.template.spec.nodeSelector['`+ + tieBreakerNodeLabel+`']}"`, + tieBreakerLabelUnst) + + // CDI: every Deployment in the namespace kept off the tie-breaker. + expectKubectlFields(t, device, "cdi Deployments avoid the tie-breaker", + `get deploy -n cdi -o jsonpath="{.items[*].spec.template.spec.nodeSelector['`+ + tieBreakerNodeLabel+`']}"`, + -1, tieBreakerLabelUnst, tieBreakerSettleTimeout) + evetest.Checkpoint("tie-breaker-workload-exclusion-verified") + + // longhorn-manager owns the node spec that longhornNodeSetSched writes + // and sets it back. Replica placement below is the invariant that holds. + warnKubectl(device, "longhorn node scheduling on the tie-breaker", + fmt.Sprintf(`get nodes.longhorn.io %s -n longhorn-system`+ + ` -o jsonpath="{.spec.allowScheduling}"`, tieName), "false") + warnKubectl(device, "longhorn eviction of the tie-breaker", + fmt.Sprintf(`get nodes.longhorn.io %s -n longhorn-system`+ + ` -o jsonpath="{.spec.evictionRequested}"`, tieName), "true") + warnKubectl(device, "longhorn disk scheduling on the tie-breaker", + fmt.Sprintf(`get nodes.longhorn.io %s -n longhorn-system`+ + ` -o jsonpath="{.spec.disks.*.allowScheduling}"`, tieName), "false") + + // Longhorn CSI sidecars scaled to the worker count. + for _, deploy := range []string{ + "csi-attacher", "csi-provisioner", "csi-resizer", "csi-snapshotter", + } { + expectKubectl(t, device, "longhorn "+deploy+" is scaled to the worker count", + fmt.Sprintf(`get deploy %s -n longhorn-system`+ + ` -o jsonpath="{.spec.replicas}"`, deploy), + tieBreakerReplicas, tieBreakerSettleTimeout) + } + + // Only longhorn-manager is asserted. longhorn-csi-plugin stands for the + // DaemonSets that longhorn-manager creates after the phase runs, so the + // phase never patches them. + expectKubectl(t, device, "the longhorn-manager DaemonSet avoids the tie-breaker", + `get ds longhorn-manager -n longhorn-system`+ + ` -o jsonpath="{.spec.template.spec.nodeSelector['`+ + tieBreakerNodeLabel+`']}"`, + tieBreakerLabelUnst, tieBreakerSettleTimeout) + warnKubectl(device, "longhorn-csi-plugin node selector", + `get ds longhorn-csi-plugin -n longhorn-system`+ + ` -o jsonpath="{.spec.template.spec.nodeSelector['`+ + tieBreakerNodeLabel+`']}"`, + tieBreakerLabelUnst) + evetest.Checkpoint("tie-breaker-longhorn-verified") + + // The drain leaves only DaemonSet-owned pods behind, which the phase + // deliberately keeps (as `kubectl drain --ignore-daemonsets` does). + expectKubectlFields(t, device, "only DaemonSet pods remain on the tie-breaker", + fmt.Sprintf("get pods -A --field-selector spec.nodeName=%s"+ + ` -o jsonpath="{.items[*].metadata.ownerReferences[*].kind}"`, tieName), + -1, "DaemonSet", tieBreakerSettleTimeout) + evetest.Checkpoint("tie-breaker-drained") + + // Deploy an app so the placement assertions below have something to + // observe. Without a workload they would pass on an empty cluster. + niUUID := clusterConfig.AddNetworkInstance(evetest.LocalNetworkInstanceConfig{ + DisplayName: "local-ni", + Port: "ethernet0", + Subnet: evetest.IPSubnet("10.11.12.0/24"), + DHCPRange: types.IPRange{ + Start: evetest.IPAddress("10.11.12.2"), + End: evetest.IPAddress("10.11.12.254"), + }, + Gateway: evetest.IPAddress("10.11.12.1"), + EnableFlowlog: true, + MTU: 1500, + ForwardLLDP: false, + }) + appUUID := clusterConfig.AddApplication(evetest.ClusterApplicationInstanceConfig{ + ApplicationInstanceConfig: evetest.ApplicationInstanceConfig{ + DisplayName: "tie-breaker-app", + Activate: true, + Image: evetest.DockerContainer{ + ImageName: "lfedge/evetest-ubuntu-ctr", + Tag: "1.0", + }, + CPUs: 1, + MemoryBytes: 500 * evetest.MiB, + NetworkAdapters: []evetest.AppNetworkAdapter{ + evetest.VirtualNetworkAdapter{ + LogicalLabel: "vif0", + NetworkInstanceUUID: niUUID, + }, + }, + }, + DesignatedNodeName: devName[0], + Affinity: eveconfig.AffinityType_AFFINITY_TYPE_PREFERRED, + }) + cluster.ApplyConfig(clusterConfig, true, true) + cluster.WaitUntilAppIsRunning(appUUID, 20*time.Minute) + evetest.Checkpoint("app-is-running") + + // The tie-breaker exists to hold a quorum vote only, so neither the + // app's VMI nor any Longhorn replica may land on it. + expectKubectlNoField(t, device, "no VMI runs on the tie-breaker", + `get vmi -A -o jsonpath="{.items[*].status.nodeName}"`, + tieName, tieBreakerSettleTimeout) + expectKubectlNoField(t, device, "no Longhorn replica sits on the tie-breaker", + `get replicas.longhorn.io -n longhorn-system`+ + ` -o jsonpath="{.items[*].spec.nodeID}"`, + tieName, tieBreakerSettleTimeout) + + // A tie-breaker lowers the replica count of the storage class EVE picks + // for new volumes, because only the two workers can hold replicas. + expectKubectlFields(t, device, "app volumes use the two-replica storage class", + `get pvc -n eve-kube-app -o jsonpath="{.items[*].spec.storageClassName}"`, + -1, "lh-sc-rep2", tieBreakerSettleTimeout) + evetest.Checkpoint("tie-breaker-placement-verified") +} From 6de65f17e11b469cc5688bfcf0c1089f21d1f77e Mon Sep 17 00:00:00 2001 From: Andrew Durbin Date: Thu, 13 Aug 2026 17:02:29 -0600 Subject: [PATCH 2/2] pkg/kube: rebuild the kubeclient when the k3s kubeconfig changes initKubeclient builds the process-wide client-go client one time, at EvK3sReady, from /etc/rancher/k3s/k3s.yaml. No code ever rebuilds it after that. A node that joins a multi-node cluster can have its local k3s server change its TLS material after EvK3sReady fires. The cached client then fails every API call with "certificate signed by unknown authority", and it never recovers. reapplyNodeLabels uses this same client, so state.NodeLabelsInitialized never gets set, and the tie-breaker phase skips forever on its own first check ("node labels not yet initialized"). The old shell implementation did not have this problem. It ran a new kubectl process for each call, and each process read the current kubeconfig file fresh from disk. kubeconfigSyncLoop already reads the kubeconfig file on every tick, to mirror it for other consumers. This change adds a second check on the same tick: rebuild kubeclient.Default() when the file content differs from what the current client was built from, and stop the informer goroutines of the client it replaces. The very first tick always rebuilds the client one time, even when nothing changed, because there is no known content to compare against yet. The client that initKubeclient built has no stop function to call, so its informer goroutines stay until the process ends. Both costs happen at most one time per node boot. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Andrew Durbin --- pkg/kube/kube-init/monitor/monitor.go | 84 ++++++++++++++++++++++----- 1 file changed, 71 insertions(+), 13 deletions(-) diff --git a/pkg/kube/kube-init/monitor/monitor.go b/pkg/kube/kube-init/monitor/monitor.go index 5eea8defc00..f791b8d0ef9 100644 --- a/pkg/kube/kube-init/monitor/monitor.go +++ b/pkg/kube/kube-init/monitor/monitor.go @@ -610,28 +610,86 @@ func sendSIGHUPToK3s() { // Kubeconfig sync // --------------------------------------------------------------------------- +// lastKubeclientSrc is the content of state.K3sKubeconfig that the +// current kubeclient.Default() was built from, and kubeclientCancel +// stops that client's informer goroutines. Both are nil until +// rebuildKubeclientIfChanged first runs. +var ( + kubeclientMu sync.Mutex + lastKubeclientSrc []byte + kubeclientCancel context.CancelFunc +) + // SyncKubeconfig mirrors state.K3sKubeconfig into k3s.KubeconfigCopy -// when the source has changed. Other components (registration, -// debug-user) consume the copy from /run instead of reaching into -// /etc/rancher. -func SyncKubeconfig() { +// when the source has changed, and rebuilds kubeclient.Default() when +// its content no longer matches what the running client was built +// from. Other components (registration, debug-user) consume the /run +// copy instead of reaching into /etc/rancher; kube-init's own +// subsystems consume kubeclient.Default() directly. +// +// A node joining a multi-node cluster can have its local k3s server +// regenerate its TLS material after initKubeclient already cached a +// client-go client against the old material. Left unhandled, every +// subsequent API call from kubeclient.Default() fails permanently +// with "certificate signed by unknown authority" -- which blocks node- +// label initialization and, transitively, the tie-breaker phase, since +// nothing else ever rebuilds that client. +func SyncKubeconfig(ctx context.Context) { srcData, err := os.ReadFile(state.K3sKubeconfig) if err != nil { return } - if dstData, _ := os.ReadFile(k3s.KubeconfigCopy); string(dstData) == string(srcData) { - return + + if dstData, _ := os.ReadFile(k3s.KubeconfigCopy); string(dstData) != string(srcData) { + if err := os.MkdirAll(filepath.Dir(k3s.KubeconfigCopy), 0755); err != nil { + log.Printf("warning: mkdir for kubeconfig sync: %v", err) + } else if err := os.WriteFile(k3s.KubeconfigCopy, srcData, 0600); err != nil { + log.Printf("warning: kubeconfig sync failed: %v", err) + } else { + log.Printf("synced kubeconfig %s → %s", + state.K3sKubeconfig, k3s.KubeconfigCopy) + } } - if err := os.MkdirAll(filepath.Dir(k3s.KubeconfigCopy), 0755); err != nil { - log.Printf("warning: mkdir for kubeconfig sync: %v", err) + + rebuildKubeclientIfChanged(ctx, srcData) +} + +// rebuildKubeclientIfChanged replaces kubeclient.Default() with a +// client built from srcData when srcData differs from the content the +// running default client was built from. Tracked separately from the +// /run mirror above: a transient parse failure here must be retried on +// the next tick regardless of whether the mirror write already +// succeeded with that same content. +func rebuildKubeclientIfChanged(ctx context.Context, srcData []byte) { + kubeclientMu.Lock() + unchanged := lastKubeclientSrc != nil && string(lastKubeclientSrc) == string(srcData) + kubeclientMu.Unlock() + if unchanged { return } - if err := os.WriteFile(k3s.KubeconfigCopy, srcData, 0600); err != nil { - log.Printf("warning: kubeconfig sync failed: %v", err) + + kc, err := kubeclient.New(state.K3sKubeconfig) + if err != nil { + log.Printf("warning: rebuild kubeclient after kubeconfig change: %v", err) return } - log.Printf("synced kubeconfig %s → %s", - state.K3sKubeconfig, k3s.KubeconfigCopy) + kcCtx, cancel := context.WithCancel(ctx) + kc.Start(kcCtx.Done()) + kubeclient.SetDefault(kc) + log.Printf("kubeclient: rebuilt from %s after kubeconfig change", state.K3sKubeconfig) + + kubeclientMu.Lock() + lastKubeclientSrc = srcData + prevCancel := kubeclientCancel + kubeclientCancel = cancel + kubeclientMu.Unlock() + if prevCancel != nil { + // Stop the informer goroutines of the client this one replaces. + // The very first rebuild has no predecessor to stop here, since + // initKubeclient's own client is tracked only by + // kubeclient.Default(), not by this cancel chain. + prevCancel() + } } // --------------------------------------------------------------------------- @@ -702,7 +760,7 @@ func (m *Monitor) kubeconfigSyncLoop(ctx context.Context) { case <-ctx.Done(): return case <-ticker.C: - SyncKubeconfig() + SyncKubeconfig(ctx) } } }