eve-k: add cluster tie-breaker support to evetest, fix pkg/kube kubeclient regression

evetest/clusterconfig.go

@@ -20,11 +20,21 @@ import (
 // physical port used for intra-cluster communication on this node.
 // Exactly one node should have BootstrapNode set to true — its ClusterIP
 // is used as the join server IP for all nodes.
+// At most one node may have TieBreaker set to true.
 type ClusterNode struct {
 	DevName          string
 	ClusterIP        *net.IPNet
 	ClusterInterface string
 	BootstrapNode    bool
+
+	// TieBreaker marks this node as the cluster tie-breaker: the third
+	// node of an HA cluster, which exists only to give etcd a quorum
+	// vote. EVE keeps it cordoned, runs no workloads on it and places no
+	// Longhorn replicas there, which also lowers the replica count of
+	// the storage class used for new volumes.
+	//
+	// Leave it false on every node to configure no tie-breaker at all.
+	TieBreaker bool
 }
 
 // EdgeClusterConfig manages device configurations for a cluster of edge nodes.
@@ -52,21 +62,31 @@ func NewEdgeClusterConfig(
 		th.t.Fatalf("Edge Cluster requires at least one node")
 	}
 
-	// Find the bootstrap node to derive the join server IP.
-	var joinServerIP string
+	// Find the bootstrap node to derive the join server IP, and the
+	// tie-breaker node (if any) to derive its UUID below.
+	var joinServerIP, tieBreakerDevName string
 	bootstrapCount := 0
+	tieBreakerCount := 0
 	for _, node := range nodes {
 		if node.BootstrapNode {
 			bootstrapCount++
 			if node.ClusterIP != nil {
 				joinServerIP = node.ClusterIP.IP.String()
 			}
 		}
+		if node.TieBreaker {
+			tieBreakerCount++
+			tieBreakerDevName = node.DevName
+		}
 	}
 	if bootstrapCount != 1 {
 		th.t.Fatalf("Edge Cluster requires exactly one node marked "+
 			"as BootstrapNode (found %d)", bootstrapCount)
 	}
+	if tieBreakerCount > 1 {
+		th.t.Fatalf("Edge Cluster allows at most one node marked "+
+			"as TieBreaker (found %d)", tieBreakerCount)
+	}
 
 	cc := &EdgeClusterConfig{
 		th:      th,
@@ -88,6 +108,20 @@ func NewEdgeClusterConfig(
 	}
 	cc.Token = base64.StdEncoding.EncodeToString(tokenBytes)
 
+	// Resolve the tie-breaker device name to the UUID that EVE knows it
+	// by. Every node is told which node is the tie-breaker, because each
+	// one compares the configured UUID against its own to decide whether
+	// it must apply the tie-breaker role to itself.
+	var tieBreakerNodeID string
+	if tieBreakerDevName != "" {
+		id, onboarded := th.deviceUUID(tieBreakerDevName)
+		if !onboarded {
+			th.t.Fatalf("Device %q must be onboarded to be the cluster "+
+				"tie-breaker (cannot resolve UUID)", tieBreakerDevName)
+		}
+		tieBreakerNodeID = id.String()
+	}
+
 	// Apply cluster config to each device with its own ClusterIP
 	// and individually encrypted token.
 	for _, node := range nodes {
@@ -112,6 +146,7 @@ func NewEdgeClusterConfig(
 			ClusterType:           clusterType,
 			JoinServerIp:          joinServerIP,
 			EncryptedClusterToken: cipherData,
+			TieBreakerNodeId:      tieBreakerNodeID,
 		}
 		if node.ClusterIP != nil {
 			dc.Cluster.ClusterIpPrefix = node.ClusterIP.String()

evetest/setup.go

@@ -1046,6 +1046,19 @@ func (th *TestHarness) isDeviceOnboarded(devName string) bool {
 	return found && devState.ID != uuid.Nil
 }
 
+// deviceUUID returns the UUID assigned to the device during onboarding.
+// The second return value is false if the device is unknown or is not
+// onboarded yet, in which case it has no UUID to report.
+func (th *TestHarness) deviceUUID(devName string) (uuid.UUID, bool) {
+	th.devicesM.Lock()
+	defer th.devicesM.Unlock()
+	devState, found := th.devices[devName]
+	if !found || devState.ID == uuid.Nil {
+		return uuid.Nil, false
+	}
+	return devState.ID, true
+}
+
 // Wait for the device to publish its ECDH certificate to the controller.
 func (th *TestHarness) waitForDeviceECDHCert(
 	devName string, devUUID uuid.UUID) (*x509.Certificate, error) {

evetest/tests/cluster/testsuite_test.go

@@ -18,11 +18,10 @@ import (
 // purge runs before the fault-injecting VMIRS test, so a failure in the
 // ordinary app lifecycle is not masked by chaos.
 //
-// TestClusterToSingleConversion runs last and shares the three-device,
-// SeparateClusterPort requirements of TestThreeNodesCluster so the VMs
-// are reused. It is ordered after it deliberately: it converts a node
-// out of the cluster, which is a destructive change to the topology the
-// preceding test relies on.
+// Every cluster subtest re-creates its devices, because
+// clusterDeviceRequirements sets CreateFromScratchWithLiveImage, which
+// maybeReuseDevices always rejects. No subtest inherits cluster state from
+// the one before it.
 //
 // Test parameters
 // ---------------
@@ -50,6 +49,9 @@ func TestNodeClusterSuite(test *testing.T) {
 		evetest.TestCase{
 			Test: TestThreeNodesCluster,
 		},
+		evetest.TestCase{
+			Test: TestTieBreakerCluster,
+		},
 		evetest.TestCase{
 			Test: TestClusterToSingleConversion,
 		},