From 7257ce618db9b1f4f3210ae35e0ba65c18b5a644 Mon Sep 17 00:00:00 2001 From: Jeremy Poulin Date: Thu, 25 Jun 2026 15:55:46 -0400 Subject: [PATCH 1/6] OCPBUGS-84695: feat(tnf): multi-node job controller framework Implements the foundational framework for multi-node job scheduling with round-robin retry logic and drift detection: - Multi-node job support with NodeTarget struct for node-specific jobs - Round-robin retry logic with state tracking (AttemptNumber, NodeIndex, LastJobConfig) - Job config drift detection (resets retry state when config changes) - Degradation when retry attempts exhausted - IsControllerRunning() helper to prevent duplicate controllers - Shared node utility functions (GetNodeNames, DetermineReconciliationActions, IsNodeReady) - Job type labels for filtering and cleanup See docs/tnf/job-controllers.md for detailed multi-node retry behavior and config drift examples. Co-Authored-By: Claude Sonnet 4.5 --- bindata/tnfdeployment/role.yaml | 9 + docs/tnf/job-controllers.md | 270 +++++++++++++ pkg/operator/ceohelpers/node_helpers.go | 40 ++ pkg/operator/ceohelpers/node_helpers_test.go | 239 +++++++++++ pkg/tnf/pkg/jobs/jobcontroller.go | 30 ++ pkg/tnf/pkg/jobs/tnf.go | 401 ++++++++++++++++++- pkg/tnf/pkg/jobs/tnf_test.go | 221 ++++++---- pkg/tnf/pkg/jobs/utils.go | 38 +- pkg/tnf/pkg/tools/nodes.go | 58 +++ pkg/tnf/pkg/tools/tnf_labels.go | 5 + 10 files changed, 1223 insertions(+), 88 deletions(-) create mode 100644 docs/tnf/job-controllers.md create mode 100644 pkg/operator/ceohelpers/node_helpers.go create mode 100644 pkg/operator/ceohelpers/node_helpers_test.go create mode 100644 pkg/tnf/pkg/tools/tnf_labels.go diff --git a/bindata/tnfdeployment/role.yaml b/bindata/tnfdeployment/role.yaml index 37588cc1cd..0b51deb7f2 100644 --- a/bindata/tnfdeployment/role.yaml +++ b/bindata/tnfdeployment/role.yaml @@ -22,6 +22,15 @@ rules: - get - list - watch + # update-setup lists ConfigMaps labeled app.kubernetes.io/component=tnf-update-setup (see pkg/tnf/update-setup/runner.go). + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch - apiGroups: - batch resources: diff --git a/docs/tnf/job-controllers.md b/docs/tnf/job-controllers.md new file mode 100644 index 0000000000..6d64cda4e6 --- /dev/null +++ b/docs/tnf/job-controllers.md @@ -0,0 +1,270 @@ +# TNF Job Controllers + +## Overview + +Job controllers manage the lifecycle of Kubernetes Jobs that configure and maintain the Pacemaker cluster. `StartJobControllers()` adapts its behavior based on whether the external etcd transition has completed (see [Dual-Mode Operation](architecture-overview.md#dual-mode-operation) in the architecture overview). + +## Bootstrap vs Runtime Behavior + +### Bootstrap Mode (Pre-Transition) + +Drives initial transition from static pod etcd to Pacemaker control: +- Requires exactly 2 ready nodes (Pacemaker limitation) +- Exponential backoff retry (5s to 2min, ~10min total) +- Sets `TNFJobControllersDegraded` condition +- Uses mutex to prevent concurrent attempts + +### Runtime Mode (Post-Transition) + +Ensures job controllers stay running: +- Supports 1 or 2 ready nodes (handles node replacement) +- Idempotent - safe to call repeatedly +- No explicit retry (controller framework retries sync()) +- On operator restart: skips bootstrap if setup job already complete + +## StartJobControllers() Decision Flow + +```text +StartJobControllers() + │ + ├─ Check HasExternalEtcdCompletedTransition() + │ + ├─ Pre-transition (Bootstrap): + │ - Require exactly 2 ready nodes + │ - Exponential backoff retry (5s→2min, 9 steps, ~10min total) + │ - Set TNFJobControllersDegraded condition on failure + │ + └─ Post-transition (Runtime): + - Require all nodes ready (supports 1 or 2 nodes) + - Idempotent (uses jobControllersStarted flag) + - On operator restart: skip bootstrap if setup job complete +``` + +## startTnfJobcontrollers() Bootstrap Sequence + +**Fast path (operator restart, setup job complete):** +Skip bootstrap flow and start all job controllers without re-running bootstrap checks (etcd informer sync, stable revision waits). Controllers are idempotent and won't recreate completed jobs. + +**Bootstrap path (initial setup):** +1. Wait for etcd informer sync, etcd bootstrap completed, stable revision +2. Start per-node controllers: auth, after-setup +3. Start cluster-wide controllers: setup, fencing + +## Job Types and Behaviors + +See [Job Types](architecture-overview.md#job-types) in architecture overview for the complete table. + +**Node-Specific (auth, after-setup):** +- Labeled with node UID, cleaned up on node deletion +- Started when node becomes Ready +- **auth:** Sets hacluster password (required before Pacemaker operations) +- **after-setup:** Disables kubelet systemd service (Pacemaker owns it now) + +**Cluster-Wide (setup, fencing, update-setup):** +- Multi-node retry with round-robin selection +- **setup:** Creates Pacemaker cluster, marks transition complete +- **fencing:** Configures STONITH for BMC power control +- **update-setup:** Adds/removes nodes (Day 2 reconciliation) + +## Multi-Node Retry Logic + +Cluster-wide jobs (setup, fencing, update-setup) use `targetNodesFunc` to retry across multiple nodes on failure. Round-robin selection tries each valid target node before degrading. + +**Retry State Tracking:** +- `AttemptNumber`: Current attempt (1-N) +- `NodeIndex`: Which node to try next (round-robin) +- `LastJobConfig`: Detects config drift (resets state if config changes) + +**Node Selection:** Tries all nodes per attempt, with 3 max attempts: +- Attempt 1: master-0, then master-1 (if master-0 fails) +- Attempt 2: master-0, then master-1 (if master-0 fails) +- Attempt 3: master-0, then master-1 (if master-0 fails) +- Total: up to 6 tries (3 attempts × 2 nodes) +- All attempts exhausted → set Degraded condition, reset to attempt 1 (allows recovery) + +**Config Drift:** If `targetNodesFunc` returns different nodes or `jobConfigFunc` returns different config, retry state resets to attempt 1 (prevents using stale decisions). + +**Example: Fencing Job Config** +```go +// jobConfigFunc captures node UIDs and fencing secret UIDs to detect infrastructure changes +fencingJobConfigFunc := func() (string, error) { + config := map[string]interface{}{ + "nodeUIDs": []string{node1.UID, node2.UID}, // Detect node replacements + "secretUIDs": []string{secret1.UID, secret2.UID}, // Detect secret rotation + } + return json.Marshal(config) +} +``` +If a node is replaced (same name, different UID) or fencing secrets are rotated, the config changes and the fencing job is recreated with updated BMC credentials. + +## Job Flow Diagrams + +### Bootstrap Job Sequence + +StartJobControllers starts job controllers (not jobs directly): + +```text +1. Start auth + after-setup controllers (per-node) +2. Start setup + fencing controllers (cluster-wide) +3. Return (startTnfJobcontrollers complete) + +Job controllers run independently: +- Auth jobs set hacluster password +- Setup job configures Pacemaker cluster and marks transition complete +- After-setup jobs disable kubelet +- Fencing job configures STONITH + +Bootstrap complete when startTnfJobcontrollers returns. +``` + +**Important timing:** +- **Transition complete**: Marked by setup JOB when it calls etcd.RemoveStaticContainer() +- **Reconciliation guard**: Waits for setup JOB completion (ensures Pacemaker cluster exists) +- Jobs run independently - setup may finish before after-setup completes + +### Reconciliation Job Sequence (Node Added) + +```text +New node master-2 becomes Ready + │ + ├─ StartJobControllers() triggered (Ready transition) + │ └─ auth-master-0, auth-master-1, auth-master-2 controllers started + │ (Start running auth jobs to set hacluster password) + │ + ├─ ReconcilePacemakerConfig() triggered (drift detection) + │ └─ update-setup job started + │ │ + │ ├─ Wait for ALL auth jobs to complete + │ │ (Ensures hacluster password set before Pacemaker operations) + │ │ + │ └─ Add master-2 to Pacemaker cluster + │ + ├─ after-setup controllers (started by StartJobControllers) + │ └─ Wait for setup job complete, then disable kubelet + │ + └─ Reconciliation complete +``` + +**Auth job dependency:** Update-setup waits for auth jobs to complete before running Pacemaker operations (similar to how setup job waits for auth during bootstrap). This prevents failures from missing hacluster passwords. + +### Reconciliation Job Sequence (Node Removed) + +```text +Node master-1 deleted from K8s + │ + ├─ ReconcilePacemakerConfig() triggered (drift detection) + │ └─ update-setup job started (removes master-1 from Pacemaker) + │ + ├─ CleanupOrphanedJobs() triggered (periodic sync) + │ └─ Delete auth-master-1, after-setup-master-1 (node UID no longer exists) + │ + └─ Reconciliation complete +``` + +## Fencing Secret Changes + +**What:** Automatic fencing job restart when BMC credentials are updated +**When:** After external etcd transition completes +**Why:** Allows dynamic credential updates without manual intervention + +### Flow Overview + +```text +User updates secret (tnf-fencing-*) + │ + ▼ +Secret informer detects change + │ + ▼ +Is it a fencing secret? + │ + ┌───┴───┐ + No Yes + │ │ + ▼ ▼ + Skip Data changed? + │ + ┌───┴───┐ + No Yes + │ │ + ▼ ▼ + Skip Transition complete? + │ + ┌───┴───┐ + No Yes + │ │ + ▼ ▼ + Skip Restart fencing job + │ + ▼ + New credentials applied +``` + +### Three Fencing Configuration Points + +Fencing (STONITH) is configured at three distinct points: + +| When | Trigger | Method | +|------|---------|--------| +| **Bootstrap** | Both nodes Ready | Standalone fencing job in bootstrap sequence | +| **Secret changes** | `tnf-fencing-*` updated | Automatic job restart (this section) | +| **Membership changes** | Nodes added/removed | Inline during update-setup reconciliation | + +**Why three?** Different lifecycle phases need different approaches - bootstrap uses a job, secret changes trigger reactive restart, membership changes update inline during reconciliation. + +### Operational Workflow + +To update BMC credentials: + +```text +1. Edit secret: oc edit secret tnf-fencing-master-0 -n openshift-etcd + │ + ▼ +2. Save changes + │ + ▼ +3. Automatic: Fencing job restarts within seconds + │ + ▼ +4. New credentials applied to Pacemaker STONITH +``` + +**No manual intervention required.** + +## Concurrency Protection + +- **`startJobControllersMu` mutex:** Prevents concurrent job controller startup +- **Job-level idempotency:** Jobs check if work already done (e.g., `clusterExists()` before cluster creation) +- **`reconcilePacemakerConfigMutex`:** Serializes drift detection and reconciliation + +## Error Handling + +**Bootstrap (pre-transition):** +- Exponential backoff retry (5s to 2min, ~10min total) +- After exhausting retries → set `TNFJobControllersDegraded=True` + +**Runtime (post-transition):** +- No explicit retry in StartJobControllers (controller framework retries sync() every 30s) +- Multi-node jobs retry across nodes (see Multi-Node Retry Logic above) + +## Integration with Other Components + +**StartJobControllers** (this document): +- Starts auth, after-setup controllers when nodes become Ready +- Starts setup, fencing controllers during bootstrap +- Independent of drift detection + +**ReconcilePacemakerConfig** (see [reconciliation.md](reconciliation.md)): +- Detects drift between K8s and Pacemaker +- Creates update-setup ConfigMap and starts update-setup job +- **Does NOT start auth/after-setup jobs** (those run independently via StartJobControllers) + +**CleanupOrphanedJobs** (see [lifecycle_cleanup.go](../../pkg/tnf/operator/lifecycle_cleanup.go)): +- Deletes jobs for deleted nodes (by node UID) +- Runs independently during periodic sync + +**MonitorHealth** (see [lifecycle_health.go](../../pkg/tnf/operator/lifecycle_health.go)): +- Reads PacemakerCluster CR status +- Reports operator conditions +- No interaction with job controllers + diff --git a/pkg/operator/ceohelpers/node_helpers.go b/pkg/operator/ceohelpers/node_helpers.go new file mode 100644 index 0000000000..906ba8b8b8 --- /dev/null +++ b/pkg/operator/ceohelpers/node_helpers.go @@ -0,0 +1,40 @@ +package ceohelpers + +import ( + "fmt" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + corev1listers "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" +) + +// Control plane node label selectors +const ( + // ControlPlaneNodeLabelSelector is the label used to identify control plane nodes + ControlPlaneNodeLabelSelector = "node-role.kubernetes.io/control-plane" + + // ArbiterNodeLabelSelector is the label used to identify arbiter nodes in SNO+1 configurations + ArbiterNodeLabelSelector = "node-role.kubernetes.io/arbiter" +) + +// ListNodesFromInformer returns all nodes from the given informer. +// This is a convenience wrapper around creating a lister and listing all nodes. +func ListNodesFromInformer(informer cache.SharedIndexInformer) ([]*corev1.Node, error) { + if informer == nil { + return nil, fmt.Errorf("informer is nil") + } + + lister := corev1listers.NewNodeLister(informer.GetIndexer()) + return lister.List(labels.Everything()) +} + +// ListNodesBySelector returns nodes matching the given label selector from the informer. +func ListNodesBySelector(informer cache.SharedIndexInformer, selector labels.Selector) ([]*corev1.Node, error) { + if informer == nil { + return nil, fmt.Errorf("informer is nil") + } + + lister := corev1listers.NewNodeLister(informer.GetIndexer()) + return lister.List(selector) +} diff --git a/pkg/operator/ceohelpers/node_helpers_test.go b/pkg/operator/ceohelpers/node_helpers_test.go new file mode 100644 index 0000000000..a7bf003add --- /dev/null +++ b/pkg/operator/ceohelpers/node_helpers_test.go @@ -0,0 +1,239 @@ +package ceohelpers + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/tools/cache" +) + +func TestListNodesFromInformer(t *testing.T) { + tests := []struct { + name string + nodes []*corev1.Node + expectError bool + expectCount int + }{ + { + name: "list all nodes", + nodes: []*corev1.Node{ + {ObjectMeta: metav1.ObjectMeta{Name: "master-0"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "master-1"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "worker-0"}}, + }, + expectError: false, + expectCount: 3, + }, + { + name: "empty node list", + nodes: []*corev1.Node{}, + expectError: false, + expectCount: 0, + }, + { + name: "single node", + nodes: []*corev1.Node{ + {ObjectMeta: metav1.ObjectMeta{Name: "master-0"}}, + }, + expectError: false, + expectCount: 1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create fake informer with nodes + informer := createFakeNodeInformer(t, tt.nodes) + + // Execute + nodes, err := ListNodesFromInformer(informer) + + // Verify + if tt.expectError { + require.Error(t, err) + } else { + require.NoError(t, err) + require.Len(t, nodes, tt.expectCount) + } + }) + } +} + +func TestListNodesFromInformer_NilInformer(t *testing.T) { + nodes, err := ListNodesFromInformer(nil) + + require.Error(t, err) + require.Nil(t, nodes) + require.Contains(t, err.Error(), "informer is nil") +} + +func TestListNodesBySelector(t *testing.T) { + tests := []struct { + name string + nodes []*corev1.Node + selector labels.Selector + expectError bool + expectCount int + expectNames []string + }{ + { + name: "select master nodes only", + nodes: []*corev1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "master-0", + Labels: map[string]string{ControlPlaneNodeLabelSelector: ""}, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "master-1", + Labels: map[string]string{ControlPlaneNodeLabelSelector: ""}, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-0", + Labels: map[string]string{"node-role.kubernetes.io/worker": ""}, + }, + }, + }, + selector: labels.SelectorFromSet(map[string]string{ControlPlaneNodeLabelSelector: ""}), + expectError: false, + expectCount: 2, + expectNames: []string{"master-0", "master-1"}, + }, + { + name: "select arbiter nodes only", + nodes: []*corev1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "master-0", + Labels: map[string]string{ControlPlaneNodeLabelSelector: ""}, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "arbiter-0", + Labels: map[string]string{ArbiterNodeLabelSelector: ""}, + }, + }, + }, + selector: labels.SelectorFromSet(map[string]string{ArbiterNodeLabelSelector: ""}), + expectError: false, + expectCount: 1, + expectNames: []string{"arbiter-0"}, + }, + { + name: "no matching nodes", + nodes: []*corev1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-0", + Labels: map[string]string{"node-role.kubernetes.io/worker": ""}, + }, + }, + }, + selector: labels.SelectorFromSet(map[string]string{ControlPlaneNodeLabelSelector: ""}), + expectError: false, + expectCount: 0, + expectNames: []string{}, + }, + { + name: "select all nodes with Everything() selector", + nodes: []*corev1.Node{ + {ObjectMeta: metav1.ObjectMeta{Name: "master-0"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "worker-0"}}, + }, + selector: labels.Everything(), + expectError: false, + expectCount: 2, + expectNames: []string{"master-0", "worker-0"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create fake informer with nodes + informer := createFakeNodeInformer(t, tt.nodes) + + // Execute + nodes, err := ListNodesBySelector(informer, tt.selector) + + // Verify + if tt.expectError { + require.Error(t, err) + } else { + require.NoError(t, err) + require.Len(t, nodes, tt.expectCount) + + // Verify node names match + actualNames := make([]string, len(nodes)) + for i, node := range nodes { + actualNames[i] = node.Name + } + require.ElementsMatch(t, tt.expectNames, actualNames) + } + }) + } +} + +func TestListNodesBySelector_NilInformer(t *testing.T) { + selector := labels.SelectorFromSet(map[string]string{ControlPlaneNodeLabelSelector: ""}) + nodes, err := ListNodesBySelector(nil, selector) + + require.Error(t, err) + require.Nil(t, nodes) + require.Contains(t, err.Error(), "informer is nil") +} + +// createFakeNodeInformer creates a fake SharedIndexInformer populated with the given nodes +func createFakeNodeInformer(t *testing.T, nodes []*corev1.Node) cache.SharedIndexInformer { + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{}) + for _, node := range nodes { + if err := indexer.Add(node); err != nil { + t.Fatalf("indexer.Add failed: %v", err) + } + } + + return &fakeNodeInformer{indexer: indexer} +} + +// fakeNodeInformer implements cache.SharedIndexInformer for testing +type fakeNodeInformer struct { + indexer cache.Indexer +} + +func (f *fakeNodeInformer) AddEventHandler(handler cache.ResourceEventHandler) (cache.ResourceEventHandlerRegistration, error) { + return nil, nil +} +func (f *fakeNodeInformer) AddEventHandlerWithResyncPeriod(handler cache.ResourceEventHandler, resyncPeriod time.Duration) (cache.ResourceEventHandlerRegistration, error) { + return nil, nil +} +func (f *fakeNodeInformer) AddEventHandlerWithOptions(handler cache.ResourceEventHandler, options cache.HandlerOptions) (cache.ResourceEventHandlerRegistration, error) { + return nil, nil +} +func (f *fakeNodeInformer) RemoveEventHandler(handle cache.ResourceEventHandlerRegistration) error { + return nil +} +func (f *fakeNodeInformer) GetStore() cache.Store { return f.indexer } +func (f *fakeNodeInformer) GetController() cache.Controller { return nil } +func (f *fakeNodeInformer) Run(stopCh <-chan struct{}) {} +func (f *fakeNodeInformer) RunWithContext(ctx context.Context) {} +func (f *fakeNodeInformer) HasSynced() bool { return true } +func (f *fakeNodeInformer) LastSyncResourceVersion() string { return "" } +func (f *fakeNodeInformer) SetWatchErrorHandler(handler cache.WatchErrorHandler) error { + return nil +} +func (f *fakeNodeInformer) SetWatchErrorHandlerWithContext(handler cache.WatchErrorHandlerWithContext) error { + return nil +} +func (f *fakeNodeInformer) SetTransform(handler cache.TransformFunc) error { return nil } +func (f *fakeNodeInformer) IsStopped() bool { return false } +func (f *fakeNodeInformer) AddIndexers(indexers cache.Indexers) error { return nil } +func (f *fakeNodeInformer) GetIndexer() cache.Indexer { return f.indexer } diff --git a/pkg/tnf/pkg/jobs/jobcontroller.go b/pkg/tnf/pkg/jobs/jobcontroller.go index 3d11156e01..8ca45fc1c4 100644 --- a/pkg/tnf/pkg/jobs/jobcontroller.go +++ b/pkg/tnf/pkg/jobs/jobcontroller.go @@ -14,6 +14,7 @@ import ( "github.com/openshift/library-go/pkg/operator/management" "github.com/openshift/library-go/pkg/operator/v1helpers" batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/errors" @@ -233,6 +234,35 @@ func (c *JobController) syncManaged(ctx context.Context, opSpec *opv1.OperatorSp return err } + // Auto-recovery: delete jobs stuck in Failed state for > 10 minutes + // This handles controller parameter migrations and stuck jobs + if IsFailed(*job) { + // Check when the job failed + var failedTime *metav1.Time + for _, condition := range job.Status.Conditions { + if condition.Type == batchv1.JobFailed && condition.Status == corev1.ConditionTrue { + failedTime = &condition.LastTransitionTime + break + } + } + + if failedTime != nil && time.Since(failedTime.Time) > 10*time.Minute { + klog.Warningf("Job %s has been failed for %v, deleting to force restart (controller parameter migration or stuck job recovery)", + job.Name, time.Since(failedTime.Time).Round(time.Second)) + + err := c.kubeClient.BatchV1().Jobs(job.Namespace).Delete(ctx, job.Name, metav1.DeleteOptions{ + PropagationPolicy: ptr.To(metav1.DeletePropagationBackground), + }) + if err != nil && !apierrors.IsNotFound(err) { + klog.Warningf("Failed to delete stuck job %s: %v", job.Name, err) + } else { + klog.Infof("Deleted stuck failed job %s, will recreate on next sync", job.Name) + } + // Return early - next sync will recreate the job + return nil + } + } + // Create an OperatorStatusApplyConfiguration with generations status := applyoperatorv1.OperatorStatus(). WithGenerations(&applyoperatorv1.GenerationStatusApplyConfiguration{ diff --git a/pkg/tnf/pkg/jobs/tnf.go b/pkg/tnf/pkg/jobs/tnf.go index f0fe57538d..e0f13c59b0 100644 --- a/pkg/tnf/pkg/jobs/tnf.go +++ b/pkg/tnf/pkg/jobs/tnf.go @@ -12,16 +12,33 @@ import ( "github.com/openshift/library-go/pkg/controller/factory" "github.com/openshift/library-go/pkg/operator/v1helpers" batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" + "k8s.io/utils/ptr" "github.com/openshift/cluster-etcd-operator/bindata" "github.com/openshift/cluster-etcd-operator/pkg/operator/operatorclient" "github.com/openshift/cluster-etcd-operator/pkg/tnf/pkg/tools" ) +// NodeTarget identifies a specific node for job scheduling and lifecycle management. +// When set, the job is tied to this node's identity (named with node suffix, labeled with UID for cleanup). +type NodeTarget struct { + Name string // Node name for scheduling and job naming + UID string // Node UID for job labeling (enables cleanup on node deletion/replacement) +} + +// TargetNodesFunc calculates the list of valid target nodes for a job. +// Called by job controller before each attempt to get fresh node state. +type TargetNodesFunc func() ([]*corev1.Node, error) + +// JobConfigFunc returns a serialized JSON string representing the job's configuration. +// Used to detect config drift - if the returned value changes, the job needs to be restarted. +type JobConfigFunc func() (string, error) + var ( // runningControllers tracks which controllers are already running to prevent duplicates runningControllers = make(map[string]bool) @@ -32,16 +49,317 @@ var ( restartJobLocks = make(map[string]*sync.Mutex) // restartJobLocksMutex protects the restartJobLocks map restartJobLocksMutex sync.Mutex + + // retryState tracks multi-node retry state for jobs using TargetNodesFunc + // Map key is job name, value tracks current attempt and node index + retryState = make(map[string]*JobRetryState) + // retryStateMutex protects access to retryState map + retryStateMutex sync.Mutex ) -func RunTNFJobController(ctx context.Context, jobType tools.JobType, nodeName *string, controllerContext *controllercmd.ControllerContext, operatorClient v1helpers.StaticPodOperatorClient, kubeClient kubernetes.Interface, kubeInformersForNamespaces v1helpers.KubeInformersForNamespaces, conditions []string) { - nodeNameForLogs := "undefined" - if nodeName != nil { - nodeNameForLogs = *nodeName +// JobRetryState tracks retry progress for multi-node jobs +type JobRetryState struct { + mu sync.Mutex // Protects fields below + AttemptNumber int // Current attempt (1-N) + NodeIndex int // Index of node to try in current attempt + TargetNodes []string // Cached node names from last targetNodesFunc call + MaxRetryAttempts int // Maximum attempts before degrading + LastFailTime time.Time // When last failure occurred + LastJobConfig string // Serialized config when job was created (for drift detection) +} + +// syncMultiNodeJobState manages the retry state for a multi-node job. +// This should be called before the job hook to ensure state is current. +// It handles: +// - Checking if target nodes changed (resets state) +// - Checking if job config changed (resets state) +// - Detecting failed jobs and incrementing to next node +// - Deleting failed jobs so they can be recreated on next node +// - Setting degraded condition when max retries exhausted +func syncMultiNodeJobState(ctx context.Context, jobName string, targetNodesFunc TargetNodesFunc, jobConfigFunc JobConfigFunc, maxRetryAttempts int, kubeClient kubernetes.Interface, operatorClient v1helpers.StaticPodOperatorClient) error { + // Lock the global state map + retryStateMutex.Lock() + state, exists := retryState[jobName] + if !exists { + // Initialize state for new job + targetNodes, err := targetNodesFunc() + if err != nil { + retryStateMutex.Unlock() + return fmt.Errorf("failed to get target nodes: %w", err) + } + if len(targetNodes) == 0 { + retryStateMutex.Unlock() + return fmt.Errorf("no target nodes available for job") + } + + // Get initial job config + var initialConfig string + if jobConfigFunc != nil { + initialConfig, err = jobConfigFunc() + if err != nil { + retryStateMutex.Unlock() + return fmt.Errorf("failed to get initial job config: %w", err) + } + } + + state = &JobRetryState{ + AttemptNumber: 1, + NodeIndex: 0, + TargetNodes: tools.GetNodeNames(targetNodes), + MaxRetryAttempts: maxRetryAttempts, + LastJobConfig: initialConfig, + } + retryState[jobName] = state + retryStateMutex.Unlock() + klog.Infof("Starting job %s - attempt %d/%d, will try nodes: %v", + jobName, state.AttemptNumber, state.MaxRetryAttempts, state.TargetNodes) + return nil + } + retryStateMutex.Unlock() + + // Lock this job's state for the rest of the sync + state.mu.Lock() + defer state.mu.Unlock() + + // Check if target nodes have changed + targetNodes, err := targetNodesFunc() + if err != nil { + return fmt.Errorf("failed to get target nodes: %w", err) + } + if len(targetNodes) == 0 { + return fmt.Errorf("no target nodes available for job") + } + + currentTargetNodes := tools.GetNodeNames(targetNodes) + nodesChanged := !slicesEqual(state.TargetNodes, currentTargetNodes) + + // Check if job config has changed + var configChanged bool + var currentConfig string + if jobConfigFunc != nil { + currentConfig, err = jobConfigFunc() + if err != nil { + return fmt.Errorf("failed to get current job config: %w", err) + } + configChanged = state.LastJobConfig != currentConfig + } + + // If nodes or config changed, restart job + if nodesChanged || configChanged { + if nodesChanged { + klog.Infof("Job %s target nodes changed from %v to %v - resetting retry state and deleting job", + jobName, state.TargetNodes, currentTargetNodes) + } + if configChanged { + klog.Infof("Job %s config changed - resetting retry state and deleting job. Old: %s, New: %s", + jobName, state.LastJobConfig, currentConfig) + } + + state.AttemptNumber = 1 + state.NodeIndex = 0 + state.TargetNodes = currentTargetNodes + state.LastJobConfig = currentConfig + + // Delete existing job if it exists (config drift or wrong target node) + _, err := kubeClient.BatchV1().Jobs(operatorclient.TargetNamespace).Get(ctx, jobName, v1.GetOptions{}) + if err == nil { + // Job exists - delete it so we can recreate with new config + if err := DeleteAndWait(ctx, kubeClient, jobName, operatorclient.TargetNamespace); err != nil { + return fmt.Errorf("failed to delete job %s after config/nodes changed: %w", jobName, err) + } + klog.Infof("Deleted job %s after config/nodes changed", jobName) + } else if !apierrors.IsNotFound(err) { + return fmt.Errorf("failed to check for existing job %s: %w", jobName, err) + } + return nil + } + + // Get existing job (if any) + existingJob, err := kubeClient.BatchV1().Jobs(operatorclient.TargetNamespace).Get(ctx, jobName, v1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + // No job exists - nothing to sync (will be created by JobController) + return nil + } + return fmt.Errorf("failed to get job %s: %w", jobName, err) + } + + // Job exists - check if it's done + if IsComplete(*existingJob) { + // Success - clear state and degraded condition + klog.Infof("Job %s completed successfully", jobName) + resetJobRetryState(jobName) + + // Clear degraded condition if it was set + _, _, err := v1helpers.UpdateStatus(ctx, operatorClient, v1helpers.UpdateConditionFn(operatorv1.OperatorCondition{ + Type: jobName + operatorv1.OperatorStatusTypeDegraded, + Status: operatorv1.ConditionFalse, + Reason: "AsExpected", + Message: fmt.Sprintf("Job %s completed successfully", jobName), + })) + if err != nil { + klog.Errorf("Failed to clear degraded condition for %s: %v", jobName, err) + } + + return nil + } + + if IsFailed(*existingJob) || IsStopped(*existingJob) { + // Failed - move to next node + currentNodeIndex := state.NodeIndex + klog.Infof("Job %s failed on node index %d - moving to next node", jobName, currentNodeIndex) + + // Increment to next node + state.NodeIndex++ + + // Check if we've exhausted all nodes in this attempt + if state.NodeIndex >= len(targetNodes) { + if state.AttemptNumber >= state.MaxRetryAttempts { + // Exceeded max attempts - set degraded condition and reset to attempt 1 + klog.Warningf("Job %s exhausted all %d attempts (tried %d nodes each), marking degraded", + jobName, state.MaxRetryAttempts, len(targetNodes)) + + // Set degraded condition to indicate job has failed after all retries + _, _, err := v1helpers.UpdateStatus(ctx, operatorClient, v1helpers.UpdateConditionFn(operatorv1.OperatorCondition{ + Type: jobName + operatorv1.OperatorStatusTypeDegraded, + Status: operatorv1.ConditionTrue, + Reason: "MaxRetriesExceeded", + Message: fmt.Sprintf("Job failed after %d attempts across all nodes", state.MaxRetryAttempts), + })) + if err != nil { + klog.Errorf("Failed to set degraded condition for %s: %v", jobName, err) + } + + // Reset to attempt 1 and continue trying (degraded condition remains set until success) + state.AttemptNumber = 1 + state.NodeIndex = 0 + } else { + // Start new attempt + state.AttemptNumber++ + state.NodeIndex = 0 + klog.Infof("Job %s exhausted all nodes in attempt %d, starting attempt %d/%d", + jobName, state.AttemptNumber-1, state.AttemptNumber, state.MaxRetryAttempts) + } + } + + // Delete the failed job so it can be recreated on next node + klog.Infof("Deleting failed job %s to retry on node index %d", jobName, state.NodeIndex) + if err := DeleteAndWait(ctx, kubeClient, jobName, operatorclient.TargetNamespace); err != nil { + return fmt.Errorf("failed to delete failed job: %w", err) + } + } + + // Job is running - nothing to do + return nil +} + +// configureMultiNodeJob configures a job based on current retry state. +// This is a pure function that just reads state and configures the job. +// State management is done by syncMultiNodeJobState. +func configureMultiNodeJob(ctx context.Context, job *batchv1.Job, targetNodesFunc TargetNodesFunc, jobConfigFunc JobConfigFunc, maxRetryAttempts int, kubeClient kubernetes.Interface, operatorClient v1helpers.StaticPodOperatorClient) error { + jobName := job.Name + + // Get current state (should have been initialized by syncMultiNodeJobState) + retryStateMutex.Lock() + state, exists := retryState[jobName] + retryStateMutex.Unlock() + + if !exists { + // State should have been created by syncMultiNodeJobState, but handle gracefully + if err := syncMultiNodeJobState(ctx, jobName, targetNodesFunc, jobConfigFunc, maxRetryAttempts, kubeClient, operatorClient); err != nil { + return err + } + retryStateMutex.Lock() + state = retryState[jobName] + retryStateMutex.Unlock() + } + + // Get current target nodes + targetNodes, err := targetNodesFunc() + if err != nil { + return fmt.Errorf("failed to get target nodes: %w", err) + } + if len(targetNodes) == 0 { + return fmt.Errorf("no target nodes available for job") + } + + // Lock state for reading + state.mu.Lock() + nodeIndex := state.NodeIndex + attemptNumber := state.AttemptNumber + state.mu.Unlock() + + // Validate node index + if nodeIndex >= len(targetNodes) { + return fmt.Errorf("invalid node index %d (only %d nodes available)", nodeIndex, len(targetNodes)) } - // Check if a controller for this jobType and nodeName is already running - controllerKey := jobType.GetJobName(nodeName) + selectedNode := targetNodes[nodeIndex] + klog.V(4).Infof("Job %s attempt %d/%d: scheduling on node %s (index %d/%d)", + jobName, attemptNumber, maxRetryAttempts, selectedNode.Name, + nodeIndex+1, len(targetNodes)) + + // Configure job to run on selected node + job.Spec.Template.Spec.NodeName = selectedNode.Name + job.Labels["tnf.etcd.openshift.io/attempt"] = fmt.Sprintf("%d", attemptNumber) + job.Labels["tnf.etcd.openshift.io/node-index"] = fmt.Sprintf("%d", nodeIndex) + + return nil +} + +// slicesEqual checks if two string slices have the same elements in the same order +func slicesEqual(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +// resetJobRetryState clears the retry state for a job (called on success or when starting fresh) +func resetJobRetryState(jobName string) { + retryStateMutex.Lock() + defer retryStateMutex.Unlock() + delete(retryState, jobName) + klog.V(2).Infof("Reset retry state for job %s", jobName) +} + +// IsControllerRunning checks if a controller is already running for the given job name +func IsControllerRunning(jobName string) bool { + runningControllersMutex.Lock() + defer runningControllersMutex.Unlock() + return runningControllers[jobName] +} + +// RunTNFJobController starts a job controller for the specified job type. +// +// Parameters: +// - nodeTarget: If non-nil, ties the job to this specific node (job name includes node suffix, +// job is labeled with node UID for cleanup, and pod is scheduled on this node). +// Use for node-specific jobs like auth and after-setup. +// - targetNodesFunc: Optional function that returns list of target nodes to try (for multi-node retry logic). +// Job controller will try nodes serially until one succeeds, calling this function before each attempt. +// Only used when nodeTarget is nil. Use for cluster-wide jobs like update-setup. +// - jobConfigFunc: Optional function that returns job configuration as JSON string (for config drift detection). +// If config changes, job is deleted and recreated with new config. +// - retries: Number of retries before marking degraded. Meaning depends on job type: +// - Single-node (nodeTarget): sets backoffLimit (Kubernetes retries on same node) +// - Multi-node (targetNodesFunc): sets maxRetryAttempts (loop across different nodes, backoffLimit=0) +// - Pass 0 for no retries +func RunTNFJobController(ctx context.Context, jobType tools.JobType, nodeTarget *NodeTarget, targetNodesFunc TargetNodesFunc, jobConfigFunc JobConfigFunc, retries int, controllerContext *controllercmd.ControllerContext, operatorClient v1helpers.StaticPodOperatorClient, kubeClient kubernetes.Interface, kubeInformersForNamespaces v1helpers.KubeInformersForNamespaces, conditions []string) { + nodeNameForLogs := "any" + var jobNodeName *string + if nodeTarget != nil { + nodeNameForLogs = nodeTarget.Name + jobNodeName = &nodeTarget.Name + } + + // Check if a controller for this jobType and node is already running + controllerKey := jobType.GetJobName(jobNodeName) runningControllersMutex.Lock() if runningControllers[controllerKey] { runningControllersMutex.Unlock() @@ -54,7 +372,7 @@ func RunTNFJobController(ctx context.Context, jobType tools.JobType, nodeName *s klog.Infof("starting Two Node Fencing job controller for command %q on node %q", jobType.GetSubCommand(), nodeNameForLogs) tnfJobController := NewJobController( - jobType.GetJobName(nodeName), + jobType.GetJobName(jobNodeName), bindata.MustAsset("tnfdeployment/job.yaml"), controllerContext.EventRecorder, operatorClient, @@ -64,11 +382,35 @@ func RunTNFJobController(ctx context.Context, jobType tools.JobType, nodeName *s []factory.Informer{}, []JobHookFunc{ func(_ *operatorv1.OperatorSpec, job *batchv1.Job) error { - if nodeName != nil { - job.Spec.Template.Spec.NodeName = *nodeName - } - job.SetName(jobType.GetJobName(nodeName)) + // Set job name first so it's available for logging in configuration functions + job.SetName(jobType.GetJobName(jobNodeName)) job.Labels["app.kubernetes.io/name"] = jobType.GetNameLabelValue() + + // Configure job based on node target and retry strategy + if nodeTarget != nil { + // Single-node job: schedule on node, label with UID, set backoffLimit for retries + job.Spec.Template.Spec.NodeName = nodeTarget.Name + job.Labels["node"] = nodeTarget.UID + job.Spec.BackoffLimit = ptr.To(int32(retries)) + } else if targetNodesFunc != nil { + // Multi-node job: sync state first (handles transitions), then configure + // syncMultiNodeJobState manages state transitions based on job status + if err := syncMultiNodeJobState(ctx, job.Name, targetNodesFunc, jobConfigFunc, retries, kubeClient, operatorClient); err != nil { + return err + } + // Now configure job based on current state (pure function) + job.Spec.BackoffLimit = ptr.To(int32(0)) + if err := configureMultiNodeJob(ctx, job, targetNodesFunc, jobConfigFunc, retries, kubeClient, operatorClient); err != nil { + return err + } + } else { + // No specific targeting - use default backoffLimit from yaml + if retries > 0 { + job.Spec.BackoffLimit = ptr.To(int32(retries)) + } + } + + // Set image and command job.Spec.Template.Spec.Containers[0].Image = os.Getenv("OPERATOR_IMAGE") job.Spec.Template.Spec.Containers[0].Command[1] = jobType.GetSubCommand() return nil @@ -85,10 +427,28 @@ func RunTNFJobController(ctx context.Context, jobType tools.JobType, nodeName *s }() } +// RestartJobOrRunController ensures a job controller is running, restarting the job if it already exists. +// +// Parameters: +// - nodeTarget: If non-nil, ties the job to this specific node (job name includes node suffix, +// job is labeled with node UID for cleanup, and pod is scheduled on this node). +// Use for node-specific jobs like auth and after-setup. +// - targetNodesFunc: Optional function that returns list of target nodes to try (for multi-node retry logic). +// Job controller will try nodes serially until one succeeds, calling this function before each attempt. +// Only used when nodeTarget is nil. Use for cluster-wide jobs like update-setup. +// - jobConfigFunc: Optional function that returns job configuration as JSON string (for config drift detection). +// If config changes, job is deleted and recreated with new config. +// - retries: Number of retries before marking degraded. Meaning depends on job type: +// - Single-node (nodeTarget): sets backoffLimit (Kubernetes retries on same node) +// - Multi-node (targetNodesFunc): sets maxRetryAttempts (loop across different nodes, backoffLimit=0) +// - Pass 0 for no retries func RestartJobOrRunController( ctx context.Context, jobType tools.JobType, - nodeName *string, + nodeTarget *NodeTarget, + targetNodesFunc TargetNodesFunc, + jobConfigFunc JobConfigFunc, + retries int, controllerContext *controllercmd.ControllerContext, operatorClient v1helpers.StaticPodOperatorClient, kubeClient kubernetes.Interface, @@ -96,9 +456,14 @@ func RestartJobOrRunController( conditions []string, existingJobCompletionTimeout time.Duration) error { - // Acquire a lock for this specific jobType/nodeName combination to prevent parallel execution - jobName := jobType.GetJobName(nodeName) + // Determine job name based on node target + var jobNodeName *string + if nodeTarget != nil { + jobNodeName = &nodeTarget.Name + } + jobName := jobType.GetJobName(jobNodeName) + // Acquire a lock for this specific job to prevent parallel execution restartJobLocksMutex.Lock() jobLock, exists := restartJobLocks[jobName] if !exists { @@ -121,10 +486,11 @@ func RestartJobOrRunController( } // always try to run the controller, CEO might have been restarted - RunTNFJobController(ctx, jobType, nodeName, controllerContext, operatorClient, kubeClient, kubeInformersForNamespaces, conditions) + RunTNFJobController(ctx, jobType, nodeTarget, targetNodesFunc, jobConfigFunc, retries, controllerContext, operatorClient, kubeClient, kubeInformersForNamespaces, conditions) if !jobExists { - // we are done + // No existing job - reset retry state to start fresh + resetJobRetryState(jobName) return nil } @@ -140,5 +506,8 @@ func RestartJobOrRunController( return fmt.Errorf("failed to delete existing update-setup job %s: %w", jobName, err) } + // Reset retry state when starting fresh after deleting old job + resetJobRetryState(jobName) + return nil } diff --git a/pkg/tnf/pkg/jobs/tnf_test.go b/pkg/tnf/pkg/jobs/tnf_test.go index 955532fdb9..07b050e3e9 100644 --- a/pkg/tnf/pkg/jobs/tnf_test.go +++ b/pkg/tnf/pkg/jobs/tnf_test.go @@ -1,5 +1,32 @@ package jobs +/* +TEST COVERAGE SUMMARY - tnf_test.go +==================================== + +This file tests TNF job controller management and restart logic. + +WHAT'S TESTED +------------- + +Job Controller Lifecycle: +├── TestRunTNFJobController - Controller startup and deduplication +│ ├── ✅ Start controller for cluster-wide job (setup, fencing, update-setup) +│ ├── ✅ Start controller for node-specific job (auth, after-setup) +│ ├── ✅ Skip starting controller when already running +│ ├── ✅ Start different controller when another is running +│ ├── ✅ Start controller for different node when same job type exists +│ └── ✅ Cluster-wide job with scheduling hint (scheduleOnNode) +├── TestRestartJobOrRunController - Job restart and cleanup logic +│ ├── ✅ Job does not exist - just runs controller +│ ├── ✅ Job exists and stops successfully - deletes and runs controller +│ ├── ✅ Job exists but Get returns error - returns error +│ ├── ✅ Job exists but won't stop (timeout) - returns error +│ └── ✅ Job exists, stops, but delete fails - returns error +└── TestRestartJobOrRunController_ParallelExecution - Concurrency safety + └── ✅ Parallel calls serialize via mutex (only one delete) +*/ + import ( "context" "maps" @@ -31,31 +58,39 @@ func TestRunTNFJobController(t *testing.T) { tests := []struct { name string jobType tools.JobType - nodeName *string + nodeTarget *NodeTarget + targetNodesFunc TargetNodesFunc + retries int existingControllers map[string]bool expectControllerRun bool expectedControllerKey string }{ { - name: "Start controller for auth job without node name", - jobType: tools.JobTypeAuth, - nodeName: nil, + name: "Start controller for cluster-wide job", + jobType: tools.JobTypeSetup, + nodeTarget: nil, + targetNodesFunc: nil, + retries: 3, existingControllers: make(map[string]bool), expectControllerRun: true, - expectedControllerKey: "tnf-auth-job", + expectedControllerKey: "tnf-setup-job", }, { - name: "Start controller for auth job with node name", + name: "Start controller for node-specific job", jobType: tools.JobTypeAuth, - nodeName: stringPtr("master-0"), + nodeTarget: &NodeTarget{Name: "master-0", UID: "uid-master-0"}, + targetNodesFunc: nil, + retries: 3, existingControllers: make(map[string]bool), expectControllerRun: true, expectedControllerKey: tools.JobTypeAuth.GetJobName(stringPtr("master-0")), }, { - name: "Skip starting controller when already running", - jobType: tools.JobTypeSetup, - nodeName: nil, + name: "Skip starting controller when already running", + jobType: tools.JobTypeSetup, + nodeTarget: nil, + targetNodesFunc: nil, + retries: 3, existingControllers: map[string]bool{ "tnf-setup-job": true, }, @@ -63,9 +98,11 @@ func TestRunTNFJobController(t *testing.T) { expectedControllerKey: "tnf-setup-job", }, { - name: "Start different controller when another is running", - jobType: tools.JobTypeFencing, - nodeName: nil, + name: "Start different controller when another is running", + jobType: tools.JobTypeFencing, + nodeTarget: nil, + targetNodesFunc: nil, + retries: 3, existingControllers: map[string]bool{ "tnf-setup-job": true, }, @@ -73,15 +110,27 @@ func TestRunTNFJobController(t *testing.T) { expectedControllerKey: "tnf-fencing-job", }, { - name: "Start controller for different node when same job type exists", - jobType: tools.JobTypeAuth, - nodeName: stringPtr("master-1"), + name: "Start controller for different node when same job type exists", + jobType: tools.JobTypeAuth, + nodeTarget: &NodeTarget{Name: "master-1", UID: "uid-master-1"}, + targetNodesFunc: nil, + retries: 3, existingControllers: map[string]bool{ tools.JobTypeAuth.GetJobName(stringPtr("master-0")): true, }, expectControllerRun: true, expectedControllerKey: tools.JobTypeAuth.GetJobName(stringPtr("master-1")), }, + { + name: "Cluster-wide job with validNodeFunc", + jobType: tools.JobTypeUpdateSetup, + nodeTarget: nil, + targetNodesFunc: func() ([]*corev1.Node, error) { return []*corev1.Node{}, nil }, + retries: 3, + existingControllers: make(map[string]bool), + expectControllerRun: true, + expectedControllerKey: "tnf-update-setup-job", + }, } for _, tt := range tests { @@ -123,7 +172,10 @@ func TestRunTNFJobController(t *testing.T) { RunTNFJobController( ctx, tt.jobType, - tt.nodeName, + tt.nodeTarget, + tt.targetNodesFunc, + nil, // jobConfigFunc + tt.retries, controllerContext, fakeOperatorClient, fakeKubeClient, @@ -153,31 +205,39 @@ func TestRunTNFJobController(t *testing.T) { func TestRestartJobOrRunController(t *testing.T) { tests := []struct { - name string - jobType tools.JobType - nodeName *string - setupClient func() *fake.Clientset - expectError bool - errorContains string - expectJobDeleted bool - expectWaitForStopped bool + name string + jobType tools.JobType + nodeTarget *NodeTarget + targetNodesFunc TargetNodesFunc + retries int + setupClient func() *fake.Clientset + expectError bool + errorContains string + expectJobDeleted bool + expectWaitForStopped bool + expectControllerStarted bool }{ { - name: "Job does not exist - just runs controller", - jobType: tools.JobTypeAuth, - nodeName: stringPtr("master-0"), + name: "Job does not exist - just runs controller", + jobType: tools.JobTypeAuth, + nodeTarget: &NodeTarget{Name: "master-0", UID: "uid-master-0"}, + targetNodesFunc: nil, + retries: 3, setupClient: func() *fake.Clientset { // No job exists return fake.NewSimpleClientset() }, - expectError: false, - expectJobDeleted: false, - expectWaitForStopped: false, + expectError: false, + expectJobDeleted: false, + expectWaitForStopped: false, + expectControllerStarted: true, }, { - name: "Job exists and stops successfully - deletes and runs controller", - jobType: tools.JobTypeSetup, - nodeName: nil, + name: "Job exists and stops successfully - deletes and runs controller", + jobType: tools.JobTypeSetup, + nodeTarget: nil, + targetNodesFunc: nil, + retries: 3, setupClient: func() *fake.Clientset { job := &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ @@ -211,14 +271,17 @@ func TestRestartJobOrRunController(t *testing.T) { return client }, - expectError: false, - expectJobDeleted: true, - expectWaitForStopped: true, + expectError: false, + expectJobDeleted: true, + expectWaitForStopped: true, + expectControllerStarted: true, }, { - name: "Job exists but Get returns error - returns error", - jobType: tools.JobTypeAuth, - nodeName: stringPtr("master-0"), + name: "Job exists but Get returns error - returns error", + jobType: tools.JobTypeAuth, + nodeTarget: &NodeTarget{Name: "master-0", UID: "uid-master-0"}, + targetNodesFunc: nil, + retries: 3, setupClient: func() *fake.Clientset { client := fake.NewSimpleClientset() client.PrependReactor("get", "jobs", func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) { @@ -226,15 +289,18 @@ func TestRestartJobOrRunController(t *testing.T) { }) return client }, - expectError: true, - errorContains: "failed to check for existing job", - expectJobDeleted: false, - expectWaitForStopped: false, + expectError: true, + errorContains: "failed to check for existing job", + expectJobDeleted: false, + expectWaitForStopped: false, + expectControllerStarted: false, // Early error, controller not started }, { - name: "Job exists but WaitForStopped times out - returns error", - jobType: tools.JobTypeFencing, - nodeName: nil, + name: "Job exists but WaitForStopped times out - returns error", + jobType: tools.JobTypeFencing, + nodeTarget: nil, + targetNodesFunc: nil, + retries: 3, setupClient: func() *fake.Clientset { job := &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ @@ -248,15 +314,18 @@ func TestRestartJobOrRunController(t *testing.T) { } return fake.NewSimpleClientset(job) }, - expectError: true, - errorContains: "failed to wait for update-setup job", - expectJobDeleted: false, - expectWaitForStopped: true, + expectError: true, + errorContains: "failed to wait for update-setup job", + expectJobDeleted: false, + expectWaitForStopped: true, + expectControllerStarted: true, // Controller started before WaitForStopped failed }, { - name: "Job exists, stops, but delete fails - returns error", - jobType: tools.JobTypeAfterSetup, - nodeName: stringPtr("master-1"), + name: "Job exists, stops, but delete fails - returns error", + jobType: tools.JobTypeAfterSetup, + nodeTarget: &NodeTarget{Name: "master-1", UID: "uid-master-1"}, + targetNodesFunc: nil, + retries: 3, setupClient: func() *fake.Clientset { jobName := tools.JobTypeAfterSetup.GetJobName(stringPtr("master-1")) job := &batchv1.Job{ @@ -283,10 +352,11 @@ func TestRestartJobOrRunController(t *testing.T) { return client }, - expectError: true, - errorContains: "failed to delete existing update-setup job", - expectJobDeleted: false, - expectWaitForStopped: true, + expectError: true, + errorContains: "failed to delete existing update-setup job", + expectJobDeleted: false, + expectWaitForStopped: true, + expectControllerStarted: true, // Controller started before Delete failed }, } @@ -331,13 +401,16 @@ func TestRestartJobOrRunController(t *testing.T) { err := RestartJobOrRunController( ctx, tt.jobType, - tt.nodeName, + tt.nodeTarget, + tt.targetNodesFunc, + nil, // jobConfigFunc + tt.retries, controllerContext, fakeOperatorClient, client, kubeInformersForNamespaces, DefaultConditions, - 1*time.Second, // Short timeout for tests + 1*time.Second, // timeout ) // Verify @@ -351,15 +424,22 @@ func TestRestartJobOrRunController(t *testing.T) { require.NoError(t, err, "Expected no error but got: %v", err) } - // Verify controller was started (only if no early error) - jobName := tt.jobType.GetJobName(tt.nodeName) + // Determine job name based on node target + var jobNodeName *string + if tt.nodeTarget != nil { + jobNodeName = &tt.nodeTarget.Name + } + jobName := tt.jobType.GetJobName(jobNodeName) + + // Verify controller started based on explicit expectation + runningControllersMutex.Lock() + isRunning := runningControllers[jobName] + runningControllersMutex.Unlock() - // Only verify controller started if we didn't get an error before RunTNFJobController - if !tt.expectError || tt.expectWaitForStopped { - runningControllersMutex.Lock() - isRunning := runningControllers[jobName] - runningControllersMutex.Unlock() + if tt.expectControllerStarted { require.True(t, isRunning, "Expected controller to be started") + } else { + require.False(t, isRunning, "Expected controller NOT to be started") } // Verify lock was created (always created, even on error) @@ -457,13 +537,16 @@ func TestRestartJobOrRunController_ParallelExecution(t *testing.T) { errors[idx] = RestartJobOrRunController( context.Background(), tools.JobTypeSetup, - nil, + nil, // nodeTarget + nil, // targetNodesFunc + nil, // jobConfigFunc + 3, // retries controllerContext, fakeOperatorClient, client, kubeInformersForNamespaces, DefaultConditions, - 2*time.Second, + 2*time.Second, // timeout ) }(i) } diff --git a/pkg/tnf/pkg/jobs/utils.go b/pkg/tnf/pkg/jobs/utils.go index a1869e6d86..0cfd119683 100644 --- a/pkg/tnf/pkg/jobs/utils.go +++ b/pkg/tnf/pkg/jobs/utils.go @@ -60,7 +60,8 @@ func waitWithConditionFunc(ctx context.Context, kubeClient kubernetes.Interface, }) } -// DeleteAndWait deletes a job and waits until it disappears from the API +// DeleteAndWait deletes a job and waits until it disappears from the API. +// Uses Background propagation to automatically delete child pods. func DeleteAndWait(ctx context.Context, kubeClient kubernetes.Interface, jobName string, jobNamespace string) error { klog.V(4).Infof("deleting oldJob %s", jobName) oldJob, err := kubeClient.BatchV1().Jobs(jobNamespace).Get(ctx, jobName, v1.GetOptions{}) @@ -73,7 +74,11 @@ func DeleteAndWait(ctx context.Context, kubeClient kubernetes.Interface, jobName } oldJobUID := oldJob.GetUID() - err = kubeClient.BatchV1().Jobs(jobNamespace).Delete(ctx, jobName, v1.DeleteOptions{}) + // Delete job with Background propagation to automatically clean up pods + propagationPolicy := v1.DeletePropagationBackground + err = kubeClient.BatchV1().Jobs(jobNamespace).Delete(ctx, jobName, v1.DeleteOptions{ + PropagationPolicy: &propagationPolicy, + }) if err != nil { if apierrors.IsNotFound(err) { return nil @@ -98,7 +103,34 @@ func DeleteAndWait(ctx context.Context, kubeClient kubernetes.Interface, jobName } func IsStopped(job batchv1.Job) bool { - return IsComplete(job) || IsFailed(job) + // Check for explicit conditions + if IsComplete(job) || IsFailed(job) { + return true + } + + // Check for FailureTarget condition (Kubernetes 1.31+) + // FailureTarget means job is targeting failure but pods may still be terminating + if IsConditionTrue(job.Status.Conditions, batchv1.JobConditionType("FailureTarget")) { + klog.V(2).Infof("Job %s considered stopped: FailureTarget condition is True", job.Name) + return true + } + + // Also check if job has no active pods and has exceeded backoff limit + // This handles cases where pods are stuck (e.g., Terminating on dead node) and + // Kubernetes hasn't set the Failed condition yet + if job.Status.Active == 0 && job.Status.Failed > 0 { + backoffLimit := int32(6) // Kubernetes default + if job.Spec.BackoffLimit != nil { + backoffLimit = *job.Spec.BackoffLimit + } + if job.Status.Failed > backoffLimit { + klog.V(2).Infof("Job %s considered stopped: Active=0, Failed=%d > BackoffLimit=%d (no explicit condition set)", + job.Name, job.Status.Failed, backoffLimit) + return true + } + } + + return false } func IsComplete(job batchv1.Job) bool { diff --git a/pkg/tnf/pkg/tools/nodes.go b/pkg/tnf/pkg/tools/nodes.go index e1603ff8d0..4499a903ad 100644 --- a/pkg/tnf/pkg/tools/nodes.go +++ b/pkg/tnf/pkg/tools/nodes.go @@ -41,3 +41,61 @@ func GetNodeIPForPacemaker(node corev1.Node) (string, error) { // fallback return addresses[0].Address, nil } + +// GetNodeNames extracts node names from a slice of nodes. +func GetNodeNames(nodes []*corev1.Node) []string { + names := make([]string, len(nodes)) + for i, node := range nodes { + names[i] = node.Name + } + return names +} + +// ipAddressesEqual compares two IP address strings for equality. +// Handles both IPv4 and IPv6, normalizing them before comparison. +func ipAddressesEqual(ip1, ip2 string) bool { + parsedIP1 := net.ParseIP(ip1) + parsedIP2 := net.ParseIP(ip2) + + // If either fails to parse, fall back to string comparison + if parsedIP1 == nil || parsedIP2 == nil { + return ip1 == ip2 + } + + return parsedIP1.Equal(parsedIP2) +} + +// DetermineReconciliationActions compares desired and current node membership to determine +// which nodes to add/remove. Desired state is the source of truth (K8s nodes). +// Compares both name AND IP to detect replacements (same name, different IP). +// +// Parameters: +// - desiredNodes: map of node names to IPs that should be in the cluster (from K8s API or ConfigMap) +// - currentNodes: map of node names to IPs currently in the cluster (from PacemakerCluster CR or pacemaker directly) +// +// Returns: +// - nodesToRemove: nodes in current but not in desired, or with IP mismatch +// - nodesToAdd: nodes in desired but not in current, or with IP mismatch +func DetermineReconciliationActions(desiredNodes, currentNodes map[string]string) (nodesToRemove, nodesToAdd []string) { + // Remove: current nodes not in desired OR with mismatched IPs + for nodeName, currentIP := range currentNodes { + desiredIP, existsInDesired := desiredNodes[nodeName] + if !existsInDesired { + // Node deleted from K8s + nodesToRemove = append(nodesToRemove, nodeName) + } else if !ipAddressesEqual(desiredIP, currentIP) { + // Node exists in both but IP changed - remove old, add new + nodesToRemove = append(nodesToRemove, nodeName) + nodesToAdd = append(nodesToAdd, nodeName) + } + } + + // Add: desired nodes not in current at all + for nodeName := range desiredNodes { + if _, exists := currentNodes[nodeName]; !exists { + nodesToAdd = append(nodesToAdd, nodeName) + } + } + + return nodesToRemove, nodesToAdd +} diff --git a/pkg/tnf/pkg/tools/tnf_labels.go b/pkg/tnf/pkg/tools/tnf_labels.go new file mode 100644 index 0000000000..f6029b64fc --- /dev/null +++ b/pkg/tnf/pkg/tools/tnf_labels.go @@ -0,0 +1,5 @@ +package tools + +// TnfUpdateSetupComponentValue is app.kubernetes.io/component for update-setup snapshot ConfigMaps. +// CEO creates ConfigMaps with this value; the in-cluster update-setup job lists by the same selector. +const TnfUpdateSetupComponentValue = "tnf-update-setup" From d13ff5c71ed20b4a8abb8b561687e2cbdee65dab Mon Sep 17 00:00:00 2001 From: Jeremy Poulin Date: Thu, 25 Jun 2026 15:56:03 -0400 Subject: [PATCH 2/6] OCPBUGS-84695: feat(tnf): bootstrap job controllers and startup logic Implements job controller startup with dual-mode behavior (bootstrap vs runtime) and individual job implementations: - StartJobControllers() with bootstrap vs runtime separation - Bootstrap mode: exponential backoff retry, requires exactly 2 ready nodes, sets degraded condition - Runtime mode: idempotent restart support, handles 1-2 nodes, fast path when setup job complete - startTnfJobcontrollers(): waits for etcd bootstrap, stable revision, creates per-node and cluster-wide controllers - Auth job implementation: sets hacluster password (required before Pacemaker operations) - After-setup job implementation: * Pre-transition: waits for setup job completion * Post-transition: waits for update-setup job containing this node * Disables kubelet service (Pacemaker owns it now) * Prevents kubelet disable race during Day 2 node addition - Update-setup controller check: on operator restart, starts controller for stopped update-setup job See docs/tnf/job-controllers.md for dual-mode operation and bootstrap sequence. Co-Authored-By: Claude Sonnet 4.5 --- pkg/tnf/after-setup/runner.go | 166 ++++++- pkg/tnf/auth/runner.go | 3 +- pkg/tnf/operator/lifecycle_job_controllers.go | 334 ++++++++++++++ .../lifecycle_job_controllers_test.go | 417 ++++++++++++++++++ pkg/tnf/pkg/pcs/auth.go | 6 +- pkg/tnf/pkg/tools/jobs.go | 13 +- 6 files changed, 916 insertions(+), 23 deletions(-) create mode 100644 pkg/tnf/operator/lifecycle_job_controllers.go create mode 100644 pkg/tnf/operator/lifecycle_job_controllers_test.go diff --git a/pkg/tnf/after-setup/runner.go b/pkg/tnf/after-setup/runner.go index 7e4722d005..b85d69a6be 100644 --- a/pkg/tnf/after-setup/runner.go +++ b/pkg/tnf/after-setup/runner.go @@ -2,19 +2,28 @@ package after_setup import ( "context" + "encoding/json" "fmt" + "os" + "strconv" + operatorv1 "github.com/openshift/api/operator/v1" batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/server" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/klog/v2" + "k8s.io/utils/clock" + "github.com/openshift/cluster-etcd-operator/pkg/operator" + "github.com/openshift/cluster-etcd-operator/pkg/operator/ceohelpers" "github.com/openshift/cluster-etcd-operator/pkg/tnf/pkg/jobs" "github.com/openshift/cluster-etcd-operator/pkg/tnf/pkg/kubelet" "github.com/openshift/cluster-etcd-operator/pkg/tnf/pkg/tools" + "github.com/openshift/library-go/pkg/operator/genericoperatorclient" ) func RunTnfAfterSetup() error { @@ -46,41 +55,170 @@ func RunTnfAfterSetup() error { klog.Info("Running TNF after setup") - klog.Info("Waiting for completed setup job") + // Get current node name from environment + currentNodeName := os.Getenv("MY_NODE_NAME") + if currentNodeName == "" { + return fmt.Errorf("MY_NODE_NAME environment variable not set") + } + klog.Infof("After-setup for node: %s", currentNodeName) + + // Check if external etcd transition is complete + operatorClient, dynamicInformers, err := genericoperatorclient.NewStaticPodOperatorClient(clock.RealClock{}, clientConfig, operatorv1.GroupVersion.WithResource("etcds"), operatorv1.GroupVersion.WithKind("Etcd"), operator.ExtractStaticPodOperatorSpec, operator.ExtractStaticPodOperatorStatus) + if err != nil { + return err + } + dynamicInformers.Start(ctx.Done()) + dynamicInformers.WaitForCacheSync(ctx.Done()) + + transitionComplete, err := ceohelpers.HasExternalEtcdCompletedTransition(ctx, operatorClient) + if err != nil { + return fmt.Errorf("failed to check external etcd transition status: %w", err) + } + + if !transitionComplete { + // Pre-transition (bootstrap): wait for setup job + klog.Info("Pre-transition: waiting for setup job to complete") + err = waitForSetupJobCompletion(ctx, kubeClient) + if err != nil { + return err + } + } else { + // Post-transition (Day 2): wait for update-setup job that includes this node + klog.Infof("Post-transition: waiting for update-setup job containing node %s to complete", currentNodeName) + err = waitForUpdateSetupJobCompletion(ctx, kubeClient, currentNodeName) + if err != nil { + return err + } + } + + // disable kubelet service, it's managed by pacemaker now + err = kubelet.Disable(ctx) + if err != nil { + klog.Errorf("Failed to disable kubelet service: %v", err) + return err + } + + klog.Info("TNF after setup done") + + return nil +} + +// waitForSetupJobCompletion waits for the setup job to complete (bootstrap phase) +func waitForSetupJobCompletion(ctx context.Context, kubeClient kubernetes.Interface) error { + klog.Info("Waiting for setup job to complete") setupDone := func(context.Context) (done bool, err error) { setupJobs, err := kubeClient.BatchV1().Jobs("openshift-etcd").List(ctx, metav1.ListOptions{ LabelSelector: fmt.Sprintf("app.kubernetes.io/name=%s", tools.JobTypeSetup.GetNameLabelValue()), }) if err != nil { - klog.Warningf("Failed to list setupJobs: %v", err) + klog.Warningf("Failed to list setup jobs: %v", err) return false, nil } if setupJobs.Items == nil || len(setupJobs.Items) != 1 { - klog.Warningf("Expected 1 job, got %d", len(setupJobs.Items)) + klog.V(4).Infof("Expected 1 setup job, got %d", len(setupJobs.Items)) return false, nil } job := setupJobs.Items[0] if !jobs.IsConditionTrue(job.Status.Conditions, batchv1.JobComplete) { - klog.Warningf("Job %s not complete", job.Name) + klog.V(4).Infof("Setup job %s not yet complete", job.Name) return false, nil } klog.Info("Setup job completed successfully") return true, nil } - err = wait.PollUntilContextTimeout(ctx, tools.JobPollInterval, tools.SetupJobCompletedTimeout, true, setupDone) + err := wait.PollUntilContextTimeout(ctx, tools.JobPollInterval, tools.SetupJobCompletedTimeout, true, setupDone) if err != nil { - klog.Errorf("Timed out waiting for setup job to complete: %v", err) - return err + return fmt.Errorf("timed out waiting for setup job to complete: %w", err) } + return nil +} - // disable kubelet service, it's managed by pacemaker now - err = kubelet.Disable(ctx) - if err != nil { - klog.Errorf("Failed to disable kubelet service: %v", err) - return err - } +// nodeInfo mirrors the structure used in lifecycle_reconciliation.go for ConfigMap node lists +type nodeInfo struct { + Name string `json:"name"` + IP string `json:"ip"` +} - klog.Info("TNF after setup done") +// waitForUpdateSetupJobCompletion waits for an update-setup job that includes the current node (Day 2 phase) +func waitForUpdateSetupJobCompletion(ctx context.Context, kubeClient kubernetes.Interface, nodeName string) error { + klog.Infof("Waiting for update-setup job containing node %s to complete", nodeName) + + updateSetupDone := func(context.Context) (done bool, err error) { + // Get all update-setup ConfigMaps + configMaps, err := kubeClient.CoreV1().ConfigMaps("openshift-etcd").List(ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("app.kubernetes.io/name=%s", tools.JobTypeUpdateSetup.GetNameLabelValue()), + }) + if err != nil { + klog.Warningf("Failed to list update-setup ConfigMaps: %v", err) + return false, nil + } + + // Find the latest generation ConfigMap that contains this node + var latestGeneration int64 = -1 + var targetConfigMap *corev1.ConfigMap + for i := range configMaps.Items { + cm := &configMaps.Items[i] + genStr := cm.Data["generation"] + gen, err := strconv.ParseInt(genStr, 10, 64) + if err != nil { + klog.Warningf("Failed to parse generation from ConfigMap %s: %v", cm.Name, err) + continue + } + + // Decode node list from ConfigMap + nodesJSON := cm.Data["nodes"] + var nodes []nodeInfo + if err := json.Unmarshal([]byte(nodesJSON), &nodes); err != nil { + klog.Warningf("Failed to unmarshal nodes from ConfigMap %s: %v", cm.Name, err) + continue + } + + // Check if this node is in the ConfigMap + nodeFound := false + for _, node := range nodes { + if node.Name == nodeName { + nodeFound = true + break + } + } + + // Track latest generation that contains this node + if nodeFound && gen > latestGeneration { + latestGeneration = gen + targetConfigMap = cm + } + } + if targetConfigMap == nil { + klog.V(4).Infof("No update-setup ConfigMap found containing node %s", nodeName) + return false, nil + } + + klog.V(4).Infof("Found update-setup ConfigMap generation %d containing node %s", latestGeneration, nodeName) + + // Check if update-setup job for this generation is complete + updateSetupJobs, err := kubeClient.BatchV1().Jobs("openshift-etcd").List(ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("app.kubernetes.io/name=%s", tools.JobTypeUpdateSetup.GetNameLabelValue()), + }) + if err != nil { + klog.Warningf("Failed to list update-setup jobs: %v", err) + return false, nil + } + + for _, job := range updateSetupJobs.Items { + if jobs.IsConditionTrue(job.Status.Conditions, batchv1.JobComplete) { + klog.Infof("Update-setup job %s completed successfully (contains node %s)", job.Name, nodeName) + return true, nil + } + } + + klog.V(4).Infof("Update-setup job for generation %d not yet complete", latestGeneration) + return false, nil + } + + err := wait.PollUntilContextTimeout(ctx, tools.JobPollInterval, tools.UpdateSetupJobCompletedTimeout, true, updateSetupDone) + if err != nil { + return fmt.Errorf("timed out waiting for update-setup job containing node %s to complete: %w", nodeName, err) + } return nil } diff --git a/pkg/tnf/auth/runner.go b/pkg/tnf/auth/runner.go index e583fb7f3d..615b6b2adf 100644 --- a/pkg/tnf/auth/runner.go +++ b/pkg/tnf/auth/runner.go @@ -49,8 +49,7 @@ func RunTnfAuth() error { klog.Info("Running TNF auth") - // create tnf cluster config - cfg, err := config.GetClusterConfig(ctx, kubeClient) + cfg, err := config.GetClusterConfigIgnoreMissingNode(ctx, kubeClient) if err != nil { klog.Errorf("Failed to get cluster config: %v", err) return err diff --git a/pkg/tnf/operator/lifecycle_job_controllers.go b/pkg/tnf/operator/lifecycle_job_controllers.go new file mode 100644 index 0000000000..9ae65930d4 --- /dev/null +++ b/pkg/tnf/operator/lifecycle_job_controllers.go @@ -0,0 +1,334 @@ +package operator + +import ( + "context" + "fmt" + "time" + + operatorv1 "github.com/openshift/api/operator/v1" + operatorv1informers "github.com/openshift/client-go/operator/informers/externalversions/operator/v1" + "github.com/openshift/library-go/pkg/controller/controllercmd" + "github.com/openshift/library-go/pkg/operator/v1helpers" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" + + "github.com/openshift/cluster-etcd-operator/pkg/operator/bootstrapteardown" + "github.com/openshift/cluster-etcd-operator/pkg/operator/ceohelpers" + "github.com/openshift/cluster-etcd-operator/pkg/operator/operatorclient" + "github.com/openshift/cluster-etcd-operator/pkg/tnf/pkg/etcd" + "github.com/openshift/cluster-etcd-operator/pkg/tnf/pkg/jobs" + "github.com/openshift/cluster-etcd-operator/pkg/tnf/pkg/tools" +) + +const ( + // Operator condition types + conditionTypeTNFJobControllersDegraded = "TNFJobControllersDegraded" +) + +var ( + // startTnfJobcontrollersFunc is a variable to allow mocking in tests + startTnfJobcontrollersFunc = startTnfJobcontrollers + + // retryBackoffConfig allows customizing retry behavior for tests + retryBackoffConfig = wait.Backoff{ + Duration: 5 * time.Second, + Factor: 2.0, + Steps: 9, // ~10 minutes total: 5s + 10s + 20s + 40s + 80s + 120s + 120s + 120s + 120s + Cap: 2 * time.Minute, + } +) + +// StartJobControllers starts TNF job controllers based on transition state. +// +// Before ExternalEtcdTransitionCompleted: +// - Requires exactly 2 ready control plane nodes +// - Uses exponential backoff retry (5s to 2min, ~10 min total) +// - Sets TNFJobControllersDegraded condition on failure +// - Mutex protects concurrent retry attempts +// +// After ExternalEtcdTransitionCompleted: +// - Accepts any number of ready control plane nodes (handles single-node case) +// - Idempotent (safe to call repeatedly, handles operator restarts) +// - No retry logic (controller framework retries sync() on error) +// - No mutex needed (job-level locking provides adequate protection) +// +// The job controllers are idempotent and will not create duplicate jobs. +func (c *PacemakerLifecycleManager) StartJobControllers(ctx context.Context) error { + // Check if external etcd transition is complete + transitionComplete, err := ceohelpers.HasExternalEtcdCompletedTransition(ctx, c.operatorClient) + if err != nil { + return fmt.Errorf("failed to check external etcd transition status: %w", err) + } + + // Check if node informer has synced + if c.nodeInformer == nil || !c.nodeInformer.HasSynced() { + klog.V(4).Infof("Skipping job controller startup - node informer not synced yet") + return nil + } + + // Get K8s control plane nodes + k8sNodes, err := ceohelpers.ListNodesFromInformer(c.nodeInformer) + if err != nil { + return fmt.Errorf("failed to list control plane nodes: %w", err) + } + + // Before transition: require exactly 2 ready control plane nodes for initial setup + // After transition: require all control plane nodes ready (handles single-node case) + if !transitionComplete { + // Pacemaker only supports 2 control plane nodes - initial setup requires exactly 2 + if len(k8sNodes) > 2 { + klog.Warningf("Found %d control plane nodes - pacemaker only supports 2, skipping initial setup", len(k8sNodes)) + return nil + } + if len(k8sNodes) < 2 { + klog.V(4).Infof("Waiting for 2 control plane nodes for initial setup (current: %d)", len(k8sNodes)) + return nil + } + + // Check both control plane nodes are Ready + for _, node := range k8sNodes { + if !tools.IsNodeReady(node) { + klog.V(4).Infof("Control plane node %s not Ready - waiting for both nodes before initial setup", node.Name) + return nil + } + } + + // Initial transition: use retry logic with exponential backoff and specific degraded condition + klog.V(2).Infof("Both control plane nodes ready - starting initial job controllers with retry") + return c.retryInitialTransitionOrDegrade(ctx, k8sNodes) + } else { + // Post-transition: idempotent call (handles operator restart, single-node case) + // Uses mutex to prevent concurrent startup attempts + + // Defensive check: ensure we have at least one control plane node + if len(k8sNodes) == 0 { + klog.V(4).Infof("No control plane nodes found, skipping job controller startup") + return nil + } + + // Check all control plane nodes are Ready + for _, node := range k8sNodes { + if !tools.IsNodeReady(node) { + klog.V(4).Infof("Control plane node %s not Ready - waiting for all control plane nodes before starting job controllers", node.Name) + return nil + } + } + + // Check if controllers have already been started + c.startJobControllersMu.Lock() + if c.jobControllersStarted { + c.startJobControllersMu.Unlock() + klog.V(4).Infof("Job controllers already started, skipping") + return nil + } + c.startJobControllersMu.Unlock() + + klog.V(2).Infof("Transition complete - ensuring job controllers running with %d ready control plane nodes", len(k8sNodes)) + err = c.startJobControllersWithLock(ctx, k8sNodes) + if err != nil { + return err + } + + // Mark as started on success + c.startJobControllersMu.Lock() + c.jobControllersStarted = true + c.startJobControllersMu.Unlock() + klog.Infof("Job controllers successfully started") + return nil + } +} + +// retryInitialTransitionOrDegrade starts TNF job controllers with exponential backoff retry. +// Sets TNFJobControllersDegraded condition on failure or success. +// This is only used for initial transition (before ExternalEtcdTransitionCompleted). +func (c *PacemakerLifecycleManager) retryInitialTransitionOrDegrade(ctx context.Context, nodes []*corev1.Node) error { + // Retry with exponential backoff to handle transient failures + var setupErr error + err := wait.ExponentialBackoffWithContext(ctx, retryBackoffConfig, func(ctx context.Context) (bool, error) { + setupErr = c.startJobControllersWithLock(ctx, nodes) + if setupErr != nil { + klog.Warningf("failed to setup TNF job controllers, will retry: %v", setupErr) + return false, nil + } + return true, nil + }) + + if err != nil || setupErr != nil { + // Prefer setupErr but fall back to err if setupErr is nil (e.g., context cancelled) + displayErr := setupErr + if displayErr == nil { + displayErr = err + } + klog.Errorf("failed to setup TNF job controllers after retries: %v", displayErr) + + // Degrade the operator to indicate TNF job controller setup failed + _, _, updateErr := v1helpers.UpdateStatus(ctx, c.operatorClient, v1helpers.UpdateConditionFn(operatorv1.OperatorCondition{ + Type: conditionTypeTNFJobControllersDegraded, + Status: operatorv1.ConditionTrue, + Reason: "SetupFailed", + Message: fmt.Sprintf("Failed to setup TNF job controllers after retries: %v", displayErr), + })) + if updateErr != nil { + klog.Errorf("failed to update operator status to degraded: %v", updateErr) + } + return displayErr + } + + // Clear any previous degraded condition on success + _, _, updateErr := v1helpers.UpdateStatus(ctx, c.operatorClient, v1helpers.UpdateConditionFn(operatorv1.OperatorCondition{ + Type: conditionTypeTNFJobControllersDegraded, + Status: operatorv1.ConditionFalse, + Reason: "AsExpected", + Message: "TNF job controllers setup completed successfully", + })) + if updateErr != nil { + klog.Errorf("failed to update operator status: %v", updateErr) + } + + return nil +} + +// startJobControllersWithLock is a wrapper that acquires the mutex before calling startTnfJobcontrollersFunc. +// This ensures only one job controller startup runs at a time, preventing: +// - Concurrent waits on etcd bootstrap completion +// - Concurrent waits on stable revision +// - Duplicate job controller creation attempts +// - Race conditions in the wait logic +// +// Used by both bootstrap path (via retryInitialTransitionOrDegrade) and post-transition path. +func (c *PacemakerLifecycleManager) startJobControllersWithLock(ctx context.Context, nodes []*corev1.Node) error { + c.startJobControllersMu.Lock() + defer c.startJobControllersMu.Unlock() + + return startTnfJobcontrollersFunc(ctx, nodes, c.controllerContext, c.operatorClient, c.kubeClient, c.kubeInformersForNamespaces, c.etcdInformer, c) +} + +// startTnfJobcontrollers creates TNF job controllers for the given nodes. +// It waits for etcd bootstrap completion, ensures all nodes are at latest revision, +// then creates auth, setup, fencing, and after-setup job controllers. +// Waits for after-setup jobs to complete to avoid races with update jobs. +func startTnfJobcontrollers( + ctx context.Context, + nodeList []*corev1.Node, + controllerContext *controllercmd.ControllerContext, + operatorClient v1helpers.StaticPodOperatorClient, + kubeClient kubernetes.Interface, + kubeInformersForNamespaces v1helpers.KubeInformersForNamespaces, + etcdInformer operatorv1informers.EtcdInformer, + lifecycleManager *PacemakerLifecycleManager) error { + + // Check if setup job already exists and is complete (operator restart scenario) + // If so, skip bootstrap flow and just ensure controllers are running + setupJobName := tools.JobTypeSetup.GetJobName(nil) + setupJob, err := kubeClient.BatchV1().Jobs(operatorclient.TargetNamespace).Get(ctx, setupJobName, metav1.GetOptions{}) + if err == nil && jobs.IsComplete(*setupJob) { + klog.Infof("Setup job already complete - skipping bootstrap flow, ensuring controllers are running") + + // Just start the controllers without going through bootstrap/setup again + // This prevents recreating jobs and racing with reconciliation + for _, node := range nodeList { + nodeTarget := &jobs.NodeTarget{Name: node.Name, UID: string(node.UID)} + jobs.RunTNFJobController(ctx, tools.JobTypeAuth, nodeTarget, nil, nil, 3, controllerContext, operatorClient, kubeClient, kubeInformersForNamespaces, jobs.DefaultConditions) + jobs.RunTNFJobController(ctx, tools.JobTypeAfterSetup, nodeTarget, nil, nil, 3, controllerContext, operatorClient, kubeClient, kubeInformersForNamespaces, jobs.DefaultConditions) + } + + clusterWideTargetNodesFunc := func() ([]*corev1.Node, error) { + return lifecycleManager.getActivePacemakerNodes() + } + + jobs.RunTNFJobController(ctx, tools.JobTypeSetup, nil, clusterWideTargetNodesFunc, nil, 3, controllerContext, operatorClient, kubeClient, kubeInformersForNamespaces, jobs.AllConditions) + + fencingJobConfigFunc := createFencingJobConfigFunc(nodeList, kubeInformersForNamespaces) + jobs.RunTNFJobController(ctx, tools.JobTypeFencing, nil, clusterWideTargetNodesFunc, fencingJobConfigFunc, 3, controllerContext, operatorClient, kubeClient, kubeInformersForNamespaces, jobs.DefaultConditions) + + // Check if there's a stopped update-setup job that needs a controller + // (operator may have restarted while update-setup was running/stopped) + updateSetupJobName := tools.JobTypeUpdateSetup.GetJobName(nil) + updateSetupJob, err := kubeClient.BatchV1().Jobs(operatorclient.TargetNamespace).Get(ctx, updateSetupJobName, metav1.GetOptions{}) + if err == nil && jobs.IsStopped(*updateSetupJob) && !jobs.IsControllerRunning(updateSetupJobName) { + klog.Infof("Found stopped update-setup job without controller - starting controller to update conditions") + jobs.RunTNFJobController(ctx, tools.JobTypeUpdateSetup, nil, clusterWideTargetNodesFunc, nil, 3, controllerContext, operatorClient, kubeClient, kubeInformersForNamespaces, jobs.DefaultConditions) + } + + klog.Infof("Controllers running for existing completed jobs") + return nil + } + if err != nil && !apierrors.IsNotFound(err) { + klog.Warningf("Failed to check setup job status: %v - proceeding with bootstrap flow", err) + } + + klog.Infof("Running TNF setup procedure. Waiting for etcd bootstrap to complete") + + // Wait for the etcd informer to sync before checking bootstrap status + // This ensures operatorClient.GetStaticPodOperatorState() has data to work with + klog.Infof("waiting for etcd informer to sync...") + if !cache.WaitForCacheSync(ctx.Done(), etcdInformer.Informer().HasSynced) { + return fmt.Errorf("failed to sync etcd informer") + } + klog.Infof("etcd informer synced") + + if err := waitForEtcdBootstrapCompleted(ctx, operatorClient); err != nil { + return fmt.Errorf("failed to wait for etcd bootstrap: %w", err) + } + + // Wait for all nodes to have their installers complete (creates /var/lib/etcd) + klog.Infof("bootstrap completed, waiting for all nodes to reach latest revision") + if err := etcd.WaitForStableRevision(ctx, operatorClient); err != nil { + return fmt.Errorf("failed to wait for all nodes at latest revision: %w", err) + } + + klog.Infof("all nodes at latest revision, creating TNF job controllers") + + // the order of job creation does not matter, the jobs wait on each other as needed + for _, node := range nodeList { + // Node-specific jobs: auth and after-setup are tied to individual nodes + // Single-node: retries=3 sets backoffLimit (Kubernetes retries on same node) + nodeTarget := &jobs.NodeTarget{Name: node.Name, UID: string(node.UID)} + jobs.RunTNFJobController(ctx, tools.JobTypeAuth, nodeTarget, nil, nil, 3, controllerContext, operatorClient, kubeClient, kubeInformersForNamespaces, jobs.DefaultConditions) + jobs.RunTNFJobController(ctx, tools.JobTypeAfterSetup, nodeTarget, nil, nil, 3, controllerContext, operatorClient, kubeClient, kubeInformersForNamespaces, jobs.DefaultConditions) + } + + // Create targetNodesFunc for multi-node retry across all control plane nodes + // Used by setup and fencing jobs which can run on any node + clusterWideTargetNodesFunc := func() ([]*corev1.Node, error) { + return lifecycleManager.getActivePacemakerNodes() + } + + // Cluster-wide jobs: setup and fencing can run on any node + // Multi-node: retries=3 means try all nodes 3 times before degrading + jobs.RunTNFJobController(ctx, tools.JobTypeSetup, nil, clusterWideTargetNodesFunc, nil, 3, controllerContext, operatorClient, kubeClient, kubeInformersForNamespaces, jobs.AllConditions) + + // Fencing job with drift detection: captures node UIDs + fencing secret UIDs + fencingJobConfigFunc := createFencingJobConfigFunc(nodeList, kubeInformersForNamespaces) + jobs.RunTNFJobController(ctx, tools.JobTypeFencing, nil, clusterWideTargetNodesFunc, fencingJobConfigFunc, 3, controllerContext, operatorClient, kubeClient, kubeInformersForNamespaces, jobs.DefaultConditions) + + return nil +} + +// waitForEtcdBootstrapCompleted waits for etcd bootstrap to complete. +// If etcd is not yet running in cluster, it waits for bootstrap teardown. +func waitForEtcdBootstrapCompleted(ctx context.Context, operatorClient v1helpers.StaticPodOperatorClient) error { + isEtcdRunningInCluster, err := ceohelpers.IsEtcdRunningInCluster(ctx, operatorClient) + if err != nil { + return fmt.Errorf("failed to check if bootstrap is completed: %w", err) + } + if !isEtcdRunningInCluster { + klog.Infof("waiting for bootstrap to complete with etcd running in cluster") + clientConfig, err := rest.InClusterConfig() + if err != nil { + return fmt.Errorf("failed to get in-cluster config: %w", err) + } + err = bootstrapteardown.WaitForEtcdBootstrap(ctx, clientConfig) + if err != nil { + return fmt.Errorf("failed to wait for bootstrap to complete: %w", err) + } + } + return nil +} + diff --git a/pkg/tnf/operator/lifecycle_job_controllers_test.go b/pkg/tnf/operator/lifecycle_job_controllers_test.go new file mode 100644 index 0000000000..bbf9de4d6b --- /dev/null +++ b/pkg/tnf/operator/lifecycle_job_controllers_test.go @@ -0,0 +1,417 @@ +package operator + +/* +TEST COVERAGE SUMMARY - lifecycle_job_controllers_test.go +========================================================== + +This file tests TNF job controller startup logic for Two-Node Fencing clusters. + +WHAT'S TESTED +------------- + +Job Controller Startup: +├── TestRetryInitialTransitionOrDegrade - Bootstrap retry and degraded condition +│ ├── ✅ Success on first attempt (condition cleared) +│ ├── ✅ Success after retries (condition cleared) +│ └── ✅ Failure after all retries (condition set to degraded) +└── TestStartJobControllers - Entry point logic and path selection + ├── Before transition (bootstrap path): + │ ├── ✅ 2 ready nodes → bootstrap with retry + │ ├── ✅ 1 ready node → wait + │ └── ✅ 3 nodes → skip (pacemaker only supports 2) + └── After transition (post-transition path): + ├── ✅ 2 ready nodes → ensure running + ├── ✅ 1 ready node → ensure running (single-node case) + └── ✅ 0 ready nodes → skip +*/ + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/tools/cache" + "k8s.io/utils/clock" + + operatorv1 "github.com/openshift/api/operator/v1" + operatorversionedclientfake "github.com/openshift/client-go/operator/clientset/versioned/fake" + extinfops "github.com/openshift/client-go/operator/informers/externalversions" + operatorv1informers "github.com/openshift/client-go/operator/informers/externalversions/operator/v1" + "github.com/openshift/cluster-etcd-operator/pkg/operator/ceohelpers" + "github.com/openshift/cluster-etcd-operator/pkg/operator/operatorclient" + u "github.com/openshift/cluster-etcd-operator/pkg/testutils" + "github.com/openshift/library-go/pkg/controller/controllercmd" + "github.com/openshift/library-go/pkg/operator/events" + "github.com/openshift/library-go/pkg/operator/v1helpers" +) + +// TestRetryInitialTransitionOrDegrade tests the exponential backoff retry logic +// and TNFJobControllersDegraded condition management during initial bootstrap. +func TestRetryInitialTransitionOrDegrade(t *testing.T) { + tests := []struct { + name string + setupMockStartFunc func() func(context.Context, []*corev1.Node, *controllercmd.ControllerContext, v1helpers.StaticPodOperatorClient, kubernetes.Interface, v1helpers.KubeInformersForNamespaces, operatorv1informers.EtcdInformer, *PacemakerLifecycleManager) error + expectDegradedCondition bool + expectDegradedStatus operatorv1.ConditionStatus + expectRetries bool + }{ + { + name: "Success on first attempt", + setupMockStartFunc: func() func(context.Context, []*corev1.Node, *controllercmd.ControllerContext, v1helpers.StaticPodOperatorClient, kubernetes.Interface, v1helpers.KubeInformersForNamespaces, operatorv1informers.EtcdInformer) error { + return func(_ context.Context, _ []*corev1.Node, _ *controllercmd.ControllerContext, _ v1helpers.StaticPodOperatorClient, _ kubernetes.Interface, _ v1helpers.KubeInformersForNamespaces, _ operatorv1informers.EtcdInformer, _ *PacemakerLifecycleManager) error { + return nil + } + }, + expectDegradedCondition: true, + expectDegradedStatus: operatorv1.ConditionFalse, + expectRetries: false, + }, + { + name: "Success after retries", + setupMockStartFunc: func() func(context.Context, []*corev1.Node, *controllercmd.ControllerContext, v1helpers.StaticPodOperatorClient, kubernetes.Interface, v1helpers.KubeInformersForNamespaces, operatorv1informers.EtcdInformer) error { + attemptCount := 0 + return func(_ context.Context, _ []*corev1.Node, _ *controllercmd.ControllerContext, _ v1helpers.StaticPodOperatorClient, _ kubernetes.Interface, _ v1helpers.KubeInformersForNamespaces, _ operatorv1informers.EtcdInformer, _ *PacemakerLifecycleManager) error { + attemptCount++ + if attemptCount < 3 { + return &retryableError{msg: "temporary failure"} + } + return nil + } + }, + expectDegradedCondition: true, + expectDegradedStatus: operatorv1.ConditionFalse, + expectRetries: true, + }, + { + name: "Failure after all retries", + setupMockStartFunc: func() func(context.Context, []*corev1.Node, *controllercmd.ControllerContext, v1helpers.StaticPodOperatorClient, kubernetes.Interface, v1helpers.KubeInformersForNamespaces, operatorv1informers.EtcdInformer) error { + return func(_ context.Context, _ []*corev1.Node, _ *controllercmd.ControllerContext, _ v1helpers.StaticPodOperatorClient, _ kubernetes.Interface, _ v1helpers.KubeInformersForNamespaces, _ operatorv1informers.EtcdInformer, _ *PacemakerLifecycleManager) error { + return &retryableError{msg: "persistent failure"} + } + }, + expectDegradedCondition: true, + expectDegradedStatus: operatorv1.ConditionTrue, + expectRetries: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Setup test environment + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + fakeKubeClient := fake.NewSimpleClientset() + fakeOperatorClient := v1helpers.NewFakeStaticPodOperatorClient( + &operatorv1.StaticPodOperatorSpec{}, + u.StaticPodOperatorStatus(), + nil, + nil, + ) + + eventRecorder := events.NewRecorder(fakeKubeClient.CoreV1().Events(operatorclient.TargetNamespace), + "test-retry", &corev1.ObjectReference{}, clock.RealClock{}) + + controllerContext := &controllercmd.ControllerContext{ + EventRecorder: eventRecorder, + } + + kubeInformersForNamespaces := v1helpers.NewKubeInformersForNamespaces( + fakeKubeClient, + operatorclient.TargetNamespace, + ) + + // Create etcd informer + operatorClientFake := operatorversionedclientfake.NewClientset() + etcdInformers := extinfops.NewSharedInformerFactory(operatorClientFake, 10*time.Minute) + + // Create lifecycle manager + manager := &PacemakerLifecycleManager{ + operatorClient: fakeOperatorClient, + kubeClient: fakeKubeClient, + controllerContext: controllerContext, + kubeInformersForNamespaces: kubeInformersForNamespaces, + etcdInformer: etcdInformers.Operator().V1().Etcds(), + } + + // Store original startTnfJobcontrollersFunc and replace with mock + originalStartFunc := startTnfJobcontrollersFunc + startTnfJobcontrollersFunc = tt.setupMockStartFunc() + defer func() { startTnfJobcontrollersFunc = originalStartFunc }() + + // Store original backoff config and use faster settings for testing + originalBackoff := retryBackoffConfig + retryBackoffConfig = wait.Backoff{ + Duration: 100 * time.Millisecond, + Factor: 2.0, + Steps: 5, // Much shorter for testing + Cap: 500 * time.Millisecond, + } + defer func() { retryBackoffConfig = originalBackoff }() + + // Create test nodes + nodes := []*corev1.Node{ + {ObjectMeta: metav1.ObjectMeta{Name: "master-0"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "master-1"}}, + } + + // Run retryInitialTransitionOrDegrade + err := manager.retryInitialTransitionOrDegrade(ctx, nodes) + + // Verify error expectation + if tt.expectDegradedStatus == operatorv1.ConditionTrue { + require.Error(t, err, "Expected error when retries exhausted") + } else { + require.NoError(t, err, "Expected no error on success") + } + + // Verify the operator condition was set correctly + if tt.expectDegradedCondition { + _, status, _, err := fakeOperatorClient.GetStaticPodOperatorState() + require.NoError(t, err, "failed to get operator status") + + // Find the TNFJobControllersDegraded condition + var foundCondition *operatorv1.OperatorCondition + for i, condition := range status.Conditions { + if condition.Type == conditionTypeTNFJobControllersDegraded { + foundCondition = &status.Conditions[i] + break + } + } + + require.NotNil(t, foundCondition, "TNFJobControllersDegraded condition not found") + require.Equal(t, tt.expectDegradedStatus, foundCondition.Status, + "Expected degraded status %v but got %v", tt.expectDegradedStatus, foundCondition.Status) + + if tt.expectDegradedStatus == operatorv1.ConditionTrue { + require.Equal(t, "SetupFailed", foundCondition.Reason, + "Expected reason SetupFailed but got %s", foundCondition.Reason) + require.Contains(t, foundCondition.Message, "Failed to setup TNF job controllers", + "Expected message to contain failure info") + } else { + require.Equal(t, "AsExpected", foundCondition.Reason, + "Expected reason AsExpected but got %s", foundCondition.Reason) + require.Contains(t, foundCondition.Message, "successfully", + "Expected success message") + } + } + }) + } +} + +// TestStartJobControllers tests the entry point logic for starting job controllers. +// This includes transition checks, node count validation, and path selection. +func TestStartJobControllers(t *testing.T) { + tests := []struct { + name string + transitionComplete bool + nodeCount int + readyNodeCount int + expectStartCalled bool + expectRetryPath bool + }{ + { + name: "before transition - 2 ready nodes - bootstrap with retry", + transitionComplete: false, + nodeCount: 2, + readyNodeCount: 2, + expectStartCalled: true, + expectRetryPath: true, + }, + { + name: "before transition - 1 ready node - wait", + transitionComplete: false, + nodeCount: 2, + readyNodeCount: 1, + expectStartCalled: false, + expectRetryPath: false, + }, + { + name: "before transition - 3 nodes - skip (pacemaker only supports 2)", + transitionComplete: false, + nodeCount: 3, + readyNodeCount: 3, + expectStartCalled: false, + expectRetryPath: false, + }, + { + name: "after transition - 2 ready nodes - ensure running", + transitionComplete: true, + nodeCount: 2, + readyNodeCount: 2, + expectStartCalled: true, + expectRetryPath: false, + }, + { + name: "after transition - 1 ready node - ensure running (handles single-node)", + transitionComplete: true, + nodeCount: 1, + readyNodeCount: 1, + expectStartCalled: true, + expectRetryPath: false, + }, + { + name: "after transition - 0 ready nodes - skip", + transitionComplete: true, + nodeCount: 1, + readyNodeCount: 0, + expectStartCalled: false, + expectRetryPath: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + + // Create fake clients + fakeKubeClient := fake.NewSimpleClientset() + + // Create nodes + var nodes []runtime.Object + for i := 0; i < tt.nodeCount; i++ { + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("master-%d", i), + Labels: map[string]string{"node-role.kubernetes.io/control-plane": ""}, + }, + Status: corev1.NodeStatus{}, + } + // Mark nodes as ready up to readyNodeCount + if i < tt.readyNodeCount { + node.Status.Conditions = []corev1.NodeCondition{ + {Type: corev1.NodeReady, Status: corev1.ConditionTrue}, + } + } else { + node.Status.Conditions = []corev1.NodeCondition{ + {Type: corev1.NodeReady, Status: corev1.ConditionFalse}, + } + } + nodes = append(nodes, node) + } + + // Add nodes to fake client + for _, node := range nodes { + _, err := fakeKubeClient.CoreV1().Nodes().Create(ctx, node.(*corev1.Node), metav1.CreateOptions{}) + require.NoError(t, err) + } + + // Create operator client with transition state + var conditions []operatorv1.OperatorCondition + if tt.transitionComplete { + conditions = append(conditions, operatorv1.OperatorCondition{ + Type: ceohelpers.OperatorConditionExternalEtcdHasCompletedTransition, + Status: operatorv1.ConditionTrue, + }) + } + fakeOperatorClient := v1helpers.NewFakeStaticPodOperatorClient( + &operatorv1.StaticPodOperatorSpec{}, + &operatorv1.StaticPodOperatorStatus{ + OperatorStatus: operatorv1.OperatorStatus{ + Conditions: conditions, + }, + }, + nil, + nil, + ) + + eventRecorder := events.NewRecorder(fakeKubeClient.CoreV1().Events(operatorclient.TargetNamespace), + "test-start", &corev1.ObjectReference{}, clock.RealClock{}) + + controllerContext := &controllercmd.ControllerContext{ + EventRecorder: eventRecorder, + } + + kubeInformersForNamespaces := v1helpers.NewKubeInformersForNamespaces( + fakeKubeClient, + operatorclient.TargetNamespace, + ) + + // Create node informer + nodeIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{}) + for _, node := range nodes { + require.NoError(t, nodeIndexer.Add(node), "failed to add node to indexer") + } + nodeInformer := &mockNodeInformerLM{ + indexer: nodeIndexer, + synced: true, + } + + // Create etcd informer + operatorClientFake := operatorversionedclientfake.NewClientset() + etcdInformers := extinfops.NewSharedInformerFactory(operatorClientFake, 10*time.Minute) + + // Create lifecycle manager + manager := &PacemakerLifecycleManager{ + operatorClient: fakeOperatorClient, + kubeClient: fakeKubeClient, + nodeInformer: nodeInformer, + controllerContext: controllerContext, + kubeInformersForNamespaces: kubeInformersForNamespaces, + etcdInformer: etcdInformers.Operator().V1().Etcds(), + } + + // Track if startTnfJobcontrollersFunc was called + startCalled := false + originalStartFunc := startTnfJobcontrollersFunc + startTnfJobcontrollersFunc = func(_ context.Context, _ []*corev1.Node, _ *controllercmd.ControllerContext, _ v1helpers.StaticPodOperatorClient, _ kubernetes.Interface, _ v1helpers.KubeInformersForNamespaces, _ operatorv1informers.EtcdInformer, _ *PacemakerLifecycleManager) error { + startCalled = true + return nil + } + defer func() { startTnfJobcontrollersFunc = originalStartFunc }() + + // Use faster backoff for testing + originalBackoff := retryBackoffConfig + retryBackoffConfig = wait.Backoff{ + Duration: 10 * time.Millisecond, + Factor: 2.0, + Steps: 2, + Cap: 50 * time.Millisecond, + } + defer func() { retryBackoffConfig = originalBackoff }() + + // Execute + err := manager.StartJobControllers(ctx) + + // Verify + require.NoError(t, err) + require.Equal(t, tt.expectStartCalled, startCalled, + "Expected startTnfJobcontrollersFunc called=%v, got=%v", tt.expectStartCalled, startCalled) + + // If retry path expected, verify TNFJobControllersDegraded condition was set + if tt.expectRetryPath && tt.expectStartCalled { + _, status, _, err := fakeOperatorClient.GetStaticPodOperatorState() + require.NoError(t, err) + + var foundCondition *operatorv1.OperatorCondition + for i, condition := range status.Conditions { + if condition.Type == conditionTypeTNFJobControllersDegraded { + foundCondition = &status.Conditions[i] + break + } + } + + require.NotNil(t, foundCondition, "TNFJobControllersDegraded condition should be set on retry path") + require.Equal(t, operatorv1.ConditionFalse, foundCondition.Status, + "TNFJobControllersDegraded should be False on success") + } + }) + } +} + +// retryableError is a helper type for testing retry logic +type retryableError struct { + msg string +} + +func (e *retryableError) Error() string { + return e.msg +} diff --git a/pkg/tnf/pkg/pcs/auth.go b/pkg/tnf/pkg/pcs/auth.go index 52e1fef1e6..075860dcc7 100644 --- a/pkg/tnf/pkg/pcs/auth.go +++ b/pkg/tnf/pkg/pcs/auth.go @@ -46,7 +46,11 @@ func Authenticate(ctx context.Context, configClient *versioned.Clientset, cfg co return false, fmt.Errorf("failed to accept token: %w", err) } - command = fmt.Sprintf("/usr/sbin/pcs host auth %s addr=%s %s addr=%s --token %s --debug", cfg.NodeName1, cfg.NodeIP1, cfg.NodeName2, cfg.NodeIP2, TokenPath) + if cfg.NodeName2 != "" && cfg.NodeIP2 != "" { + command = fmt.Sprintf("/usr/sbin/pcs host auth %s addr=%s %s addr=%s --token %s --debug", cfg.NodeName1, cfg.NodeIP1, cfg.NodeName2, cfg.NodeIP2, TokenPath) + } else { + command = fmt.Sprintf("/usr/sbin/pcs host auth %s addr=%s --token %s --debug", cfg.NodeName1, cfg.NodeIP1, TokenPath) + } _, _, err = exec.Execute(ctx, command) if err != nil { return false, fmt.Errorf("failed to authenticate node: %w", err) diff --git a/pkg/tnf/pkg/tools/jobs.go b/pkg/tnf/pkg/tools/jobs.go index a71975d9aa..9822cdff19 100644 --- a/pkg/tnf/pkg/tools/jobs.go +++ b/pkg/tnf/pkg/tools/jobs.go @@ -14,12 +14,13 @@ import ( // setup job: waits for auth jobs to complete // after setup jobs: waits for setup job to complete const ( - JobPollInterval = 15 * time.Second - AuthJobCompletedTimeout = 10 * time.Minute - SetupJobCompletedTimeout = 20 * time.Minute - AfterSetupJobCompletedTimeout = 5 * time.Minute - AllCompletedTimeout = 30 * time.Minute - FencingJobCompletedTimeout = 25 * time.Minute + JobPollInterval = 15 * time.Second + AuthJobCompletedTimeout = 10 * time.Minute + SetupJobCompletedTimeout = 20 * time.Minute + AfterSetupJobCompletedTimeout = 5 * time.Minute + UpdateSetupJobCompletedTimeout = 20 * time.Minute + AllCompletedTimeout = 30 * time.Minute + FencingJobCompletedTimeout = 25 * time.Minute ) // JobType represent the different jobs we run, with some methods needed From 143960013996aeeea772640c6fa99c7e2d109913 Mon Sep 17 00:00:00 2001 From: Jeremy Poulin Date: Thu, 25 Jun 2026 15:56:33 -0400 Subject: [PATCH 3/6] OCPBUGS-84695: feat(tnf): drift reconciliation Implements drift detection and reconciliation for Pacemaker cluster membership: - ReconcilePacemakerConfig(): unified entry point for drift reconciliation - Detects membership drift: compares K8s nodes vs Pacemaker nodes - Detects IP changes: compares node IPs to detect node replacements - ConfigMap generation tracking: prevents re-running same update-setup job - Target node selection: intersection of K8s and Pacemaker nodes (update-setup must run on active Pacemaker node) - ConfigMap reuse: when desired state unchanged, reuses existing ConfigMap generation - Transition protection: only runs after HasExternalEtcdCompletedTransition + setup job complete - Update-setup implementation: adds/removes nodes, configures fencing, updates etcd resource - Setup job implementation: waits for auth jobs before Pacemaker operations Event handlers: - Node Add/Ready: both call ReconcilePacemakerConfig (fixes race) - Node Update: detects IP changes - Node Delete: triggers reconciliation See docs/tnf/reconciliation.md for drift detection scenarios and reconciliation workflow. Co-Authored-By: Claude Sonnet 4.5 --- docs/tnf/reconciliation.md | 494 +++++++++++ pkg/operator/ceohelpers/common.go | 15 + pkg/operator/ceohelpers/common_test.go | 89 ++ pkg/tnf/operator/lifecycle_helpers.go | 67 ++ pkg/tnf/operator/lifecycle_reconciliation.go | 811 ++++++++++++++++++ .../operator/lifecycle_reconciliation_test.go | 137 +++ pkg/tnf/operator/starter.go | 247 +++--- pkg/tnf/operator/starter_test.go | 175 ++-- pkg/tnf/setup/runner.go | 13 +- pkg/tnf/update-setup/runner.go | 628 ++++++++++++-- pkg/tnf/update-setup/runner_test.go | 138 +++ 11 files changed, 2477 insertions(+), 337 deletions(-) create mode 100644 docs/tnf/reconciliation.md create mode 100644 pkg/tnf/operator/lifecycle_helpers.go create mode 100644 pkg/tnf/operator/lifecycle_reconciliation.go create mode 100644 pkg/tnf/operator/lifecycle_reconciliation_test.go create mode 100644 pkg/tnf/update-setup/runner_test.go diff --git a/docs/tnf/reconciliation.md b/docs/tnf/reconciliation.md new file mode 100644 index 0000000000..06abaa2940 --- /dev/null +++ b/docs/tnf/reconciliation.md @@ -0,0 +1,494 @@ +# Pacemaker Reconciliation Flow + +## Overview + +Reconciliation ensures that Pacemaker cluster membership stays synchronized with Kubernetes control plane nodes. It detects drift (mismatches between K8s and Pacemaker) and automatically fixes it by adding or removing nodes from the Pacemaker cluster. + +## When Reconciliation Runs + +Reconciliation is triggered by multiple sources: + +1. **Periodic Sync** - Every 30 seconds (from `sync()` loop) +2. **Node Add Event** - When a new control plane node joins the cluster +3. **Node Update Event** - When a node's IP address changes while Ready +4. **Node Delete Event** - When a control plane node is removed from the cluster + +## High-Level Flow + +```text +┌──────────────────────────────────────────────────────────────┐ +│ ReconcilePacemakerConfig() │ +└──────────────────────────────────────────────────────────────┘ + │ + ▼ + ┌────────────────────────────────┐ + │ Safety Checks: │ + │ - Node informer synced? │ + │ - Valid node count (<= 2)? │ + │ - Transition complete? │ + │ - Setup job complete? │ + └────────────────────────────────┘ + │ + ▼ + ┌────────────────────────────────┐ + │ Get Current State: │ + │ - K8s nodes (from informer) │ + │ - Pacemaker nodes (from CR) │ + │ - Stopped jobs (from API) │ + └────────────────────────────────┘ + │ + ▼ + ┌────────────────────────────────┐ + │ Detect Drift: │ + │ - Compare node names │ + │ - Compare node IPs │ + │ - Check for stale CR │ + └────────────────────────────────┘ + │ + ┌───────┴────────┐ + │ │ + No drift Drift detected + │ │ + ▼ ▼ + ┌────────────────┐ ┌──────────────────────┐ + │ Check for │ │ Acquire mutex │ + │ stopped job │ │ (serialize updates) │ + └────────────────┘ └──────────────────────┘ + │ │ + ┌────┴────┐ ▼ + No Yes ┌──────────────────────┐ + │ │ │ Re-check drift │ + ▼ ▼ │ (after lock) │ + Return Delete └──────────────────────┘ + stopped │ + job ┌──────┴──────┐ + │ │ + Still drift No drift + │ │ + ▼ ▼ + ┌──────────────────────┐ Return + │ Check current │ + │ generation ConfigMap │ + └──────────────────────┘ + │ + ┌──────┴──────┐ + │ │ + Matches desired Different state + │ │ + ▼ ▼ + ┌──────────────┐ ┌──────────────────┐ + │ Reuse gen N │ │ Create new │ + │ Check if job:│ │ ConfigMap gen N+1│ + │ - Running? │ └──────────────────┘ + │ - Complete? │ │ + └──────────────┘ │ + │ │ + ┌───┴────┐ │ + │ │ │ + Running or Failed or │ + Complete Not exists │ + │ │ │ + ▼ │ │ + Return └───────────┘ + │ + ▼ + ┌──────────────────┐ + │ Run auth jobs │ + │ on all nodes │ + └──────────────────┘ + │ + ▼ + ┌──────────────────┐ + │ Run update-setup │ + │ job │ + └──────────────────┘ + │ + ▼ + ┌──────────────────┐ + │ Wait for │ + │ completion │ + └──────────────────┘ + │ + ▼ + ┌──────────────────┐ + │ Run after-setup │ + │ jobs on all nodes│ + └──────────────────┘ +``` + +## Drift Detection + +Drift occurs when Kubernetes node state doesn't match Pacemaker cluster membership. + +### What is Compared + +```text +┌─────────────────────────┐ ┌─────────────────────────┐ +│ Kubernetes Nodes │ │ Pacemaker Nodes │ +│ (from node informer) │ vs │ (from CR status) │ +└─────────────────────────┘ └─────────────────────────┘ + │ │ + │ │ + ┌────┴────┐ ┌────┴────┐ + │ Names │ │ Names │ + │ IPs │ │ IPs │ + └─────────┘ └─────────┘ +``` + +### Drift Scenarios + +**Scenario 1: Node Added to Kubernetes** + +```text +Kubernetes: [master-0, master-1, master-2] +Pacemaker: [master-0, master-1] + +Drift: YES - master-2 exists in K8s but not Pacemaker +Action: Add master-2 to Pacemaker cluster +``` + +**Scenario 2: Node Deleted from Kubernetes** + +```text +Kubernetes: [master-0] +Pacemaker: [master-0, master-1] + +Drift: YES - master-1 exists in Pacemaker but not K8s +Action: Remove master-1 from Pacemaker cluster +``` + +**Scenario 3: Node IP Changed** + +```text +Kubernetes: master-0 IP = 192.168.1.10 +Pacemaker: master-0 IP = 192.168.1.11 + +Drift: YES - Node name matches but IP differs +Action: Update Pacemaker with new IP (remove old, add new) +``` + +**Scenario 4: No Drift** + +```text +Kubernetes: [master-0 (192.168.1.10), master-1 (192.168.1.11)] +Pacemaker: [master-0 (192.168.1.10), master-1 (192.168.1.11)] + +Drift: NO - Names and IPs match exactly +Action: None +``` + +## Update-Setup ConfigMap + +When drift is detected, a ConfigMap is created (or reused) with the desired node state. + +### ConfigMap Structure + +**Name:** `tnf-update-setup-` in `openshift-etcd` namespace + +**Data Fields:** +- `generation`: Ordering number for reconciliation ordering +- `nodes`: JSON array of desired node state - `[{"name": "master-0", "ip": "192.168.1.10"}, ...]` +- `timestamp`: When ConfigMap was created + +### Generation Counter + +The generation counter ensures correctness when multiple reconciliations are triggered close together, and prevents redundant job creation when state hasn't changed. + +**Generation Reuse:** + +When drift is detected, reconciliation first checks if the current generation's ConfigMap already describes the desired state: + +```text +Current generation ConfigMap exists with nodes: [master-0, master-1] +Desired state: [master-0, master-1] + ↓ +States match → Reuse generation N + ↓ +Check if job already running or completed + ↓ +If yes → Skip restart (avoid interrupting work) +If no/failed → Restart job with same generation +``` + +**New Generation Creation:** + +Only create a new generation if desired state differs from current: + +```text +Current generation ConfigMap: [master-0, master-1] +Desired state: [master-0, master-1, master-2] + ↓ +States differ → Create new generation N+1 + ↓ +Start update-setup job for generation N+1 +``` + +**Multiple Concurrent Reconciliations:** + +```text +Event 1: Node deleted at T=0 + ↓ + Generate ConfigMap with generation=1 + +Event 2: Different node deleted at T=0.1s (before job starts) + ↓ + Generate ConfigMap with generation=2 + +update-setup job reads BOTH ConfigMaps, uses highest generation (2) + ↓ + Reconciles both deletions in a single run +``` + +**Key Properties:** +- Highest generation always wins, ensuring most recent state is applied +- Reuse generation when desired state unchanged, preventing redundant job restarts +- Skip restart if job already running or completed successfully (avoid interruption) + +## Update-Setup Job Execution + +The update-setup job runs on a Pacemaker node and reconciles Pacemaker membership to match the desired state from the ConfigMap. + +### Job Flow + +```text +┌──────────────────────────────────────────────────────────────┐ +│ update-setup job (runs on pacemaker node) │ +└──────────────────────────────────────────────────────────────┘ + │ + ▼ + ┌────────────────────────────────┐ + │ Read ConfigMaps: │ + │ - Find all tnf-update-setup-* │ + │ - Parse desired node list │ + │ - Use highest generation │ + └────────────────────────────────┘ + │ + ▼ + ┌────────────────────────────────┐ + │ Remove Nodes: │ + │ - pcs cluster node remove │ + │ - Remove unstarted etcd members│ + └────────────────────────────────┘ + │ + ▼ + ┌────────────────────────────────┐ + │ Add Nodes: │ + │ - pcs cluster node add │ + └────────────────────────────────┘ + │ + ▼ + ┌────────────────────────────────┐ + │ Update Fencing: │ + │ - Update STONITH configuration │ + │ - Add/remove fence agents │ + └────────────────────────────────┘ + │ + ▼ + ┌────────────────────────────────┐ + │ Update Etcd Resource: │ + │ - Update node IP list │ + │ - Trigger resource restart │ + └────────────────────────────────┘ + │ + ▼ + ┌────────────────────────────────┐ + │ If nodes added: │ + │ - Force new cluster │ + │ - Restart cluster │ + └────────────────────────────────┘ + │ + ▼ + ┌────────────────────────────────┐ + │ Sync and Start: │ + │ - pcs cluster sync │ + │ - pcs cluster start --all │ + └────────────────────────────────┘ +``` + +### Auth Jobs (Pre-Reconciliation) + +Before running update-setup, auth jobs run on **all nodes** to ensure Pacemaker authentication is current: + +```text +For each K8s node: + ┌────────────────────────────────┐ + │ auth job │ + │ - Set hacluster password │ + │ - Sync pacemaker auth │ + └────────────────────────────────┘ +``` + +**Why?** Ensures nodes can communicate with Pacemaker before membership changes. + +### After-Setup Jobs (Post-Reconciliation) + +After update-setup completes, after-setup jobs run on **all nodes**: + +```text +For each K8s node: + ┌────────────────────────────────┐ + │ after-setup job │ + │ - Verify node in cluster │ + │ - Start pacemaker services │ + └────────────────────────────────┘ +``` + +**Why?** Ensures all nodes have Pacemaker running after membership changes. + +## Concurrency Protection + +Multiple goroutines can call `ReconcilePacemakerConfig()` concurrently (periodic sync + node events). The function uses a two-phase check pattern: + +### Phase 1: Unlocked Drift Detection + +```text +Goroutine A Goroutine B + │ │ + ▼ ▼ +Check drift Check drift + │ │ +Drift detected Drift detected + │ │ + ▼ ▼ +``` + +**Purpose:** Allow concurrent drift detection without blocking. + +### Phase 2: Serialized Update + +```text +Goroutine A Goroutine B + │ │ + ▼ │ +Lock mutex │ + │ ▼ +Re-check drift Wait for lock + │ │ +Still drift │ + │ ▼ +Create ConfigMap Lock mutex + │ │ +Start job ▼ + │ Re-check drift + ▼ │ +Unlock mutex No drift (A fixed it) + ▼ + Unlock mutex +``` + +**Purpose:** Prevent duplicate reconciliation while allowing eventual consistency. + +### Generation Counter Protection + +The mutex also protects the generation counter increment: + +```go +reconcilePacemakerConfigMutex.Lock() +generation = updateSetupGeneration++ +reconcilePacemakerConfigMutex.Unlock() +``` + +**Why?** Ensures unique, monotonically increasing generation numbers. + +## Error Handling + +Reconciliation is designed for eventual consistency: + +**Transient Errors:** +- Node informer not synced → Skip this attempt, retry in 30s +- Setup job not complete → Skip this attempt, retry in 30s (prevents update-setup from running during initial setup) +- CR not available → Discovery mode (try nodes sequentially) +- Job creation fails → Error propagates to sync(), retry in 30s + +**Permanent Failures:** +- More than 2 control plane nodes → Warning logged, no action +- No valid target nodes → Error returned, manual intervention may be needed +- All discovery attempts failed → Error returned + +**Job State Handling:** +- Job running + desired state matches current generation → Skip restart (wait for completion) +- Job completed successfully + desired state matches current generation → Skip restart (wait for convergence) +- Job failed + desired state matches current generation → Restart job (retry with same generation) +- Job not exists + desired state matches current generation → Start job +- Job any state + desired state differs from current generation → Create new ConfigMap, restart job + +## Discovery Mode + +When PacemakerCluster CR is unavailable or stale, reconciliation enters discovery mode: + +```text +┌──────────────────────────────────────────────────────────────┐ +│ Discovery Mode (no actionable CR) │ +└──────────────────────────────────────────────────────────────┘ + │ + ▼ + ┌────────────────────────────────┐ + │ Check stopped job: │ + │ - Which node was last tried? │ + └────────────────────────────────┘ + │ + ▼ + ┌────────────────────────────────┐ + │ Try other node: │ + │ - Run update-setup on untried │ + │ node to discover pacemaker │ + └────────────────────────────────┘ + │ + ┌───────┴────────┐ + │ │ + Success Failure + │ │ + ▼ ▼ + ┌────────────────┐ ┌──────────────────┐ + │ CR populates │ │ Try next node │ + │ Exit discovery │ │ (next sync) │ + └────────────────┘ └──────────────────┘ +``` + +**Purpose:** Recover from scenarios where CR is missing or corrupted. + +## Node Replacement Scenario + +A common scenario where reconciliation is critical: + +```text +Initial State: + K8s: [master-0, master-1] + Pacemaker: [master-0, master-1] + +Step 1: Delete master-1 + K8s: [master-0] + Pacemaker: [master-0, master-1] ← DRIFT + + Reconciliation triggered: + - Remove master-1 from Pacemaker + + K8s: [master-0] + Pacemaker: [master-0] ← SYNCED + +Step 2: New master-1 boots and registers + K8s: [master-0, master-1] + Pacemaker: [master-0] ← DRIFT + + Reconciliation triggered: + - Add master-1 to Pacemaker + + K8s: [master-0, master-1] + Pacemaker: [master-0, master-1] ← SYNCED +``` + +## Integration with Health Monitoring + +Reconciliation and health monitoring work together: + +**Reconciliation:** +- Fixes drift (operational correctness) +- Triggered by node events + periodic sync +- Creates jobs and ConfigMaps + +**Health Monitoring:** +- Reports status (observability) +- Triggered by periodic sync only +- Updates operator conditions and records events + +Both operations run independently in `sync()` - a health check failure doesn't prevent reconciliation. diff --git a/pkg/operator/ceohelpers/common.go b/pkg/operator/ceohelpers/common.go index 027ca86ec5..dc7595269e 100644 --- a/pkg/operator/ceohelpers/common.go +++ b/pkg/operator/ceohelpers/common.go @@ -246,3 +246,18 @@ func CurrentRevision(status operatorv1.StaticPodOperatorStatus) (int32, error) { return latestRevision, nil } + +// IPAddressesEqual compares two IP addresses for equality. +// Handles both IPv4 and IPv6 addresses by parsing and comparing. +// Falls back to string comparison if either address fails to parse. +func IPAddressesEqual(ip1, ip2 string) bool { + parsed1 := net.ParseIP(ip1) + parsed2 := net.ParseIP(ip2) + + if parsed1 == nil || parsed2 == nil { + // If either fails to parse, do string comparison as fallback + return ip1 == ip2 + } + + return parsed1.Equal(parsed2) +} diff --git a/pkg/operator/ceohelpers/common_test.go b/pkg/operator/ceohelpers/common_test.go index b57bb74179..06ef723e35 100644 --- a/pkg/operator/ceohelpers/common_test.go +++ b/pkg/operator/ceohelpers/common_test.go @@ -228,3 +228,92 @@ func TestCurrentRevision(t *testing.T) { }) } } + +func TestIPAddressesEqual(t *testing.T) { + tests := []struct { + name string + ip1 string + ip2 string + expected bool + }{ + { + name: "IPv4 addresses equal", + ip1: "192.168.1.10", + ip2: "192.168.1.10", + expected: true, + }, + { + name: "IPv4 addresses not equal", + ip1: "192.168.1.10", + ip2: "192.168.1.11", + expected: false, + }, + { + name: "IPv6 addresses equal - same format", + ip1: "fd00::1", + ip2: "fd00::1", + expected: true, + }, + { + name: "IPv6 addresses equal - different format", + ip1: "fd00::1", + ip2: "fd00:0000:0000:0000:0000:0000:0000:0001", + expected: true, + }, + { + name: "IPv6 addresses not equal", + ip1: "fd00::1", + ip2: "fd00::2", + expected: false, + }, + { + name: "IPv6 addresses equal - expanded vs compressed", + ip1: "fd00:0000:0000:0000:0000:0000:0000:0002", + ip2: "fd00::2", + expected: true, + }, + { + name: "IPv4 vs IPv6 not equal", + ip1: "192.168.1.10", + ip2: "fd00::1", + expected: false, + }, + { + name: "empty strings equal", + ip1: "", + ip2: "", + expected: true, + }, + { + name: "empty vs non-empty not equal", + ip1: "", + ip2: "192.168.1.10", + expected: false, + }, + { + name: "invalid IP addresses - string comparison fallback (equal)", + ip1: "not-an-ip", + ip2: "not-an-ip", + expected: true, + }, + { + name: "invalid IP addresses - string comparison fallback (not equal)", + ip1: "not-an-ip", + ip2: "different-not-an-ip", + expected: false, + }, + { + name: "IPv4-mapped IPv6 addresses", + ip1: "::ffff:192.168.1.10", + ip2: "192.168.1.10", + expected: true, // IPv4-mapped IPv6 represents the same address + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := IPAddressesEqual(tt.ip1, tt.ip2) + require.Equal(t, tt.expected, result, "IPAddressesEqual(%q, %q) = %v, expected %v", tt.ip1, tt.ip2, result, tt.expected) + }) + } +} diff --git a/pkg/tnf/operator/lifecycle_helpers.go b/pkg/tnf/operator/lifecycle_helpers.go new file mode 100644 index 0000000000..5769b831b8 --- /dev/null +++ b/pkg/tnf/operator/lifecycle_helpers.go @@ -0,0 +1,67 @@ +package operator + +import ( + "fmt" + + corev1 "k8s.io/api/core/v1" + "k8s.io/klog/v2" + + pacmkrv1 "github.com/openshift/api/etcd/v1" + "github.com/openshift/cluster-etcd-operator/pkg/tnf/pkg/pacemaker" +) + +// getPacemakerNodes returns a map of node name -> IP from the PacemakerCluster CR. +func (c *PacemakerLifecycleManager) getPacemakerNodes() (map[string]string, error) { + nodes, _, err := c.getPacemakerNodesWithCR() + return nodes, err +} + +// getPacemakerNodesWithCR returns both the node map and the full CR from the PacemakerCluster CR. +// Returns (nodes, CR, error). Useful when caller needs to check CR metadata like LastUpdated. +func (c *PacemakerLifecycleManager) getPacemakerNodesWithCR() (map[string]string, *pacmkrv1.PacemakerCluster, error) { + // Check if informer exists + if c.pacemakerInformer == nil { + return nil, nil, fmt.Errorf("pacemakerInformer is nil") + } + + // Get PacemakerCluster CR from informer cache + // Note: We don't check HasSynced() here because the watch stream sometimes fails with decode errors + // even though the cache is populated via List. The cache will be refreshed on resync interval. + item, exists, err := c.pacemakerInformer.GetStore().GetByKey(pacemaker.PacemakerClusterResourceName) + if err != nil { + return nil, nil, fmt.Errorf("failed to get PacemakerCluster from cache: %w", err) + } + if !exists { + return nil, nil, fmt.Errorf("PacemakerCluster CR not found") + } + + pacemakerCR, ok := item.(*pacmkrv1.PacemakerCluster) + if !ok { + return nil, nil, fmt.Errorf("failed to convert to PacemakerCluster") + } + + if pacemakerCR.Status.Nodes == nil { + return nil, pacemakerCR, fmt.Errorf("PacemakerCluster CR has no nodes in status") + } + + // Build map of nodeName -> IP (use first address as primary) + pmNodes := make(map[string]string) + for _, node := range *pacemakerCR.Status.Nodes { + if len(node.Addresses) == 0 { + klog.Warningf("Pacemaker node %q has no addresses in CR status - skipping (possible CR population race)", node.NodeName) + continue + } + pmNodes[node.NodeName] = node.Addresses[0].Address + } + + return pmNodes, pacemakerCR, nil +} + +// getNodeNames extracts node names from a list of nodes. +func getNodeNames(nodes []*corev1.Node) []string { + names := make([]string, len(nodes)) + for i, node := range nodes { + names[i] = node.Name + } + return names +} diff --git a/pkg/tnf/operator/lifecycle_reconciliation.go b/pkg/tnf/operator/lifecycle_reconciliation.go new file mode 100644 index 0000000000..bebfc535ac --- /dev/null +++ b/pkg/tnf/operator/lifecycle_reconciliation.go @@ -0,0 +1,811 @@ +package operator + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "strconv" + "sync" + "time" + + "github.com/openshift/library-go/pkg/controller/controllercmd" + "github.com/openshift/library-go/pkg/operator/v1helpers" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" + + pacmkrv1 "github.com/openshift/api/etcd/v1" + "github.com/openshift/cluster-etcd-operator/pkg/operator/ceohelpers" + "github.com/openshift/cluster-etcd-operator/pkg/operator/operatorclient" + "github.com/openshift/cluster-etcd-operator/pkg/tnf/pkg/jobs" + "github.com/openshift/cluster-etcd-operator/pkg/tnf/pkg/tools" +) + +const ( + // pacemakerCRStalenessThreshold is how long before a PacemakerCluster CR status is considered stale. + // Status collector runs every minute, so 5 minutes means we've missed ~5 consecutive updates. + pacemakerCRStalenessThreshold = 5 * time.Minute + + // maxUpdateSetupConfigMaps is the maximum number of update-setup ConfigMaps to keep for debugging. + // Older ConfigMaps are cleaned up to prevent unbounded growth. + maxUpdateSetupConfigMaps = 5 +) + +var ( + // updateSetupGeneration orders update-setup ConfigMaps when events arrive close together (OCPBUGS-84695). + updateSetupGeneration int64 = 0 + + // reconcilePacemakerConfigMutex serializes ReconcilePacemakerConfig to prevent time-of-check-time-of-use + // races between drift detection and update-setup triggering. Multiple concurrent calls can detect + // drift simultaneously, but only one should proceed with reconciliation at a time. + reconcilePacemakerConfigMutex sync.Mutex + + // updateSetupFunc is a variable to allow mocking in tests + updateSetupFunc = updateSetup +) + +// ReconcilePacemakerConfig performs drift detection and reconciliation after external etcd transition completes. +// Compares K8s node state with pacemaker membership and triggers update-setup if drift is detected. +// Also handles orphaned jobs by starting JobController to reconcile operator conditions. +// This method is called: +// 1. Periodically from sync() (every 30s) +// 2. On node Add events +// 3. On node Update events (IP changes while Ready) +// 4. On node Delete events +// +// Returns nil if no action needed or reconciliation triggered successfully. +// +// Concurrency model: Multiple goroutines can enter this function concurrently and perform +// initial checks (informer sync, node readiness, drift detection). Once drift is detected, +// reconcilePacemakerConfigMutex serializes the check-and-trigger path to prevent time-of-check-time-of-use +// races where multiple goroutines would redundantly create ConfigMaps and trigger jobs. The second +// goroutine will either find no drift (first goroutine fixed it) or trigger a new reconciliation +// with a higher generation number. The generation counter ensures correctness (highest wins). +func (c *PacemakerLifecycleManager) ReconcilePacemakerConfig(ctx context.Context) error { + // Check if node informer has synced + if c.nodeInformer == nil || !c.nodeInformer.HasSynced() { + klog.V(4).Infof("Skipping drift reconciliation - node informer not synced yet") + return nil + } + + // Get K8s control plane nodes + k8sNodes, err := ceohelpers.ListNodesFromInformer(c.nodeInformer) + if err != nil { + return fmt.Errorf("failed to list control plane nodes: %w", err) + } + + // Validate node count (pacemaker only supports 2 nodes) + if len(k8sNodes) > 2 { + klog.Warningf("Found %d control plane nodes - pacemaker behavior undefined for >2 nodes, no reconciliation action taken", len(k8sNodes)) + return nil + } + + // Check transition status to determine readiness requirements + transitionComplete, err := ceohelpers.HasExternalEtcdCompletedTransition(ctx, c.operatorClient) + if err != nil { + return fmt.Errorf("failed to check external etcd transition status: %w", err) + } + + // Before transition: require all nodes Ready (stable bootstrap environment) + // After transition: allow NotReady nodes (this is exactly when we need drift reconciliation for node replacement) + if !transitionComplete { + for _, node := range k8sNodes { + if !tools.IsNodeReady(node) { + klog.V(4).Infof("Node %s is not Ready - skipping drift reconciliation during bootstrap (will retry in next sync)", node.Name) + return nil + } + } + } + + // Get pacemaker nodes from PacemakerCluster CR + pacemakerNodes, pacemakerCR, err := c.getPacemakerNodesWithCR() + pacemakerNodesAvailable := (err == nil) + if !pacemakerNodesAvailable { + klog.V(4).Infof("PacemakerCluster CR not available: %v - will attempt discovery reconciliation", err) + } else { + // Check if CR status is stale (not updated recently) + if isPacemakerCRStale(pacemakerCR) { + klog.Warningf("PacemakerCluster CR status is stale (last updated %v) - treating as unavailable for discovery reconciliation", pacemakerCR.Status.LastUpdated) + pacemakerNodesAvailable = false + pacemakerNodes = nil + } + } + + // Check for stopped update-setup job from previous run (handles perma-degraded/perma-progressing) + // This needs to run on every sync to catch jobs that stopped while operator was down + stoppedJob, err := c.getStoppedUpdateSetupJob(ctx) + if err != nil { + klog.Warningf("Failed to check for stopped update-setup job: %v", err) + // Don't return error - continue with drift detection + } + + // Detect drift (compares node names and IPs) + var hasDrift bool + if pacemakerNodesAvailable { + hasDrift = c.detectDrift(k8sNodes, pacemakerNodes) + } else { + // CR doesn't exist - treat as drift (unknown pacemaker state = needs reconciliation) + hasDrift = true + klog.Infof("PacemakerCluster CR not available - treating as drift for discovery reconciliation") + } + + // Determine if we need to take action + needsReconciliation := false + var reason string + + // Reason 1: Stopped unsuccessful job exists + if stoppedJob != nil && !jobs.IsComplete(*stoppedJob) { + needsReconciliation = true + reason = fmt.Sprintf("stopped unsuccessful job %s exists", stoppedJob.Name) + } + + // Reason 2: Drift detected + if hasDrift { + needsReconciliation = true + if reason == "" { + // Format node names for better readability + k8sNodeNames := getNodeNames(k8sNodes) + pmNodeNames := make([]string, 0, len(pacemakerNodes)) + for nodeName := range pacemakerNodes { + pmNodeNames = append(pmNodeNames, nodeName) + } + reason = fmt.Sprintf("drift detected between K8s %v and pacemaker %v", k8sNodeNames, pmNodeNames) + } else { + reason = fmt.Sprintf("%s AND drift detected", reason) + } + } + + // If no action needed, return early + if !needsReconciliation { + klog.V(4).Infof("No reconciliation needed - no drift and no stopped jobs") + return nil + } + + klog.Infof("Reconciliation needed: %s", reason) + + // Serialize reconciliation trigger to prevent time-of-check-time-of-use race between drift detection + // and update-setup start. Multiple concurrent callers can detect drift, but only one should check-and-trigger at a time. + reconcilePacemakerConfigMutex.Lock() + defer reconcilePacemakerConfigMutex.Unlock() + + // Re-fetch CR state after acquiring lock to use current state (CR may have been updated by status collector) + pacemakerNodes, pacemakerCR, err = c.getPacemakerNodesWithCR() + pacemakerNodesAvailable = (err == nil) + if !pacemakerNodesAvailable { + klog.V(4).Infof("PacemakerCluster CR not available after lock: %v - will attempt discovery reconciliation", err) + } else { + if isPacemakerCRStale(pacemakerCR) { + klog.Warningf("PacemakerCluster CR status is stale after lock (last updated %v) - treating as unavailable for discovery reconciliation", pacemakerCR.Status.LastUpdated) + pacemakerNodesAvailable = false + pacemakerNodes = nil + } + } + + // Re-check drift and stopped job after acquiring lock (another goroutine may have changed state) + stoppedJob, err = c.getStoppedUpdateSetupJob(ctx) + if err != nil { + klog.Warningf("Failed to re-check for stopped update-setup job: %v", err) + } + hasDrift = c.detectDrift(k8sNodes, pacemakerNodes) + + // Decision tree: + // 1. If drift exists → ensure ConfigMap exists and matches desired state, start controller if needed + // 2. If stopped unsuccessful job but NO drift → delete job (cluster is correct, stale failure) + // 3. Otherwise → nothing to do + + if hasDrift { + // Continue to ConfigMap creation and controller start below... + } else if stoppedJob != nil && !jobs.IsComplete(*stoppedJob) { + // No drift, but stopped unsuccessful job exists - cluster is correct, just delete the stale job + klog.Infof("No drift detected, but stopped job %s exists - deleting stale job to clear conditions", stoppedJob.Name) + if err := jobs.DeleteAndWait(ctx, c.kubeClient, stoppedJob.Name, operatorclient.TargetNamespace); err != nil { + return fmt.Errorf("failed to delete stopped job: %w", err) + } + klog.Infof("Successfully deleted stopped job %s - JobController will clear conditions", stoppedJob.Name) + return nil + } else { + klog.V(4).Infof("No action needed after acquiring lock - another goroutine may have handled it") + return nil + } + + // Calculate valid target nodes for update-setup job + var validTargetNodes []*corev1.Node + if pacemakerNodesAvailable { + // CR exists - use intersection: nodes that exist in BOTH K8s and pacemaker + intersection := c.getIntersection(k8sNodes, pacemakerNodes) + if len(intersection) == 0 { + return fmt.Errorf("no nodes in both K8s and pacemaker - manual intervention may be required") + } + validTargetNodes = intersection + klog.Infof("Found %d nodes in intersection (K8s ∩ pacemaker): %v", len(validTargetNodes), getNodeNames(validTargetNodes)) + } else { + // CR unavailable/stale - try nodes sequentially to discover which has pacemaker running + // Use stopped job to determine which node we already tried (to avoid retrying same node) + validTargetNodes = c.selectDiscoveryTargetNodes(k8sNodes, stoppedJob) + if len(validTargetNodes) == 0 { + return fmt.Errorf("no valid nodes for discovery - all nodes have been tried and failed") + } + klog.Infof("Discovery mode (no actionable CR): will try nodes %v", getNodeNames(validTargetNodes)) + } + + // Create function that calculates valid target nodes + // Job controller will call this before each attempt (to get fresh node state) + validNodeFunc := func() ([]*corev1.Node, error) { + return c.getActivePacemakerNodes() + } + + // Call update-setup with valid target nodes (creates/updates ConfigMap and ensures controller is running) + // Use updateSetupFunc to allow mocking in tests + return updateSetupFunc( + validTargetNodes, + validNodeFunc, + k8sNodes, + pacemakerNodes, + ctx, + c.controllerContext, + c.operatorClient, + c.kubeClient, + c.kubeInformersForNamespaces, + ) +} + +// detectDrift compares K8s nodes with pacemaker nodes and returns true if drift exists. +// Checks both node names and IPs (supports IPv4 and IPv6). +func (c *PacemakerLifecycleManager) detectDrift(k8sNodes []*corev1.Node, pacemakerNodes map[string]string) bool { + // Check count mismatch + if len(k8sNodes) != len(pacemakerNodes) { + return true + } + + // Check each K8s node exists in pacemaker with matching IP + for _, k8sNode := range k8sNodes { + k8sIP, err := tools.GetNodeIPForPacemaker(*k8sNode) + if err != nil { + klog.Warningf("Failed to get IP for K8s node %s: %v", k8sNode.Name, err) + continue + } + + pmIP, exists := pacemakerNodes[k8sNode.Name] + if !exists { + return true + } + + if !ceohelpers.IPAddressesEqual(k8sIP, pmIP) { + return true + } + } + + // Check for nodes in pacemaker but not in K8s + for pmNodeName := range pacemakerNodes { + found := false + for _, k8sNode := range k8sNodes { + if k8sNode.Name == pmNodeName { + found = true + break + } + } + if !found { + return true + } + } + + return false +} + +// getActivePacemakerNodes returns the active pacemaker nodes for targeting jobs and CronJobs. +// Uses intersection logic: nodes that exist in BOTH K8s (ready) and pacemaker. +// If PacemakerCluster CR doesn't exist or is stale, returns all ready control plane nodes. +// This is the shared logic used by both update-setup job and status collector CronJob. +func (c *PacemakerLifecycleManager) getActivePacemakerNodes() ([]*corev1.Node, error) { + // Check if node informer has synced + if c.nodeInformer == nil || !c.nodeInformer.HasSynced() { + return nil, fmt.Errorf("node informer not synced yet") + } + + // Get K8s control plane nodes + k8sNodes, err := ceohelpers.ListNodesFromInformer(c.nodeInformer) + if err != nil { + return nil, fmt.Errorf("failed to list control plane nodes: %w", err) + } + + // Filter to Ready nodes only + readyNodes := []*corev1.Node{} + for _, node := range k8sNodes { + if tools.IsNodeReady(node) { + readyNodes = append(readyNodes, node) + } + } + + if len(readyNodes) == 0 { + return nil, fmt.Errorf("no ready control plane nodes found") + } + + // Try to get pacemaker nodes from CR + pacemakerNodes, pacemakerCR, err := c.getPacemakerNodesWithCR() + pacemakerNodesAvailable := (err == nil && !isPacemakerCRStale(pacemakerCR)) + + if pacemakerNodesAvailable { + // CR exists and is fresh - use intersection logic (K8s ∩ pacemaker) + intersection := c.getIntersection(readyNodes, pacemakerNodes) + if len(intersection) > 0 { + klog.V(2).Infof("Valid targets (intersection, ready only): %v", getNodeNames(intersection)) + return intersection, nil + } + // No intersection - fall through to using all ready nodes + klog.Warningf("No nodes in intersection (K8s ∩ pacemaker) - using all ready nodes") + } else { + if err != nil { + klog.V(4).Infof("PacemakerCluster CR not available: %v - using all ready nodes", err) + } else { + klog.Warningf("PacemakerCluster CR status is stale (last updated %v) - using all ready nodes", pacemakerCR.Status.LastUpdated) + } + } + + // CR doesn't exist, is stale, or no intersection - use all ready nodes + klog.V(2).Infof("Valid targets (all ready nodes): %v", getNodeNames(readyNodes)) + return readyNodes, nil +} + +// getIntersection returns nodes that exist in BOTH K8s and pacemaker. +// Used to determine valid target nodes for update-setup operations. +// Returns nodes sorted by name for deterministic ordering. +func (c *PacemakerLifecycleManager) getIntersection(k8sNodes []*corev1.Node, pacemakerNodes map[string]string) []*corev1.Node { + intersection := []*corev1.Node{} + for _, k8sNode := range k8sNodes { + if _, exists := pacemakerNodes[k8sNode.Name]; exists { + intersection = append(intersection, k8sNode) + } + } + // Sort by name for deterministic target selection + sort.Slice(intersection, func(i, j int) bool { + return intersection[i].Name < intersection[j].Name + }) + return intersection +} + +// isPacemakerCRStale checks if the PacemakerCluster CR status is stale (hasn't been updated recently). +// A stale CR indicates the status collector isn't running or pacemaker isn't responding. +func isPacemakerCRStale(cr *pacmkrv1.PacemakerCluster) bool { + if cr == nil { + return true + } + + timeSinceUpdate := time.Since(cr.Status.LastUpdated.Time) + return timeSinceUpdate > pacemakerCRStalenessThreshold +} + +// selectDiscoveryTargetNodes selects which nodes to try when PacemakerCluster CR is unavailable or stale. +// Used during initial cluster setup OR when status collector isn't updating the CR (pacemaker issue). +// Strategy: Try nodes sequentially, excluding nodes where previous attempts failed. +// - If no stopped job exists, return all nodes sorted by name +// - If a stopped unsuccessful job exists, exclude that node and return remaining sorted nodes +// - Returns empty list if all nodes have been tried and failed +func (c *PacemakerLifecycleManager) selectDiscoveryTargetNodes(k8sNodes []*corev1.Node, stoppedJob *batchv1.Job) []*corev1.Node { + if len(k8sNodes) == 0 { + return []*corev1.Node{} + } + + // Sort nodes by name for deterministic ordering + sortedNodes := make([]*corev1.Node, len(k8sNodes)) + copy(sortedNodes, k8sNodes) + sort.Slice(sortedNodes, func(i, j int) bool { + return sortedNodes[i].Name < sortedNodes[j].Name + }) + + // If no stopped job, return all nodes + if stoppedJob == nil { + klog.V(4).Infof("No previous job attempt - will try all nodes starting with: %s", sortedNodes[0].Name) + return sortedNodes + } + + // If stopped job succeeded, we're done (shouldn't reach here, but handle gracefully) + if jobs.IsComplete(*stoppedJob) { + klog.V(4).Infof("Previous job succeeded - will try all nodes starting with: %s", sortedNodes[0].Name) + return sortedNodes + } + + // Stopped job failed - find which node it ran on and exclude it + failedNodeName := stoppedJob.Spec.Template.Spec.NodeName + if failedNodeName == "" { + klog.Warningf("Stopped job %s has no NodeName - cannot determine which node failed, will try all nodes", stoppedJob.Name) + return sortedNodes + } + + klog.Infof("Previous job failed on node %s - excluding from valid targets", failedNodeName) + + // Build list of nodes excluding the failed one + remainingNodes := []*corev1.Node{} + for _, node := range sortedNodes { + if node.Name != failedNodeName { + remainingNodes = append(remainingNodes, node) + } + } + + if len(remainingNodes) == 0 { + klog.Warningf("All %d nodes have been tried - no remaining nodes to attempt", len(k8sNodes)) + return []*corev1.Node{} + } + + klog.Infof("Will try %d remaining node(s) starting with: %s", len(remainingNodes), remainingNodes[0].Name) + return remainingNodes +} + +// isUpdateSetupRunning checks if any update-setup job is currently running. +func (c *PacemakerLifecycleManager) isUpdateSetupRunning(ctx context.Context) (bool, error) { + // List all update-setup jobs by name label + jobList, err := c.kubeClient.BatchV1().Jobs(operatorclient.TargetNamespace).List(ctx, v1.ListOptions{ + LabelSelector: "app.kubernetes.io/name=tnf-update-setup-job", + }) + if err != nil { + return false, fmt.Errorf("failed to list update-setup jobs: %w", err) + } + + // Check if any job is still running (not Complete and not Failed) + for _, job := range jobList.Items { + if !jobs.IsStopped(job) { + klog.V(4).Infof("Update-setup job %s is still running", job.Name) + return true, nil + } + } + + return false, nil +} + +// getStoppedUpdateSetupJob returns the stopped update-setup job if one exists, nil otherwise. +// A stopped job is one that has completed (successfully or failed). +func (c *PacemakerLifecycleManager) getStoppedUpdateSetupJob(ctx context.Context) (*batchv1.Job, error) { + jobName := tools.JobTypeUpdateSetup.GetJobName(nil) + job, err := c.kubeClient.BatchV1().Jobs(operatorclient.TargetNamespace).Get(ctx, jobName, v1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return nil, nil + } + return nil, fmt.Errorf("failed to get update-setup job: %w", err) + } + + if jobs.IsStopped(*job) { + return job, nil + } + + return nil, nil +} + +// getNextUpdateSetupGeneration increments and returns the next generation counter. +// Caller must hold reconcilePacemakerConfigMutex. +func getNextUpdateSetupGeneration() int64 { + updateSetupGeneration++ + return updateSetupGeneration +} + +func getCurrentUpdateSetupGeneration() int64 { + return updateSetupGeneration +} + +// initUpdateSetupGeneration scans existing update-setup ConfigMaps and initializes the generation +// counter to max(existing)+1 to prevent reusing stale ConfigMaps after operator restart. +// Must be called once during lifecycle manager initialization before any reconciliation loops run. +func (c *PacemakerLifecycleManager) initUpdateSetupGeneration(ctx context.Context) error { + cmList, err := c.kubeClient.CoreV1().ConfigMaps(operatorclient.TargetNamespace).List(ctx, v1.ListOptions{ + LabelSelector: "app.kubernetes.io/component=" + tools.TnfUpdateSetupComponentValue, + }) + if err != nil { + return fmt.Errorf("failed to list existing update-setup ConfigMaps: %w", err) + } + + var maxGen int64 = 0 + for _, cm := range cmList.Items { + genStr := cm.Data["generation"] + if genStr == "" { + continue + } + gen, err := strconv.ParseInt(genStr, 10, 64) + if err != nil { + klog.Warningf("Found update-setup ConfigMap %s with invalid generation %q: %v", cm.Name, genStr, err) + continue + } + if gen > maxGen { + maxGen = gen + } + } + + updateSetupGeneration = maxGen + if maxGen > 0 { + klog.Infof("Initialized update-setup generation counter to %d (found %d existing ConfigMaps)", maxGen, len(cmList.Items)) + } else { + klog.V(2).Infof("No existing update-setup ConfigMaps found, starting generation counter at 0") + } + return nil +} + +// updateSetup writes a snapshot ConfigMap, runs auth on all nodes, tries update-setup on valid targets, then after-setup on all. +// validTargetNodes: nodes that can run the update-setup job (initial list for logging) +// validNodeFunc: function that calculates fresh list of valid nodes on each retry attempt +// allK8sNodes: all K8s nodes for the ConfigMap snapshot +// pacemakerNodes: current pacemaker membership (name -> IP) from PacemakerCluster CR +func updateSetup( + validTargetNodes []*corev1.Node, + validNodeFunc jobs.TargetNodesFunc, + allK8sNodes []*corev1.Node, + pacemakerNodes map[string]string, + ctx context.Context, + controllerContext *controllercmd.ControllerContext, + operatorClient v1helpers.StaticPodOperatorClient, + kubeClient kubernetes.Interface, + kubeInformersForNamespaces v1helpers.KubeInformersForNamespaces, +) error { + + if len(validTargetNodes) == 0 { + return fmt.Errorf("no valid target nodes for update-setup - manual intervention may be required") + } + + // Encode desired state for comparison + nodeListData, err := encodeNodeList(allK8sNodes) + if err != nil { + return fmt.Errorf("failed to encode node list: %w", err) + } + + // Check if current generation matches desired state - if so, reuse it instead of creating new one + currentGeneration := getCurrentUpdateSetupGeneration() + var generation int64 + var shouldCreateNewConfigMap bool + + if currentGeneration > 0 { + currentCMName := fmt.Sprintf("tnf-update-setup-%d", currentGeneration) + currentCM, err := kubeClient.CoreV1().ConfigMaps(operatorclient.TargetNamespace).Get(ctx, currentCMName, v1.GetOptions{}) + if err == nil { + // Compare desired state with current ConfigMap (only check "nodes" - the desired end state) + if currentCM.Data["nodes"] == nodeListData { + klog.Infof("Desired state matches current generation %d - reusing existing ConfigMap", currentGeneration) + generation = currentGeneration + shouldCreateNewConfigMap = false + } else { + klog.Infof("Desired state differs from generation %d - creating new generation", currentGeneration) + generation = getNextUpdateSetupGeneration() + shouldCreateNewConfigMap = true + } + } else { + // Current ConfigMap doesn't exist (deleted or error) - create new + klog.V(2).Infof("Current generation %d ConfigMap not found: %v - creating new generation", currentGeneration, err) + generation = getNextUpdateSetupGeneration() + shouldCreateNewConfigMap = true + } + } else { + // First generation + generation = getNextUpdateSetupGeneration() + shouldCreateNewConfigMap = true + } + + klog.Infof("Generation %d: Will try nodes %v (desired state: %v)", + generation, getNodeNames(validTargetNodes), getNodeNames(allK8sNodes)) + + cmName := fmt.Sprintf("tnf-update-setup-%d", generation) + + // Only create ConfigMap if desired state differs from current generation + if shouldCreateNewConfigMap { + cmData := map[string]string{ + "nodes": nodeListData, // Desired state: which K8s nodes should be in pacemaker + "generation": fmt.Sprintf("%d", generation), + "timestamp": time.Now().Format(time.RFC3339), + } + cm := &corev1.ConfigMap{ + ObjectMeta: v1.ObjectMeta{ + Name: cmName, + Namespace: operatorclient.TargetNamespace, + Labels: map[string]string{ + "app.kubernetes.io/component": tools.TnfUpdateSetupComponentValue, + "tnf.etcd.openshift.io/generation": fmt.Sprintf("%d", generation), + }, + }, + Data: cmData, + } + + _, err = kubeClient.CoreV1().ConfigMaps(operatorclient.TargetNamespace).Create(ctx, cm, v1.CreateOptions{}) + if err != nil { + return fmt.Errorf("failed to create ConfigMap %s: %w", cmName, err) + } + klog.Infof("Created new ConfigMap %s for generation %d", cmName, generation) + + // Clean up old ConfigMaps (keep 5 most recent) + if err := cleanupOldUpdateSetupConfigMaps(ctx, kubeClient, generation); err != nil { + klog.Warningf("Failed to cleanup old update-setup ConfigMaps: %v", err) + } + } + + // When reusing ConfigMap (desired state unchanged), check if job already exists and is working. + // Skip restarting if job is still running OR completed successfully - pacemaker just needs time to converge. + // This prevents redundant job creation when: + // 1. First job completes and updates pacemaker + // 2. Status collector updates CR with partial pacemaker state + // 3. Reconciliation triggered again, detects drift (pacemaker not fully converged yet) + // 4. Desired state matches current generation → would restart job unnecessarily + if !shouldCreateNewConfigMap { + jobName := tools.JobTypeUpdateSetup.GetJobName(nil) + existingJob, err := kubeClient.BatchV1().Jobs(operatorclient.TargetNamespace).Get(ctx, jobName, v1.GetOptions{}) + if err == nil { + // Job exists - check state + if !jobs.IsStopped(*existingJob) { + // Job still running - skip restart to avoid interrupting in-progress work + klog.Infof("Reusing generation %d - job still running, skipping restart (waiting for job completion)", generation) + return nil + } else if jobs.IsComplete(*existingJob) { + // Job completed successfully - skip restart, pacemaker convergence in progress + klog.Infof("Reusing generation %d - job already completed successfully, skipping restart (waiting for pacemaker convergence)", generation) + return nil + } + // Job stopped but not complete (failed) - fall through to restart + klog.Infof("Reusing generation %d - existing job failed, restarting", generation) + } else if !apierrors.IsNotFound(err) { + // Error checking job (not NotFound) - log warning and fall through to restart + klog.Warningf("Failed to check existing job for generation %d: %v - will restart", generation, err) + } else { + // Job doesn't exist - fall through to start + klog.Infof("Reusing generation %d - no existing job found, starting", generation) + } + } + + // Restart update-setup job with new information (deletes existing job immediately, starts controller) + // Controller will manage job lifecycle via sync loop and retry across valid nodes + // Note: Auth and after-setup jobs are managed by separate background controllers (lifecycle_job_controllers.go) + klog.Infof("Starting update-setup job controller for generation %d with %d initial valid target(s): %v", generation, len(validTargetNodes), getNodeNames(validTargetNodes)) + + // Create jobConfigFunc that captures config for drift detection + jobConfigFunc := func() (string, error) { + nodes, err := validNodeFunc() + if err != nil { + return "", err + } + nodeUIDs := make([]string, len(nodes)) + for i, node := range nodes { + nodeUIDs[i] = string(node.UID) + } + sort.Strings(nodeUIDs) // Sort for stable comparison + + config := map[string]interface{}{ + "nodeUIDs": nodeUIDs, + "configMapGeneration": generation, + } + configJSON, err := json.Marshal(config) + if err != nil { + return "", fmt.Errorf("failed to marshal job config: %w", err) + } + return string(configJSON), nil + } + + const retries = 3 // Multi-node: try all valid nodes 3 times before degrading (backoffLimit=0) + timeout := getJobTimeout(tools.JobTypeUpdateSetup) + if err := jobs.RestartJobOrRunController(ctx, tools.JobTypeUpdateSetup, nil, validNodeFunc, jobConfigFunc, retries, + controllerContext, operatorClient, kubeClient, kubeInformersForNamespaces, + jobs.DefaultConditions, timeout); err != nil { + return fmt.Errorf("failed to restart update-setup job controller: %w", err) + } + + return nil +} + +// getJobTimeout returns the appropriate timeout for a given job type. +func getJobTimeout(jobType tools.JobType) time.Duration { + switch jobType { + case tools.JobTypeAuth: + return tools.AuthJobCompletedTimeout + case tools.JobTypeUpdateSetup: + return tools.SetupJobCompletedTimeout + case tools.JobTypeAfterSetup: + return tools.AfterSetupJobCompletedTimeout + default: + return tools.AllCompletedTimeout + } +} + +// encodeNodeList encodes a list of nodes to JSON. +// Only includes name and IP to avoid unnecessary ConfigMap recreation when labels/addresses change. +func encodeNodeList(nodes []*corev1.Node) (string, error) { + type nodeInfo struct { + Name string `json:"name"` + IP string `json:"ip"` + } + + infos := make([]nodeInfo, 0, len(nodes)) + for _, node := range nodes { + ip, err := tools.GetNodeIPForPacemaker(*node) + if err != nil { + klog.Warningf("Failed to get IP for node %s: %v - skipping from ConfigMap", node.Name, err) + continue + } + infos = append(infos, nodeInfo{ + Name: node.Name, + IP: ip, + }) + } + + // Sort by name to ensure stable ConfigMap comparison (avoid order-sensitive duplicates) + sort.Slice(infos, func(i, j int) bool { + return infos[i].Name < infos[j].Name + }) + + data, err := json.Marshal(infos) + if err != nil { + return "", err + } + return string(data), nil +} + +// encodeStringList encodes a list of strings to JSON. +func encodeStringList(list []string) string { + if len(list) == 0 { + return "[]" + } + data, err := json.Marshal(list) + if err != nil { + // Should never happen for []string + klog.Errorf("failed to encode string list: %v", err) + return "[]" + } + return string(data) +} + +// cleanupOldUpdateSetupConfigMaps deletes old update-setup ConfigMaps, keeping the 5 most recent for debugging. +// This prevents ConfigMap accumulation while preserving recent history for comparison and troubleshooting. +func cleanupOldUpdateSetupConfigMaps(ctx context.Context, kubeClient kubernetes.Interface, currentGeneration int64) error { + cmList, err := kubeClient.CoreV1().ConfigMaps(operatorclient.TargetNamespace).List(ctx, v1.ListOptions{ + LabelSelector: "app.kubernetes.io/component=" + tools.TnfUpdateSetupComponentValue, + }) + if err != nil { + return fmt.Errorf("failed to list update-setup ConfigMaps: %w", err) + } + + if len(cmList.Items) <= maxUpdateSetupConfigMaps { + klog.V(4).Infof("Found %d update-setup ConfigMaps (limit: %d) - no cleanup needed", len(cmList.Items), maxUpdateSetupConfigMaps) + return nil + } + + // Parse generations and sort by generation number (descending) + type cmWithGen struct { + cm corev1.ConfigMap + gen int64 + } + var cms []cmWithGen + for _, cm := range cmList.Items { + genStr := cm.Data["generation"] + if genStr == "" { + klog.Warningf("ConfigMap %s has no generation field, skipping cleanup", cm.Name) + continue + } + gen, err := strconv.ParseInt(genStr, 10, 64) + if err != nil { + klog.Warningf("ConfigMap %s has invalid generation %q: %v, skipping cleanup", cm.Name, genStr, err) + continue + } + cms = append(cms, cmWithGen{cm: cm, gen: gen}) + } + + // Sort by generation (newest first) + sort.Slice(cms, func(i, j int) bool { + return cms[i].gen > cms[j].gen + }) + + // Delete ConfigMaps beyond the limit + deletedCount := 0 + for i := maxUpdateSetupConfigMaps; i < len(cms); i++ { + cm := cms[i].cm + klog.V(2).Infof("Deleting old update-setup ConfigMap %s (generation %d, keeping %d most recent)", cm.Name, cms[i].gen, maxUpdateSetupConfigMaps) + if err := kubeClient.CoreV1().ConfigMaps(operatorclient.TargetNamespace).Delete(ctx, cm.Name, v1.DeleteOptions{}); err != nil { + if !apierrors.IsNotFound(err) { + klog.Warningf("Failed to delete old ConfigMap %s: %v", cm.Name, err) + } + } else { + deletedCount++ + } + } + + if deletedCount > 0 { + klog.Infof("Cleaned up %d old update-setup ConfigMaps (keeping %d most recent)", deletedCount, maxUpdateSetupConfigMaps) + } else { + klog.V(4).Infof("No old update-setup ConfigMaps to clean up") + } + + return nil +} diff --git a/pkg/tnf/operator/lifecycle_reconciliation_test.go b/pkg/tnf/operator/lifecycle_reconciliation_test.go new file mode 100644 index 0000000000..c4ba4372b4 --- /dev/null +++ b/pkg/tnf/operator/lifecycle_reconciliation_test.go @@ -0,0 +1,137 @@ +package operator + +/* +TEST COVERAGE SUMMARY - lifecycle_reconciliation_test.go +========================================================= + +This file tests drift detection and reconciliation decision logic for Two-Node Fencing clusters. + +WHAT'S TESTED +------------- + +Drift Detection and Reconciliation: +└── TestDetermineReconciliationActions - Reconciliation decision logic (single source of truth) + ├── ✅ No drift (both nodes match name+IP) + ├── ✅ Node deleted from K8s (remove from pacemaker) + ├── ✅ Node added to K8s (add to pacemaker) + ├── ✅ Node replaced (different name, remove old + add new) + ├── ✅ Node IP changed (same name, remove + re-add) + ├── ✅ Pacemaker empty (add all K8s nodes) + └── ✅ K8s empty (remove all pacemaker nodes) +*/ + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/openshift/cluster-etcd-operator/pkg/tnf/pkg/tools" +) + +// TestDetermineReconciliationActions tests the reconciliation action decision logic. +// This is the single source of truth for reconciliation decisions. +func TestDetermineReconciliationActions(t *testing.T) { + tests := []struct { + name string + k8sNodes map[string]string // name -> IP + pacemakerNodes map[string]string // name -> IP + expectRemove []string + expectAdd []string + }{ + { + name: "no drift - both nodes match (name+IP)", + k8sNodes: map[string]string{ + "master-0": "192.168.1.10", + "master-1": "192.168.1.11", + }, + pacemakerNodes: map[string]string{ + "master-0": "192.168.1.10", + "master-1": "192.168.1.11", + }, + expectRemove: nil, + expectAdd: nil, + }, + { + name: "node deleted from k8s - remove from pacemaker", + k8sNodes: map[string]string{ + "master-0": "192.168.1.10", + }, + pacemakerNodes: map[string]string{ + "master-0": "192.168.1.10", + "master-1": "192.168.1.11", + }, + expectRemove: []string{"master-1"}, + expectAdd: nil, + }, + { + name: "node added to k8s - add to pacemaker", + k8sNodes: map[string]string{ + "master-0": "192.168.1.10", + "master-1": "192.168.1.11", + }, + pacemakerNodes: map[string]string{ + "master-0": "192.168.1.10", + }, + expectRemove: nil, + expectAdd: []string{"master-1"}, + }, + { + name: "node replaced (different name) - remove old, add new", + k8sNodes: map[string]string{ + "master-0": "192.168.1.10", + "new-master-1": "192.168.1.20", + }, + pacemakerNodes: map[string]string{ + "master-0": "192.168.1.10", + "old-master-1": "192.168.1.11", + }, + expectRemove: []string{"old-master-1"}, + expectAdd: []string{"new-master-1"}, + }, + { + name: "node IP changed (same name) - remove old, add new", + k8sNodes: map[string]string{ + "master-0": "192.168.1.10", + "master-1": "192.168.1.20", // IP changed + }, + pacemakerNodes: map[string]string{ + "master-0": "192.168.1.10", + "master-1": "192.168.1.11", // Old IP + }, + expectRemove: []string{"master-1"}, + expectAdd: []string{"master-1"}, + }, + { + name: "pacemaker empty - add all k8s nodes", + k8sNodes: map[string]string{ + "master-0": "192.168.1.10", + "master-1": "192.168.1.11", + }, + pacemakerNodes: map[string]string{}, + expectRemove: nil, + expectAdd: []string{"master-0", "master-1"}, + }, + { + name: "k8s empty - remove all pacemaker nodes", + k8sNodes: map[string]string{}, + pacemakerNodes: map[string]string{ + "master-0": "192.168.1.10", + "master-1": "192.168.1.11", + }, + expectRemove: []string{"master-0", "master-1"}, + expectAdd: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + nodesToRemove, nodesToAdd := tools.DetermineReconciliationActions(tt.k8sNodes, tt.pacemakerNodes) + + // Use ElementsMatch for order-independent comparison + require.ElementsMatch(t, tt.expectRemove, nodesToRemove, + "Expected to remove %v but got %v", tt.expectRemove, nodesToRemove) + require.ElementsMatch(t, tt.expectAdd, nodesToAdd, + "Expected to add %v but got %v", tt.expectAdd, nodesToAdd) + }) + } +} diff --git a/pkg/tnf/operator/starter.go b/pkg/tnf/operator/starter.go index 12e18430dc..1ef13823b2 100644 --- a/pkg/tnf/operator/starter.go +++ b/pkg/tnf/operator/starter.go @@ -3,26 +3,26 @@ package operator import ( "bytes" "context" + "encoding/json" "fmt" "os" + "sort" "time" - operatorv1 "github.com/openshift/api/operator/v1" configv1informers "github.com/openshift/client-go/config/informers/externalversions/config/v1" operatorv1informers "github.com/openshift/client-go/operator/informers/externalversions/operator/v1" "github.com/openshift/library-go/pkg/controller/controllercmd" "github.com/openshift/library-go/pkg/operator/resource/resourceapply" "github.com/openshift/library-go/pkg/operator/staticresourcecontroller" "github.com/openshift/library-go/pkg/operator/v1helpers" - batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" - corev1listers "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/cache" "k8s.io/klog/v2" @@ -32,15 +32,9 @@ import ( "github.com/openshift/cluster-etcd-operator/pkg/operator/externaletcdsupportcontroller" "github.com/openshift/cluster-etcd-operator/pkg/operator/operatorclient" "github.com/openshift/cluster-etcd-operator/pkg/tnf/pkg/jobs" - "github.com/openshift/cluster-etcd-operator/pkg/tnf/pkg/pacemaker" "github.com/openshift/cluster-etcd-operator/pkg/tnf/pkg/tools" ) -const ( - // pacemakerStatusCollectorName is the name of the Pacemaker status collector CronJob - pacemakerStatusCollectorName = "pacemaker-status-collector" -) - // HandleDualReplicaClusters checks feature gate and control plane topology, // and handles dual replica aka two node fencing clusters func HandleDualReplicaClusters( @@ -77,66 +71,12 @@ func HandleDualReplicaClusters( if err := runTnfResourceController(ctx, controllerContext, kubeClient, dynamicClient, operatorClient, kubeInformersForNamespaces); err != nil { return false, err } - runPacemakerControllers(ctx, controllerContext, operatorClient, kubeClient, etcdInformer) - // we need node names for assigning auth and after-setup jobs to specific nodes - controlPlaneNodeLister := corev1listers.NewNodeLister(controlPlaneNodeInformer.GetIndexer()) - klog.Infof("watching for nodes...") - _, err = controlPlaneNodeInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: func(obj any) { - node, ok := obj.(*corev1.Node) - if !ok { - klog.Warningf("failed to convert added object to Node %+v", obj) - return - } - - // ignore nodes which are not ready yet - if !tools.IsNodeReady(node) { - klog.Infof("added node %s is not ready yet, skipping handling", node.GetName()) - return - } - - // this potentially needs some time when we wait for etcd bootstrap to complete, so run it in a goroutine, - // to not block the event handler - klog.Infof("node added and ready: %s", node.GetName()) - go handleNodesWithRetry(controllerContext, controlPlaneNodeLister, ctx, operatorClient, kubeClient, kubeInformersForNamespaces, etcdInformer) - }, - UpdateFunc: func(oldObj, newObj any) { - oldNode, oldOk := oldObj.(*corev1.Node) - newNode, newOk := newObj.(*corev1.Node) - if !oldOk || !newOk { - klog.Warningf("failed to convert updated object to Node, old=%+v, new=%+v", oldObj, newObj) - return - } - - // only handle if node transitioned from not ready to ready - oldReady := tools.IsNodeReady(oldNode) - newReady := tools.IsNodeReady(newNode) - if !oldReady && newReady { - klog.Infof("node %s transitioned to ready state", newNode.GetName()) - // this potentially needs some time when we wait for etcd bootstrap to complete, so run it in a goroutine, - // to not block the event handler - go handleNodesWithRetry(controllerContext, controlPlaneNodeLister, ctx, operatorClient, kubeClient, kubeInformersForNamespaces, etcdInformer) - } - }, - DeleteFunc: func(obj any) { - node, ok := obj.(*corev1.Node) - if !ok { - klog.Warningf("failed to convert deleted object to Node %+v", obj) - return - } - klog.Infof("node deleted: %s", node.GetName()) - - // always handle node deletion - // this potentially needs some time when we wait for etcd bootstrap to complete, so run it in a goroutine, - // to not block the event handler - go handleNodesWithRetry(controllerContext, controlPlaneNodeLister, ctx, operatorClient, kubeClient, kubeInformersForNamespaces, etcdInformer) - }, - }) - if err != nil { - klog.Errorf("failed to add eventhandler to control plane informer: %v", err) - return false, err - } + // Start pacemaker controllers (lifecycle manager, status collector) + // PacemakerLifecycleManager handles ALL node lifecycle events: + // - UpdateFunc: Ready transitions for initial bootstrap + // - AddFunc/DeleteFunc: drift-driven reconciliation + runPacemakerControllers(ctx, controllerContext, operatorClient, kubeClient, kubeInformersForNamespaces, etcdInformer, controlPlaneNodeInformer) // we need to update fencing config when fencing secrets change // adding a secret informer to the jobcontroller would trigger fencing setup for *every* secret change, @@ -144,10 +84,10 @@ func HandleDualReplicaClusters( klog.Infof("watching for secrets...") _, err = kubeInformersForNamespaces.InformersFor(operatorclient.TargetNamespace).Core().V1().Secrets().Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj any) { - go handleFencingSecretChange(ctx, nil, obj, controllerContext, operatorClient, kubeClient, kubeInformersForNamespaces) + go handleFencingSecretChange(ctx, nil, obj, controllerContext, operatorClient, kubeClient, kubeInformersForNamespaces, controlPlaneNodeInformer) }, UpdateFunc: func(oldObj, newObj any) { - go handleFencingSecretChange(ctx, oldObj, newObj, controllerContext, operatorClient, kubeClient, kubeInformersForNamespaces) + go handleFencingSecretChange(ctx, oldObj, newObj, controllerContext, operatorClient, kubeClient, kubeInformersForNamespaces, controlPlaneNodeInformer) }, DeleteFunc: func(obj any) { // nothing to do @@ -220,35 +160,12 @@ func runTnfResourceController(ctx context.Context, controllerContext *controller return nil } -func runPacemakerControllers(ctx context.Context, controllerContext *controllercmd.ControllerContext, operatorClient v1helpers.StaticPodOperatorClient, kubeClient kubernetes.Interface, etcdInformer operatorv1informers.EtcdInformer) { - // Wait for external etcd transition before creating any Pacemaker controllers +func runPacemakerControllers(ctx context.Context, controllerContext *controllercmd.ControllerContext, operatorClient v1helpers.StaticPodOperatorClient, kubeClient kubernetes.Interface, kubeInformersForNamespaces v1helpers.KubeInformersForNamespaces, etcdInformer operatorv1informers.EtcdInformer, nodeInformer cache.SharedIndexInformer) { + // Pacemaker controllers start after PacemakerCluster CRD is established. + // The lifecycle manager's sync() handles bootstrap vs post-transition modes internally. // This runs in a background goroutine to avoid blocking the main thread. go func() { - klog.Infof("waiting for etcd to transition to external before creating Pacemaker controllers") - - // Wait for external etcd transition to complete - for { - if err := ceohelpers.WaitForEtcdCondition( - ctx, etcdInformer, operatorClient, ceohelpers.HasExternalEtcdCompletedTransition, - 10*time.Second, 30*time.Minute, "external etcd transition", - ); err != nil { - if ctx.Err() != nil { - klog.Infof("context done while waiting for external etcd transition: %v", err) - return - } - klog.Warningf("external etcd transition not complete yet, will retry in 1m: %v", err) - select { - case <-time.After(time.Minute): - continue - case <-ctx.Done(): - return - } - } - // External etcd transition is complete, break out of the wait loop - break - } - - klog.Infof("etcd has transitioned to external; verifying PacemakerCluster CRD is established") + klog.Infof("waiting for PacemakerCluster CRD to be established before starting Pacemaker controllers") // The PacemakerCluster CRD is applied by the static resource controller. // Wait for it to be established before starting the informer. @@ -258,7 +175,7 @@ func runPacemakerControllers(ctx context.Context, controllerContext *controllerc return } - // Retry until the CRD is established or context is cancelled + // Wait for CRD to be established. for { err = wait.PollUntilContextTimeout(ctx, 2*time.Second, time.Minute, true, func(ctx context.Context) (bool, error) { crd, getErr := apiextClient.ApiextensionsV1().CustomResourceDefinitions().Get(ctx, "pacemakerclusters.etcd.openshift.io", metav1.GetOptions{}) @@ -275,7 +192,6 @@ func runPacemakerControllers(ctx context.Context, controllerContext *controllerc return false, nil }) if err == nil { - // CRD is established, break out of the retry loop break } if ctx.Err() != nil { @@ -290,79 +206,86 @@ func runPacemakerControllers(ctx context.Context, controllerContext *controllerc return } } + klog.Infof("PacemakerCluster CRD is established") - // Now create the healthcheck controller after transition is complete - klog.Infof("creating Pacemaker healthcheck controller") - healthCheckController, pacemakerInformer, err := pacemaker.NewHealthCheck( + // Prerequisites met: create and start lifecycle manager controller. + lifecycleController, lifecycleManager, pacemakerInformer, err := NewPacemakerLifecycleManager( operatorClient, kubeClient, controllerContext.EventRecorder, controllerContext.KubeConfig, + nodeInformer, + controllerContext, + kubeInformersForNamespaces, + etcdInformer, ) if err != nil { - klog.Errorf("Failed to create Pacemaker healthcheck controller: %v", err) + klog.Errorf("Failed to create Pacemaker lifecycle manager: %v", err) return } - // Start the PacemakerCluster informer now that the CRD exists - // The controller will wait for this informer to sync before processing events - klog.Infof("starting PacemakerCluster informer") + // Start the PacemakerCluster informer (controller waits for sync before processing events). go pacemakerInformer.Run(ctx.Done()) - // Start the healthcheck controller - klog.Infof("starting Pacemaker healthcheck controller") - go healthCheckController.Run(ctx, 1) + // Start the lifecycle manager controller. + // PacemakerLifecycleManager.sync() handles both bootstrap and post-transition: + // - Bootstrap: StartJobControllers() drives external etcd transition + // - Post-transition: MonitorHealth(), ReconcilePacemakerConfig(), CleanupOrphanedJobs() + go lifecycleController.Run(ctx, 1) + + // Create shared validNodeFunc for both update-setup job and status collector CronJob + // Returns intersection of K8s (ready) nodes and pacemaker nodes, or all ready nodes if CR unavailable + validNodeFunc := func() ([]*corev1.Node, error) { + return lifecycleManager.getActivePacemakerNodes() + } + + // Start the status collector CronJob with the same valid node logic + runPacemakerStatusCollectorCronJob(ctx, controllerContext, operatorClient, kubeClient, validNodeFunc) - // Start the status collector CronJob - klog.Infof("starting Pacemaker status collector CronJob") - runPacemakerStatusCollectorCronJob(ctx, controllerContext, operatorClient, kubeClient) + klog.Infof("started Pacemaker controllers (lifecycle manager, status collector)") }() } -func runPacemakerStatusCollectorCronJob(ctx context.Context, controllerContext *controllercmd.ControllerContext, operatorClient v1helpers.StaticPodOperatorClient, kubeClient kubernetes.Interface) { - // Start the cronjob controller to create a CronJob for periodic status collection - // The CronJob runs "tnf-monitor collect" which executes "sudo -n pcs status xml" - // and updates the PacemakerCluster CR - klog.Infof("starting Pacemaker status collector CronJob controller") - statusCronJobController := jobs.NewCronJobController( - pacemakerStatusCollectorName, - bindata.MustAsset("tnfdeployment/cronjob.yaml"), - operatorClient, - kubeClient, - controllerContext.EventRecorder, - func(_ *operatorv1.OperatorSpec, cronJob *batchv1.CronJob) error { - // Set the name and namespace - cronJob.SetName(pacemakerStatusCollectorName) - cronJob.SetNamespace(operatorclient.TargetNamespace) - - // Set the schedule - run every minute - cronJob.Spec.Schedule = "* * * * *" +// createFencingJobConfigFunc creates a JobConfigFunc for the fencing job. +// Returns a function that captures node UIDs and fencing secret UIDs for drift detection. +func createFencingJobConfigFunc(nodeList []*corev1.Node, kubeInformersForNamespaces v1helpers.KubeInformersForNamespaces) jobs.JobConfigFunc { + return func() (string, error) { + // Collect node UIDs + nodeUIDs := make([]string, len(nodeList)) + for i, node := range nodeList { + nodeUIDs[i] = string(node.UID) + } + sort.Strings(nodeUIDs) - // Initialize labels maps if nil and set labels at all levels - if cronJob.Labels == nil { - cronJob.Labels = make(map[string]string) - } - cronJob.Labels["app.kubernetes.io/name"] = pacemakerStatusCollectorName + // Collect fencing secret UIDs from informer + secretsLister := kubeInformersForNamespaces.InformersFor(operatorclient.TargetNamespace).Core().V1().Secrets().Lister() + allSecrets, err := secretsLister.List(labels.Everything()) + if err != nil { + return "", fmt.Errorf("failed to list secrets: %w", err) + } - if cronJob.Spec.JobTemplate.Labels == nil { - cronJob.Spec.JobTemplate.Labels = make(map[string]string) + var secretUIDs []string + for _, secret := range allSecrets { + if tools.IsFencingSecret(secret.Name) { + secretUIDs = append(secretUIDs, string(secret.UID)) } - cronJob.Spec.JobTemplate.Labels["app.kubernetes.io/name"] = pacemakerStatusCollectorName + } + sort.Strings(secretUIDs) - if cronJob.Spec.JobTemplate.Spec.Template.Labels == nil { - cronJob.Spec.JobTemplate.Spec.Template.Labels = make(map[string]string) - } - cronJob.Spec.JobTemplate.Spec.Template.Labels["app.kubernetes.io/name"] = pacemakerStatusCollectorName + // Create config map with both node and secret UIDs + config := map[string]interface{}{ + "nodeUIDs": nodeUIDs, + "secretUIDs": secretUIDs, + } - // Configure the container - cronJob.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Image = os.Getenv("OPERATOR_IMAGE") - cronJob.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Command = []string{"tnf-monitor", "collect", "-v=4"} + configJSON, err := json.Marshal(config) + if err != nil { + return "", fmt.Errorf("failed to marshal fencing config: %w", err) + } - return nil - }, - ) - go statusCronJobController.Run(ctx, 1) + return string(configJSON), nil + } } func handleFencingSecretChange( @@ -372,6 +295,7 @@ func handleFencingSecretChange( operatorClient v1helpers.StaticPodOperatorClient, kubeClient kubernetes.Interface, kubeInformersForNamespaces v1helpers.KubeInformersForNamespaces, + nodeInformer cache.SharedIndexInformer, ) { // obj can be nil, always restart fencing job in that case @@ -413,7 +337,30 @@ func handleFencingSecretChange( } } - err := jobs.RestartJobOrRunController(ctx, tools.JobTypeFencing, nil, controllerContext, operatorClient, kubeClient, kubeInformersForNamespaces, jobs.DefaultConditions, tools.FencingJobCompletedTimeout) + // OCPBUGS-84695: before handoff, ignore secret-driven fencing (install orders fencing via startTnfJobcontrollers). + transitionComplete, err := ceohelpers.HasExternalEtcdCompletedTransition(ctx, operatorClient) + if err != nil { + klog.Errorf("failed to check external etcd transition for fencing secret handler: %v", err) + return + } + if !transitionComplete { + klog.V(4).Infof("skipping fencing job restart from fencing secret change until %s is True", + ceohelpers.OperatorConditionExternalEtcdHasCompletedTransition) + return + } + + // Get control plane nodes for jobConfigFunc + nodes, err := ceohelpers.ListNodesFromInformer(nodeInformer) + if err != nil { + klog.Errorf("failed to list control plane nodes for fencing job config: %v", err) + return + } + + // Create jobConfigFunc to capture node UIDs + fencing secret UIDs for drift detection + fencingJobConfigFunc := createFencingJobConfigFunc(nodes, kubeInformersForNamespaces) + + // Restart fencing job with drift detection + err = jobs.RestartJobOrRunController(ctx, tools.JobTypeFencing, nil, nil, fencingJobConfigFunc, 3, controllerContext, operatorClient, kubeClient, kubeInformersForNamespaces, jobs.DefaultConditions, tools.FencingJobCompletedTimeout) if err != nil { klog.Errorf("failed to restart fencing job: %v", err) return diff --git a/pkg/tnf/operator/starter_test.go b/pkg/tnf/operator/starter_test.go index fffb46c346..53752b63ae 100644 --- a/pkg/tnf/operator/starter_test.go +++ b/pkg/tnf/operator/starter_test.go @@ -1,15 +1,35 @@ package operator +/* +TEST COVERAGE SUMMARY - starter_test.go +======================================== + +This file tests TNF bootstrap and initial setup logic for Two-Node Fencing clusters. + +WHAT'S TESTED +------------- + +Bootstrap Scenarios: +├── TestHandleDualReplicaClusters - Dual-replica bootstrap logic +│ ├── ✅ Non-TNF cluster (skip TNF bootstrap) +│ ├── ✅ 1 ready node (wait for 2nd node) +│ ├── ✅ 2 ready nodes, no jobs (run bootstrap) +│ └── ✅ 2 ready nodes, existing jobs (already bootstrapped) +├── TestSetupJobConditionsBasedOnExternalEtcd - Job condition selection +│ ├── ✅ Before external etcd transition (AllConditions) +│ └── ✅ After external etcd transition (DefaultConditions) +└── TestHandleFencingSecretChange_skipsBeforeExternalEtcdTransition + └── ✅ Skips fencing secret watch before external etcd transition +*/ + import ( "context" - "errors" "maps" "slices" "testing" "time" "github.com/stretchr/testify/require" - "k8s.io/apimachinery/pkg/util/wait" configv1 "github.com/openshift/api/config/v1" operatorv1 "github.com/openshift/api/operator/v1" @@ -33,7 +53,6 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/kubernetes/scheme" - corev1listers "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" "k8s.io/utils/clock" @@ -217,7 +236,7 @@ func getArgs(t *testing.T, dualReplicaControlPlaneEnabled bool) args { ) controlPlaneNodeInformer := corev1informers.NewFilteredNodeInformer(fakeKubeClient, 1*time.Hour, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, func(listOptions *metav1.ListOptions) { - listOptions.LabelSelector = "node-role.kubernetes.io/master" + listOptions.LabelSelector = "node-role.kubernetes.io/control-plane" }) etcdIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) @@ -268,115 +287,49 @@ func getArgs(t *testing.T, dualReplicaControlPlaneEnabled bool) args { } } -func TestHandleNodesWithRetry(t *testing.T) { - tests := []struct { - name string - setupMockHandleNodes func() func(*controllercmd.ControllerContext, corev1listers.NodeLister, context.Context, v1helpers.StaticPodOperatorClient, kubernetes.Interface, v1helpers.KubeInformersForNamespaces, operatorv1informers.EtcdInformer) error - expectDegradedCondition bool - expectDegradedStatus operatorv1.ConditionStatus - expectRetries bool - }{ - { - name: "Success on first attempt", - setupMockHandleNodes: func() func(*controllercmd.ControllerContext, corev1listers.NodeLister, context.Context, v1helpers.StaticPodOperatorClient, kubernetes.Interface, v1helpers.KubeInformersForNamespaces, operatorv1informers.EtcdInformer) error { - return func(_ *controllercmd.ControllerContext, _ corev1listers.NodeLister, _ context.Context, _ v1helpers.StaticPodOperatorClient, _ kubernetes.Interface, _ v1helpers.KubeInformersForNamespaces, _ operatorv1informers.EtcdInformer) error { - return nil - } - }, - expectDegradedCondition: true, - expectDegradedStatus: operatorv1.ConditionFalse, - expectRetries: false, - }, - { - name: "Success after retries", - setupMockHandleNodes: func() func(*controllercmd.ControllerContext, corev1listers.NodeLister, context.Context, v1helpers.StaticPodOperatorClient, kubernetes.Interface, v1helpers.KubeInformersForNamespaces, operatorv1informers.EtcdInformer) error { - attemptCount := 0 - return func(_ *controllercmd.ControllerContext, _ corev1listers.NodeLister, _ context.Context, _ v1helpers.StaticPodOperatorClient, _ kubernetes.Interface, _ v1helpers.KubeInformersForNamespaces, _ operatorv1informers.EtcdInformer) error { - attemptCount++ - if attemptCount < 3 { - return errors.New("temporary failure") - } - return nil - } - }, - expectDegradedCondition: true, - expectDegradedStatus: operatorv1.ConditionFalse, - expectRetries: true, - }, - { - name: "Failure after all retries", - setupMockHandleNodes: func() func(*controllercmd.ControllerContext, corev1listers.NodeLister, context.Context, v1helpers.StaticPodOperatorClient, kubernetes.Interface, v1helpers.KubeInformersForNamespaces, operatorv1informers.EtcdInformer) error { - return func(_ *controllercmd.ControllerContext, _ corev1listers.NodeLister, _ context.Context, _ v1helpers.StaticPodOperatorClient, _ kubernetes.Interface, _ v1helpers.KubeInformersForNamespaces, _ operatorv1informers.EtcdInformer) error { - return errors.New("persistent failure") - } - }, - expectDegradedCondition: true, - expectDegradedStatus: operatorv1.ConditionTrue, - expectRetries: true, - }, +func TestHandleFencingSecretChange_skipsBeforeExternalEtcdTransition(t *testing.T) { + ctx := context.Background() + fakeKube := fake.NewSimpleClientset() + + fakeOperatorClient := v1helpers.NewFakeStaticPodOperatorClient( + &operatorv1.StaticPodOperatorSpec{}, + u.StaticPodOperatorStatus(), + nil, + nil, + ) + + eventRecorder := events.NewRecorder( + fakeKube.CoreV1().Events(operatorclient.TargetNamespace), + "test-fencing-secret", + &corev1.ObjectReference{}, + clock.RealClock{}, + ) + controllerContext := &controllercmd.ControllerContext{ + EventRecorder: eventRecorder, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Setup test environment - testArgs := getArgs(t, true) - - // Create NodeLister from the informer - controlPlaneNodeLister := corev1listers.NewNodeLister(testArgs.controlPlaneNodeInformer.GetIndexer()) - - // Store original handleNodesFunc and replace with mock - originalHandleNodesFunc := handleNodesFunc - handleNodesFunc = tt.setupMockHandleNodes() - defer func() { handleNodesFunc = originalHandleNodesFunc }() - - // Store original backoff config and use faster settings for testing - originalBackoff := retryBackoffConfig - retryBackoffConfig = wait.Backoff{ - Duration: 100 * time.Millisecond, - Factor: 2.0, - Steps: 5, // Much shorter for testing - Cap: 500 * time.Millisecond, - } - defer func() { retryBackoffConfig = originalBackoff }() - - // For faster testing, use a reasonable timeout - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - // Run handleNodesWithRetry - handleNodesWithRetry(testArgs.controllerContext, controlPlaneNodeLister, ctx, testArgs.operatorClient, - testArgs.kubeClient, testArgs.kubeInformersForNamespaces, testArgs.etcdInformer) - - // Verify the operator condition was set correctly - if tt.expectDegradedCondition { - _, status, _, err := testArgs.operatorClient.GetStaticPodOperatorState() - require.NoError(t, err, "failed to get operator status") - - // Find the TNFJobControllersDegraded condition - var foundCondition *operatorv1.OperatorCondition - for i, condition := range status.Conditions { - if condition.Type == "TNFJobControllersDegraded" { - foundCondition = &status.Conditions[i] - break - } - } + kubeInformersForNamespaces := v1helpers.NewKubeInformersForNamespaces( + fakeKube, + "", + operatorclient.GlobalUserSpecifiedConfigNamespace, + operatorclient.GlobalMachineSpecifiedConfigNamespace, + operatorclient.TargetNamespace, + operatorclient.OperatorNamespace, + "kube-system", + ) - require.NotNil(t, foundCondition, "TNFJobControllersDegraded condition not found") - require.Equal(t, tt.expectDegradedStatus, foundCondition.Status, - "Expected degraded status %v but got %v", tt.expectDegradedStatus, foundCondition.Status) - - if tt.expectDegradedStatus == operatorv1.ConditionTrue { - require.Equal(t, "SetupFailed", foundCondition.Reason, - "Expected reason SetupFailed but got %s", foundCondition.Reason) - require.Contains(t, foundCondition.Message, "Failed to setup TNF job controllers", - "Expected message to contain failure info") - } else { - require.Equal(t, "AsExpected", foundCondition.Reason, - "Expected reason AsExpected but got %s", foundCondition.Reason) - require.Contains(t, foundCondition.Message, "successfully", - "Expected success message") - } - } - }) + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fencing-credentials-master-0", + Namespace: operatorclient.TargetNamespace, + }, + Data: map[string][]byte{"token": []byte("x")}, } + + // Pass nil nodeInformer - function returns early before using it when transition not complete + handleFencingSecretChange(ctx, nil, secret, controllerContext, fakeOperatorClient, fakeKube, kubeInformersForNamespaces, nil) + + jobList, err := fakeKube.BatchV1().Jobs(operatorclient.TargetNamespace).List(ctx, metav1.ListOptions{}) + require.NoError(t, err) + require.Empty(t, jobList.Items, "fencing job should not be created from secrets before external etcd transition completes") } diff --git a/pkg/tnf/setup/runner.go b/pkg/tnf/setup/runner.go index 22bee826dc..fac2f5babc 100644 --- a/pkg/tnf/setup/runner.go +++ b/pkg/tnf/setup/runner.go @@ -105,12 +105,6 @@ func RunTnfSetup() error { return err } - // register pacemaker alert agents for fencing taint/untaint - err = pcs.ConfigureAlerts(ctx) - if err != nil { - return err - } - // Etcd handover // configure etcd resource - it won't start etcd before CEO managed etcd is removed per node @@ -134,6 +128,13 @@ func RunTnfSetup() error { return err } + // register pacemaker alert agents for fencing taint/untaint + // this is done last so core functionality (etcd resource, stonith) works even if alerts fail + err = pcs.ConfigureAlerts(ctx) + if err != nil { + return err + } + // get pcs cib cib, err := pcs.GetCIB(ctx) if err != nil { diff --git a/pkg/tnf/update-setup/runner.go b/pkg/tnf/update-setup/runner.go index 84834b82fe..61f26ddc52 100644 --- a/pkg/tnf/update-setup/runner.go +++ b/pkg/tnf/update-setup/runner.go @@ -2,13 +2,20 @@ package updatesetup import ( "context" + "encoding/json" + "encoding/xml" "fmt" + "net" + "net/url" "os" + "sort" + "strconv" "strings" "time" operatorv1 "github.com/openshift/api/operator/v1" - "github.com/openshift/library-go/pkg/operator/genericoperatorclient" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apiserver/pkg/server" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -16,12 +23,53 @@ import ( "k8s.io/utils/clock" "github.com/openshift/cluster-etcd-operator/pkg/operator" + "github.com/openshift/cluster-etcd-operator/pkg/operator/ceohelpers" + "github.com/openshift/cluster-etcd-operator/pkg/operator/operatorclient" "github.com/openshift/cluster-etcd-operator/pkg/tnf/pkg/config" "github.com/openshift/cluster-etcd-operator/pkg/tnf/pkg/etcd" "github.com/openshift/cluster-etcd-operator/pkg/tnf/pkg/exec" + "github.com/openshift/cluster-etcd-operator/pkg/tnf/pkg/pacemaker" "github.com/openshift/cluster-etcd-operator/pkg/tnf/pkg/pcs" + "github.com/openshift/cluster-etcd-operator/pkg/tnf/pkg/tools" + "github.com/openshift/library-go/pkg/operator/genericoperatorclient" ) +// pickLatestUpdateSetupConfigMap returns the ConfigMap with the highest generation. +// During normal operation, there should only be one ConfigMap (old ones are cleaned up), +// but this picks the latest to be safe. +func pickLatestUpdateSetupConfigMap(items []corev1.ConfigMap) (*corev1.ConfigMap, error) { + var best *corev1.ConfigMap + var bestGen int64 = -1 + for i := range items { + cm := &items[i] + genStr := cm.Data["generation"] + g, err := strconv.ParseInt(genStr, 10, 64) + if err != nil { + klog.Warningf("update-setup ConfigMap %s has invalid generation %q: %v", cm.Name, genStr, err) + continue + } + if g > bestGen { + bestGen = g + best = cm + } + } + if best == nil { + return nil, fmt.Errorf("no update-setup ConfigMap found") + } + return best, nil +} + +func decodeStringList(data string) ([]string, error) { + if strings.TrimSpace(data) == "" || data == "[]" { + return []string{}, nil + } + var list []string + if err := json.Unmarshal([]byte(data), &list); err != nil { + return nil, err + } + return list, nil +} + func RunTnfUpdateSetup() error { klog.Info("Setting up clients etc. for TNF update-setup") @@ -52,29 +100,51 @@ func RunTnfUpdateSetup() error { dynamicInformers.Start(ctx.Done()) dynamicInformers.WaitForCacheSync(ctx.Done()) - // Get the current node name from environment + // Get the current node name from environment (used for cluster config and logging only) currentNodeName := os.Getenv("MY_NODE_NAME") if currentNodeName == "" { return fmt.Errorf("MY_NODE_NAME environment variable not set") } - klog.Infof("Running TNF update-setup") + klog.Infof("Running TNF update-setup on node %s", currentNodeName) // check if cluster is running on this node command := "/usr/sbin/pcs cluster status" _, _, err = exec.Execute(ctx, command) if err != nil { - klog.Infof("Cluster not running (err: %v), skipping update-setup on this node", err) - return nil + return fmt.Errorf("pacemaker is not running on this node %s: %w", currentNodeName, err) + } + + // Pick the latest update-setup ConfigMap (cluster-wide, not node-specific) + cmList, err := kubeClient.CoreV1().ConfigMaps(operatorclient.TargetNamespace).List(ctx, metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/component=" + tools.TnfUpdateSetupComponentValue, + }) + if err != nil { + return fmt.Errorf("failed to list update-setup ConfigMaps: %w", err) } - // Get current cluster config from Kubernetes - cfg, err := config.GetClusterConfig(ctx, kubeClient) + cm, err := pickLatestUpdateSetupConfigMap(cmList.Items) if err != nil { return err } - // Determine which node we are and get the new IP + klog.Infof("Using ConfigMap %s (generation: %s, timestamp: %s)", + cm.Name, cm.Data["generation"], cm.Data["timestamp"]) + + capturedNodes, err := decodeNodeList(cm.Data["nodes"]) + if err != nil { + return fmt.Errorf("failed to decode node list: %w", err) + } + + klog.Infof("Running update-setup for %d-node cluster", len(capturedNodes)) + klog.V(2).Infof("Captured nodes: %v", getNodeNames(capturedNodes)) + + cfg, err := buildClusterConfigFromNodeList(capturedNodes) + if err != nil { + return err + } + + // Determine which node we are and get node IPs var currentNodeIP, otherNodeName, otherNodeIP string if cfg.NodeName1 == currentNodeName { currentNodeIP = cfg.NodeIP1 @@ -88,98 +158,238 @@ func RunTnfUpdateSetup() error { return fmt.Errorf("current node %s not found in cluster config (nodes: %s, %s)", currentNodeName, cfg.NodeName1, cfg.NodeName2) } - // find offline node - command = "/usr/sbin/pcs status nodes corosync | grep Offline | awk '{print $2}'" - stdOut, stdErr, err := exec.Execute(ctx, command) + currentNodeIPs := toSet(cfg.NodeIP1, cfg.NodeIP2) + + // Log operational context (always visible) + klog.Infof("Running update-setup on node %q (%d-node cluster)", currentNodeName, len(capturedNodes)) + + // Log IP details at debug level + if len(capturedNodes) == 1 { + klog.V(2).Infof("Single-node mode - Current node: %q (IP: %s)", currentNodeName, currentNodeIP) + } else { + klog.V(2).Infof("Two-node mode - Current: %q (IP: %s), Other: %q (IP: %s)", currentNodeName, currentNodeIP, otherNodeName, otherNodeIP) + } + + // Build desired state from ConfigMap (K8s nodes that should be in pacemaker) + desiredNodes := make(map[string]string) // nodeName -> IP + for _, node := range capturedNodes { + ip, err := tools.GetNodeIPForPacemaker(*node) + if err != nil { + klog.Warningf("Skipping node %s from desired state (no valid IP): %v", node.Name, err) + continue + } + desiredNodes[node.Name] = ip + } + + // Get current pacemaker membership + // CRITICAL: If we can't determine current state, we MUST NOT make any changes. + // Making blind changes to pacemaker (node membership, fencing, resources) could brick the cluster + // by stopping etcd or causing split-brain. + currentPacemakerNodes, err := getCurrentPacemakerNodesWithIPs(ctx) if err != nil { - klog.Errorf("Failed to find offline node: %s, stdout: %s, stderr: %s, err: %v", command, stdOut, stdErr, err) - return err + klog.Errorf("Cannot determine current pacemaker membership - refusing to make changes to avoid breaking etcd: %v", err) + return fmt.Errorf("failed to get current pacemaker membership (will retry later, no changes made): %w", err) } - offlineNodeName := strings.TrimSpace(stdOut) - if offlineNodeName == "" { - klog.Info("No offline node found, nothing to do") + // Determine what needs to change by comparing desired vs current + nodesToRemove, nodesToAdd := tools.DetermineReconciliationActions(desiredNodes, currentPacemakerNodes) + + klog.Infof("Reconciliation: desired=%d nodes, current=%d nodes, toAdd=%v, toRemove=%v", + len(desiredNodes), len(currentPacemakerNodes), nodesToAdd, nodesToRemove) + + // Early exit if no changes needed + if len(nodesToRemove) == 0 && len(nodesToAdd) == 0 { + klog.Info("No membership changes needed (desired state matches current state)") return nil } - klog.Infof("Current node: %q (IP: %s), Other node: %q (IP: %s), Offline node: %q", currentNodeName, currentNodeIP, otherNodeName, otherNodeIP, offlineNodeName) + // Remove nodes from pacemaker if needed + if len(nodesToRemove) > 0 { + for _, nodeToRemove := range nodesToRemove { + if err := removePacemakerNodeGlobally(ctx, nodeToRemove); err != nil { + return err + } + } - // don't start the cluster on the new node too early, it might result in etcd start failure because of missing manifests on the new node - klog.Info("Waiting for etcd revision update before going on...") - err = etcd.WaitForStableRevision(ctx, operatorClient) - if err != nil { - klog.Error(err, "Failed to wait for etcd container transition") - return err + // Remove unstarted etcd members (from deleted nodes) + if err := removeUnstartedEtcdMembers(ctx, currentNodeIPs); err != nil { + return err + } + } + + // Add nodes to pacemaker if needed + if len(nodesToAdd) > 0 { + // Wait for etcd revision stability before adding nodes to pacemaker + // (ensures new node has etcd manifests before cluster start) + klog.Info("Waiting for etcd revision update before adding nodes...") + if err = etcd.WaitForStableRevision(ctx, operatorClient); err != nil { + return fmt.Errorf("failed to wait for etcd revision stability: %w", err) + } + + var commands []string + for _, nodeToAdd := range nodesToAdd { + commands = append(commands, fmt.Sprintf("/usr/sbin/pcs cluster node add %s", nodeToAdd)) + } + if err = runCommands(ctx, commands); err != nil { + return err + } } - commands := []string{ - // Remove offline node from the cluster configuration - fmt.Sprintf("/usr/sbin/pcs cluster node remove %s --force --skip-offline", offlineNodeName), - // Add new node to the cluster configuration - fmt.Sprintf("/usr/sbin/pcs cluster node add %s", otherNodeName), + // Reconcile STONITH after membership changes (other node first, then current). + if len(nodesToRemove) > 0 || len(nodesToAdd) > 0 { + fencingNodes := make([]string, 0, 2) + if otherNodeName != "" { + fencingNodes = append(fencingNodes, otherNodeName) + } + fencingNodes = append(fencingNodes, currentNodeName) + if err = pcs.ConfigureFencing(ctx, kubeClient, fencingNodes); err != nil { + return fmt.Errorf("failed to configure fencing: %w", err) + } } - err = runCommands(ctx, commands) - if err != nil { - return err + + // Update etcd resource with current node IPs after any membership changes + // This runs once regardless of whether we removed, added, or both + if len(nodesToRemove) > 0 || len(nodesToAdd) > 0 { + desiredNodeIPMap := buildNodeIPMap(cfg) + + // Check current node_ip_map to avoid unnecessary updates + currentNodeIPMap, err := getCurrentNodeIPMap(ctx) + if err != nil { + klog.Warningf("Failed to get current node_ip_map (will attempt update): %v", err) + } else if currentNodeIPMap == desiredNodeIPMap { + klog.Infof("Skipping node_ip_map update (already set to %q)", desiredNodeIPMap) + } else { + klog.Infof("Updating node_ip_map from %q to %q", currentNodeIPMap, desiredNodeIPMap) + command = fmt.Sprintf("/usr/sbin/pcs resource update etcd node_ip_map=\"%s\" --wait=300", desiredNodeIPMap) + stdOut, stdErr, err := exec.Execute(ctx, command) + if err != nil { + klog.Errorf("Failed to update etcd node_ip_map: %s, stdout: %s, stderr: %s, err: %v", command, stdOut, stdErr, err) + return err + } + klog.Infof("Successfully updated etcd node_ip_map") + } } - // update fence devices - // this is needed for being able to start resources on the new node! - // node order matters here: resources can't be restarted while fencing isn't configured on all nodes! - err = pcs.ConfigureFencing(ctx, kubeClient, []string{otherNodeName, currentNodeName}) - if err != nil { - klog.Error(err, "Failed to configure fencing, skipping update of etcd! Restart update-setup job when fencing config is fixed!") - return err + // If we added nodes, force new cluster and restart + // (skip this if we only removed nodes) + if len(nodesToAdd) > 0 { + commands := []string{ + // Force new cluster on next etcd restart on this node + fmt.Sprintf("crm_attribute --lifetime reboot --node %s --name \"force_new_cluster\" --update %s", currentNodeName, currentNodeName), + } + if err = runCommands(ctx, commands); err != nil { + return err + } + + // Wait for pacemaker resource update to stabilize before starting cluster + time.Sleep(10 * time.Second) + + commands = []string{ + // Enable cluster on all nodes + "/usr/sbin/pcs cluster enable --all", + // Start cluster on all nodes + "/usr/sbin/pcs cluster start --all", + } + if err = runCommands(ctx, commands); err != nil { + return err + } } - // register pacemaker alert agents for fencing taint/untaint + // Always sync/start at end of this path (cluster was running on this host at start). + if err := runCommands(ctx, []string{ + "/usr/sbin/pcs cluster sync", + "/usr/sbin/pcs cluster start --all", + }); err != nil { + return fmt.Errorf("pcs cluster sync/start after update-setup: %w", err) + } + klog.Info("Completed pcs cluster sync and pcs cluster start --all after update-setup") + + // Register pacemaker alert agents for fencing taint/untaint + // This is done last so core functionality (node membership, fencing, etcd resource) works even if alerts fail err = pcs.ConfigureAlerts(ctx) if err != nil { return err } - commands = []string{ - // Force new cluster on next etcd restart on this node - fmt.Sprintf("crm_attribute --lifetime reboot --node %s --name \"force_new_cluster\" --update %s", currentNodeName, currentNodeName), - // Update etcd resource - fmt.Sprintf("/usr/sbin/pcs resource update etcd node_ip_map=\"%s:%s;%s:%s\" --wait=300", cfg.NodeName1, cfg.NodeIP1, cfg.NodeName2, cfg.NodeIP2), + return nil +} + +// toSet builds a set from non-empty strings +func toSet(items ...string) map[string]struct{} { + s := make(map[string]struct{}) + for _, item := range items { + if item != "" { + s[item] = struct{}{} + } } - err = runCommands(ctx, commands) + return s +} + +// getCurrentNodeIPMap queries the current node_ip_map parameter from the etcd resource +func getCurrentNodeIPMap(ctx context.Context) (string, error) { + command := "/usr/sbin/pcs resource config etcd" + stdOut, stdErr, err := exec.Execute(ctx, command) if err != nil { - return err + return "", fmt.Errorf("failed to get etcd resource config: stdout=%s, stderr=%s, err=%w", stdOut, stdErr, err) } - // remove old node from etcd members - command = "podman exec etcd /usr/bin/etcdctl member list | grep unstarted | awk -F, '{ print $1 }'" - stdOut, stdErr, err = exec.Execute(ctx, command) - if err != nil { - klog.Errorf("Failed to find unstarted etcd member: %s, stdout: %s, stderr: %s, err: %v", command, stdOut, stdErr, err) - } else { - unstartedMemberID := strings.TrimSpace(stdOut) - command = fmt.Sprintf("podman exec etcd /usr/bin/etcdctl member remove %s", unstartedMemberID) - stdOut, stdErr, err = exec.Execute(ctx, command) - if err != nil { - klog.Errorf("Failed to remove unstarted etcd member: %s, stdout: %s, stderr: %s, err: %v", command, stdOut, stdErr, err) - return err + // Parse output for node_ip_map parameter + // Expected format: "Attributes: etcd: node_ip_map=master-0:IP;master-1:IP ..." + lines := strings.Split(stdOut, "\n") + for _, line := range lines { + line = strings.TrimSpace(line) + if strings.Contains(line, "node_ip_map=") { + // Extract value after node_ip_map= + parts := strings.SplitN(line, "node_ip_map=", 2) + if len(parts) == 2 { + // Value may have quotes and other parameters after it + value := strings.TrimPrefix(parts[1], "\"") + value = strings.SplitN(value, "\"", 2)[0] // Get text before closing quote + value = strings.Fields(value)[0] // Get first word (handles unquoted case) + return value, nil + } } - klog.Infof("Removed unstarted etcd member: %s", unstartedMemberID) } - // wait a bit for things to settle - // without this the etcd start on the new node fails for some reason... - time.Sleep(10 * time.Second) + return "", fmt.Errorf("node_ip_map parameter not found in etcd resource config") +} - commands = []string{ - // Enable cluster on new node - "/usr/sbin/pcs cluster enable --all", - // Start cluster on new node - "/usr/sbin/pcs cluster start --all", +// buildNodeIPMap creates the node_ip_map parameter for pacemaker +func buildNodeIPMap(cfg config.ClusterConfig) string { + var pairs []string + if cfg.NodeName1 != "" && cfg.NodeIP1 != "" { + pairs = append(pairs, fmt.Sprintf("%s:%s", cfg.NodeName1, cfg.NodeIP1)) } - err = runCommands(ctx, commands) - if err != nil { - return err + if cfg.NodeName2 != "" && cfg.NodeIP2 != "" { + pairs = append(pairs, fmt.Sprintf("%s:%s", cfg.NodeName2, cfg.NodeIP2)) } + return strings.Join(pairs, ";") +} + +// pcsClusterNodeRemoveOutputContains401 reports whether pcs remove output suggests PCSD HTTP 401. +func pcsClusterNodeRemoveOutputContains401(stdout, stderr string) bool { + return strings.Contains(stdout, "401") || strings.Contains(stderr, "401") +} +// removePacemakerNodeGlobally runs pcs cluster node remove. Output containing "401" is logged and ignored. +func removePacemakerNodeGlobally(ctx context.Context, nodeToRemove string) error { + cmd := fmt.Sprintf("/usr/sbin/pcs cluster node remove %s --force --skip-offline", nodeToRemove) + stdOut, stdErr, err := exec.Execute(ctx, cmd) + if err != nil { + if pcsClusterNodeRemoveOutputContains401(stdOut, stdErr) { + klog.Warningf( + "pcs cluster node remove for %q did not succeed and output contained 401: "+ + "the cluster may still see a live endpoint for that member that rejects this node's PCSD credentials — "+ + "commonly a replacement node is answering at the same IP the cluster associates with the old member. "+ + "Skipping fatal error; reconcile sync/start and fencing may still help. stdout=%q stderr=%q err=%v", + nodeToRemove, stdOut, stdErr, err, + ) + return nil + } + klog.Errorf("pcs cluster node remove failed for %q: stdout=%q stderr=%q err=%v", nodeToRemove, stdOut, stdErr, err) + return fmt.Errorf("global remove pacemaker node %q: %w (stderr: %s)", nodeToRemove, err, strings.TrimSpace(stdErr)) + } + klog.Infof("Successfully removed node %q from pacemaker via pcs cluster node remove", nodeToRemove) return nil } @@ -194,3 +404,281 @@ func runCommands(ctx context.Context, commands []string) error { } return nil } + +// removeUnstartedEtcdMembers removes unstarted etcd members that don't match any current k8s node. +// When a node is removed from pacemaker, its etcd member becomes unstarted (empty name). +// This function removes all such unstarted members whose IP doesn't match any node in currentNodeIPs, +// cleaning up stale members from deleted nodes while preserving members for current nodes. +func removeUnstartedEtcdMembers(ctx context.Context, currentNodeIPs map[string]struct{}) error { + klog.Info("Checking for unstarted etcd members to remove") + + // Get etcd member list + command := "podman exec etcd /usr/bin/etcdctl member list -w json" + stdOut, stdErr, err := exec.Execute(ctx, command) + if err != nil { + return fmt.Errorf("failed to list etcd members: stdout: %s, stderr: %s, err: %w", stdOut, stdErr, err) + } + + // Parse JSON response + var memberList struct { + Members []struct { + ID uint64 `json:"ID"` + Name string `json:"name"` + PeerURLs []string `json:"peerURLs"` + ClientURLs []string `json:"clientURLs"` + IsLearner bool `json:"isLearner"` + } `json:"members"` + } + + if err := json.Unmarshal([]byte(stdOut), &memberList); err != nil { + return fmt.Errorf("failed to parse etcd member list JSON: %w", err) + } + + // Remove all unstarted members that don't match any current node + var removed []string + for _, member := range memberList.Members { + // Only consider unstarted members (empty name indicates unstarted) + if member.Name != "" { + continue + } + + memberIDHex := fmt.Sprintf("%x", member.ID) + + // Extract IP from peer URLs + var memberIP string + for _, peerURLStr := range member.PeerURLs { + parsedURL, err := url.Parse(peerURLStr) + if err != nil { + continue + } + + host, _, err := net.SplitHostPort(parsedURL.Host) + if err != nil { + // No port in URL - use host directly and strip IPv6 brackets if present + host = parsedURL.Host + if strings.HasPrefix(host, "[") && strings.HasSuffix(host, "]") { + host = host[1 : len(host)-1] + } + } + // SplitHostPort already removes brackets for IPv6 when successful + + // Validate that we got a valid IP + if net.ParseIP(host) == nil { + klog.Warningf("Extracted host %q from peer URL %q is not a valid IP", host, peerURLStr) + continue + } + + memberIP = host + break + } + + if memberIP == "" { + klog.Warningf("Could not extract IP from unstarted member %s peer URLs: %v", memberIDHex, member.PeerURLs) + continue + } + + // Check if this IP matches any current node (use normalized comparison for IPv6) + matchesCurrentNode := false + for nodeIP := range currentNodeIPs { + if ceohelpers.IPAddressesEqual(memberIP, nodeIP) { + matchesCurrentNode = true + break + } + } + if matchesCurrentNode { + klog.Infof("Unstarted etcd member %s matches a current node - not removing", memberIDHex) + klog.V(2).Infof("Member %s IP: %s", memberIDHex, memberIP) + continue + } + + // This is an unstarted member for a removed/stale node - remove it + klog.Infof("Found unstarted etcd member %s not matching any current node - removing", memberIDHex) + klog.V(2).Infof("Member %s IP: %s", memberIDHex, memberIP) + command = fmt.Sprintf("podman exec etcd /usr/bin/etcdctl member remove %s", memberIDHex) + stdOut, stdErr, err = exec.Execute(ctx, command) + if err != nil { + return fmt.Errorf("failed to remove unstarted etcd member %s: stdout: %s, stderr: %s, err: %w", memberIDHex, stdOut, stdErr, err) + } + removed = append(removed, fmt.Sprintf("%s (IP: %s)", memberIDHex, memberIP)) + klog.Infof("Removed unstarted etcd member %s", memberIDHex) + } + + if len(removed) == 0 { + klog.Info("No unstarted etcd members found for removed nodes") + } else { + klog.Infof("Successfully removed %d unstarted etcd member(s): %v", len(removed), removed) + } + + return nil +} + +func decodeNodeList(data string) ([]*corev1.Node, error) { + type nodeInfo struct { + Name string `json:"name"` + IP string `json:"ip"` + } + + var infos []nodeInfo + if err := json.Unmarshal([]byte(data), &infos); err != nil { + return nil, err + } + + nodes := make([]*corev1.Node, len(infos)) + for i, info := range infos { + // Create minimal node with name and IP address + nodes[i] = &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: info.Name, + }, + Status: corev1.NodeStatus{ + Addresses: []corev1.NodeAddress{ + { + Type: corev1.NodeInternalIP, + Address: info.IP, + }, + }, + }, + } + } + + return nodes, nil +} + +func getNodeNames(nodes []*corev1.Node) []string { + names := make([]string, len(nodes)) + for i, node := range nodes { + names[i] = node.Name + } + return names +} + +func buildClusterConfigFromNodeList(nodes []*corev1.Node) (config.ClusterConfig, error) { + cfg := config.ClusterConfig{} + + if len(nodes) == 0 || len(nodes) > 2 { + return cfg, fmt.Errorf("invalid number of nodes: %d", len(nodes)) + } + + // Sort for deterministic ordering + sort.Slice(nodes, func(i, j int) bool { + return nodes[i].Name < nodes[j].Name + }) + + for i, node := range nodes { + ip, err := tools.GetNodeIPForPacemaker(*node) + if err != nil { + return cfg, err + } + + switch i { + case 0: + cfg.NodeName1 = node.Name + cfg.NodeIP1 = ip + case 1: + cfg.NodeName2 = node.Name + cfg.NodeIP2 = ip + } + } + + return cfg, nil +} + +// getCurrentPacemakerNodesWithIPs queries pacemaker directly to get current membership with IPs. +// Returns a map of node names to IPs. +// getRuntimePacemakerNodes returns the actual runtime cluster membership from pcs status xml. +// This reflects which nodes are currently configured in the cluster, not the corosync.conf file. +// The config file can be stale (e.g., after a node is fenced), so we use runtime state for reconciliation. +func getRuntimePacemakerNodes(ctx context.Context) (map[string]bool, error) { + const pcsStatusXMLCommand = "sudo -n pcs status xml" + + ctxExec, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + stdout, stderr, err := exec.Execute(ctxExec, pcsStatusXMLCommand) + if err != nil { + return nil, fmt.Errorf("failed to execute pcs status xml: %w", err) + } + + if stderr != "" { + klog.V(4).Infof("pcs status xml produced stderr: %s", stderr) + } + + var result pacemaker.PacemakerResult + if err := xml.Unmarshal([]byte(stdout), &result); err != nil { + return nil, fmt.Errorf("failed to parse pcs status xml: %w", err) + } + + // Extract nodes that are members (not remote nodes) + runtimeNodes := make(map[string]bool) + for _, node := range result.Nodes.Node { + if node.Name != "" && node.Type == "member" { + runtimeNodes[node.Name] = true + } + } + + // Build sorted list of node names for logging + nodeNames := make([]string, 0, len(runtimeNodes)) + for nodeName := range runtimeNodes { + nodeNames = append(nodeNames, nodeName) + } + sort.Strings(nodeNames) + + klog.V(2).Infof("Runtime pacemaker cluster members: %v", nodeNames) + return runtimeNodes, nil +} + +func getCurrentPacemakerNodesWithIPs(ctx context.Context) (map[string]string, error) { + // Get runtime membership (authoritative source for reconciliation) + runtimeNodes, err := getRuntimePacemakerNodes(ctx) + if err != nil { + return nil, fmt.Errorf("failed to fetch runtime cluster membership: %w", err) + } + + // Get config file (for IP addresses) + config, err := pacemaker.FetchClusterConfig(ctx) + if err != nil { + return nil, fmt.Errorf("failed to fetch cluster config: %w", err) + } + + // Build node map using runtime membership, pulling IPs from config file + nodeMap := make(map[string]string) + for nodeName := range runtimeNodes { + // Find IP from config file + var nodeIP string + for _, configNode := range config.Nodes { + if configNode.Name == nodeName { + if len(configNode.Addrs) > 0 { + nodeIP = configNode.Addrs[0].Addr + break + } + } + } + + if nodeIP == "" { + // Node in runtime but has no valid IP - mark with empty string + // DetermineReconciliationActions will detect IP mismatch and remove+add + klog.Warningf("Node %s in runtime cluster but has no valid IP in config file - will remove and re-add", nodeName) + } + + // Include all runtime nodes, even those with no IP (using empty string) + // This triggers removal+re-add for nodes in broken state + nodeMap[nodeName] = nodeIP + } + + if len(nodeMap) == 0 { + return nil, fmt.Errorf("no nodes found in runtime cluster") + } + + klog.V(2).Infof("Current pacemaker cluster members (runtime + IPs): %v", getMapKeys(nodeMap)) + return nodeMap, nil +} + +// getMapKeys returns sorted keys from a map +func getMapKeys(m map[string]string) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} diff --git a/pkg/tnf/update-setup/runner_test.go b/pkg/tnf/update-setup/runner_test.go new file mode 100644 index 0000000000..d71f003687 --- /dev/null +++ b/pkg/tnf/update-setup/runner_test.go @@ -0,0 +1,138 @@ +package updatesetup + +/* +TEST COVERAGE SUMMARY - runner_test.go +======================================= + +This file tests the update-setup runner's ConfigMap handling for TNF reconciliation. + +WHAT'S TESTED +------------- + +ConfigMap Selection (cluster-wide model): +└── TestPickLatestUpdateSetupConfigMap - Cluster-wide ConfigMap selection by generation + ├── ✅ Picks highest generation among multiple ConfigMaps + ├── ✅ Handles single ConfigMap + ├── ✅ Returns error when no ConfigMaps exist + └── ✅ Skips invalid generation strings + +WHAT'S NOT TESTED (covered elsewhere) +-------------------------------------- +- Reconciliation logic: tested in nodehandler_test.go (single source of truth) +- Helper functions (decodeStringList): implicitly tested through business logic +- Pacemaker command execution: integration tests +- ConfigMap creation: tested in lifecycle_manager_test.go +*/ + +import ( + "testing" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// TestPickLatestUpdateSetupConfigMap verifies that the runner picks the ConfigMap +// with the highest generation number (cluster-wide, no node filtering) +func TestPickLatestUpdateSetupConfigMap(t *testing.T) { + tests := []struct { + name string + items []corev1.ConfigMap + expectName string + expectGen string + expectError bool + expectErrorMsg string + }{ + { + name: "picks highest generation", + items: []corev1.ConfigMap{ + { + ObjectMeta: metav1.ObjectMeta{Name: "old"}, + Data: map[string]string{ + "generation": "3", + "eventType": "add", + }, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "newest"}, + Data: map[string]string{ + "generation": "10", + "eventType": "drift-reconciliation", + }, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "middle"}, + Data: map[string]string{ + "generation": "7", + "eventType": "delete", + }, + }, + }, + expectName: "newest", + expectGen: "10", + expectError: false, + }, + { + name: "single ConfigMap", + items: []corev1.ConfigMap{ + { + ObjectMeta: metav1.ObjectMeta{Name: "only"}, + Data: map[string]string{ + "generation": "5", + "eventType": "add", + }, + }, + }, + expectName: "only", + expectGen: "5", + expectError: false, + }, + { + name: "no ConfigMaps returns error", + items: []corev1.ConfigMap{}, + expectError: true, + expectErrorMsg: "no update-setup ConfigMap found", + }, + { + name: "skips invalid generation", + items: []corev1.ConfigMap{ + { + ObjectMeta: metav1.ObjectMeta{Name: "valid"}, + Data: map[string]string{ + "generation": "5", + "eventType": "add", + }, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "invalid"}, + Data: map[string]string{ + "generation": "not-a-number", + "eventType": "add", + }, + }, + }, + expectName: "valid", + expectGen: "5", + expectError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cm, err := pickLatestUpdateSetupConfigMap(tt.items) + + if tt.expectError { + require.Error(t, err) + if tt.expectErrorMsg != "" { + require.Contains(t, err.Error(), tt.expectErrorMsg) + } + } else { + require.NoError(t, err) + require.Equal(t, tt.expectName, cm.Name) + require.Equal(t, tt.expectGen, cm.Data["generation"]) + } + }) + } +} + +// TestDecodeStringList verifies JSON decoding of string lists from ConfigMap From 7d637d9f4ab48a20b569af10d0c1936ff65e6ca5 Mon Sep 17 00:00:00 2001 From: Jeremy Poulin Date: Thu, 25 Jun 2026 15:57:03 -0400 Subject: [PATCH 4/6] OCPBUGS-84695: feat(tnf): lifecycle management and health monitoring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements the PacemakerLifecycleManager controller for continuous monitoring and maintenance: - PacemakerLifecycleManager: main controller with 30s sync loop - Event handlers: Node Add/Update/Delete, PacemakerCluster Update - Health monitoring: * Tracks PacemakerCluster CR staleness (grace period based on last transition time) * Detects stale status from status collector failures * Records events for health transitions (Healthy→Warning, Warning→Error, etc.) * Prevents duplicate event spam with deduplication - Orphaned job cleanup: * Detects jobs for deleted nodes (node UID no longer exists) * Deletes jobs when node permanently removed * Logs cleanup actions - Status collector CronJob: * Creates/updates CronJob for collecting Pacemaker status * Runs every 30s on control plane nodes * Updates PacemakerCluster CR with health status - Replaces nodehandler.go with lifecycle-based approach See docs/tnf/controller-lifecycle.md for sync loop, event handling, and health monitoring details. Co-Authored-By: Claude Sonnet 4.5 --- bindata/tnfdeployment/cronjob.yaml | 1 + docs/tnf/controller-lifecycle.md | 790 +++++++++ .../0000_20_etcd-operator_06_deployment.yaml | 1 + pkg/tnf/operator/lifecycle_cleanup.go | 201 +++ pkg/tnf/operator/lifecycle_health.go | 448 +++++ pkg/tnf/operator/lifecycle_manager.go | 344 ++++ pkg/tnf/operator/lifecycle_manager_test.go | 1564 +++++++++++++++++ .../operator/lifecycle_status_collector.go | 222 +++ pkg/tnf/operator/nodehandler.go | 320 ---- pkg/tnf/operator/nodehandler_test.go | 343 ---- pkg/tnf/pkg/pacemaker/client_helpers.go | 4 +- pkg/tnf/pkg/pacemaker/healthcheck.go | 629 +------ pkg/tnf/pkg/pacemaker/healthcheck_test.go | 1388 --------------- pkg/tnf/pkg/pacemaker/statuscollector.go | 11 +- pkg/tnf/pkg/pacemaker/statuscollector_test.go | 2 +- pkg/tnf/pkg/pacemaker/test_helpers.go | 442 +++++ 16 files changed, 4072 insertions(+), 2638 deletions(-) create mode 100644 docs/tnf/controller-lifecycle.md create mode 100644 pkg/tnf/operator/lifecycle_cleanup.go create mode 100644 pkg/tnf/operator/lifecycle_health.go create mode 100644 pkg/tnf/operator/lifecycle_manager.go create mode 100644 pkg/tnf/operator/lifecycle_manager_test.go create mode 100644 pkg/tnf/operator/lifecycle_status_collector.go delete mode 100644 pkg/tnf/operator/nodehandler.go delete mode 100644 pkg/tnf/operator/nodehandler_test.go delete mode 100644 pkg/tnf/pkg/pacemaker/healthcheck_test.go create mode 100644 pkg/tnf/pkg/pacemaker/test_helpers.go diff --git a/bindata/tnfdeployment/cronjob.yaml b/bindata/tnfdeployment/cronjob.yaml index 282f8eea04..1b82ab373c 100644 --- a/bindata/tnfdeployment/cronjob.yaml +++ b/bindata/tnfdeployment/cronjob.yaml @@ -16,6 +16,7 @@ spec: metadata: labels: app.kubernetes.io/name: + app.kubernetes.io/component: two-node-fencing-setup spec: activeDeadlineSeconds: 90 ttlSecondsAfterFinished: 300 diff --git a/docs/tnf/controller-lifecycle.md b/docs/tnf/controller-lifecycle.md new file mode 100644 index 0000000000..1363d186ae --- /dev/null +++ b/docs/tnf/controller-lifecycle.md @@ -0,0 +1,790 @@ +# Controller Lifecycle + +## Overview + +The PacemakerLifecycleManager is a Kubernetes controller that runs continuously as part of the cluster-etcd-operator. It manages the integration between OpenShift control plane nodes and the Pacemaker high-availability cluster. + +## Startup Sequence + +### Operator Initialization + +```text +┌──────────────────────────────────────────────────────────────┐ +│ cluster-etcd-operator starts │ +└──────────────────────────────────────────────────────────────┘ + │ + ▼ + ┌────────────────────────────────┐ + │ Static resource controller │ + │ applies PacemakerCluster CRD │ + └────────────────────────────────┘ + │ + ▼ + ┌────────────────────────────────┐ + │ runPacemakerControllers() │ + │ (background goroutine) │ + └────────────────────────────────┘ + │ + ▼ + ┌────────────────────────────────┐ + │ Wait for CRD to be established │ + │ (polls with backoff) │ + └────────────────────────────────┘ + │ + ▼ + ┌────────────────────────────────┐ + │ NewPacemakerLifecycleManager() │ + └────────────────────────────────┘ + │ + ▼ + ┌────────────────────────────────┐ + │ Start PacemakerCluster informer│ + │ (go pacemakerInformer.Run()) │ + └────────────────────────────────┘ + │ + ▼ + ┌────────────────────────────────┐ + │ Start lifecycle controller │ + │ (go lifecycleController.Run()) │ + └────────────────────────────────┘ + │ + ▼ + ┌────────────────────────────────┐ + │ Create status collector CronJob│ + │ (periodic pcs status xml) │ + └────────────────────────────────┘ +``` + +**Key Points:** +- Lifecycle manager does NOT wait for external etcd transition before starting +- The `sync()` function handles both bootstrap and runtime modes internally +- CRD must be established before informers can start + +### Controller Initialization + +```text +┌──────────────────────────────────────────────────────────────┐ +│ NewPacemakerLifecycleManager() │ +└──────────────────────────────────────────────────────────────┘ + │ + ▼ + ┌────────────────────────────────┐ + │ Create REST client for │ + │ PacemakerCluster CR │ + └────────────────────────────────┘ + │ + ▼ + ┌────────────────────────────────┐ + │ Create SharedIndexInformer │ + │ - Resource: PacemakerCluster │ + │ - Resync: 30 seconds │ + └────────────────────────────────┘ + │ + ▼ + ┌────────────────────────────────┐ + │ Initialize struct: │ + │ - recordedEvents: empty map │ + │ - previous: nil │ + │ - lifecycleCtx: from context │ + └────────────────────────────────┘ + │ + ▼ + ┌────────────────────────────────┐ + │ Create factory.Controller │ + │ - Sync: c.sync │ + │ - Resync: 30 seconds │ + └────────────────────────────────┘ + │ + ▼ + ┌────────────────────────────────┐ + │ Register node event handlers │ + │ - AddFunc │ + │ - UpdateFunc │ + │ - DeleteFunc │ + └────────────────────────────────┘ + │ + ▼ + ┌────────────────────────────────┐ + │ Return: │ + │ - controller │ + │ - manager │ + │ - informer │ + └────────────────────────────────┘ +``` + +## Periodic Sync Loop + +The controller runs `sync()` every 30 seconds plus whenever informers detect changes. + +### Sync Flow + +```text +┌──────────────────────────────────────────────────────────────┐ +│ sync(ctx, syncCtx) - called every 30 seconds │ +└──────────────────────────────────────────────────────────────┘ + │ + ▼ + ┌────────────────────────────────┐ + │ Check ExternalEtcd │ + │ TransitionCompleted? │ + └────────────────────────────────┘ + │ + ▼ + ┌────────────────────────────────┐ + │ Initialize error aggregator │ + │ var errs []error │ + └────────────────────────────────┘ + │ + ▼ + ┌────────────────────────────────┐ + │ Operation 1: │ + │ StartJobControllers() │ + │ - Runs in both modes │ + │ - Append error if fails │ + └────────────────────────────────┘ + │ + ▼ + ┌────────────────────────────────┐ + │ Operation 2: │ + │ CleanupOrphanedJobs() │ + │ - Runs in both modes │ + │ - Deletes orphaned jobs │ + │ - Deletes orphaned pods │ + │ - Append error if fails │ + └────────────────────────────────┘ + │ + ▼ + Transition complete? + │ + ┌────┴────┐ + No Yes + │ │ + ▼ ▼ + ┌──────────────┐ ┌────────────────────┐ + │ Skip ops 3-4 │ │ Operation 3: │ + └──────────────┘ │ MonitorHealth() │ + │ - Append error │ + └────────────────────┘ + │ + ▼ + ┌────────────────────┐ + │ Operation 4: │ + │ Reconcile │ + │ PacemakerConfig() │ + │ - Append error │ + └────────────────────┘ + │ + ▼ + ┌────────────────────────────────┐ + │ Return errors.NewAggregate(errs)│ + └────────────────────────────────┘ +``` + +**Error Handling:** +- Each operation continues even if others fail +- Errors are aggregated and returned together +- Controller framework marks operator Degraded if errors persist +- Next sync attempt happens in 30 seconds + +**Why Independent Operations?** +- Health check failure shouldn't prevent reconciliation +- Job startup failure shouldn't prevent cleanup +- Partial progress is better than complete failure + +## Node Event Handlers + +The controller responds to Kubernetes node lifecycle events in addition to periodic sync. + +### AddFunc (Node Added) + +```text +Node added to cluster + │ + ▼ +Is it a control plane node? + │ + ┌───┴───┐ + No Yes + │ │ + ▼ ▼ + Ignore Log: "node added: " + │ + ▼ + Spawn goroutine + │ + ▼ + ReconcilePacemakerConfig(lifecycleCtx) + │ + ▼ + (Drift detection & reconciliation) +``` + +**Why Goroutine?** Event handlers must return quickly - reconciliation can take time (job creation, API calls). + +**Why ReconcilePacemakerConfig?** Adding a node creates drift (K8s has node, Pacemaker doesn't). + +### UpdateFunc (Node Updated) + +```text +Node updated + │ + ▼ +Is it a control plane node? + │ + ┌───┴───┐ + No Yes + │ │ + ▼ ▼ + Ignore Check transition type + │ + ┌───┴───────────────┐ + │ │ + NotReady→Ready IP changed while Ready + │ │ + ▼ ▼ + Spawn goroutine Spawn goroutine + │ │ + ▼ ▼ + StartJobControllers ReconcilePacemakerConfig + (lifecycleCtx) (lifecycleCtx) + │ │ + ▼ ▼ + (Bootstrap or (Drift detection & + ensure running) reconciliation) +``` + +**Two Triggers:** + +1. **NotReady→Ready:** Node became available + - Bootstrap mode: Trigger initial transition retry + - Runtime mode: Ensure jobs are running on newly Ready node + +2. **IP Changed:** Node IP address changed while Ready + - Trigger reconciliation to update Pacemaker with new IP + - Rare scenario but critical for correctness + +### DeleteFunc (Node Deleted) + +```text +Node deleted from cluster + │ + ▼ +Is it a control plane node? + │ + ┌───┴───┐ + No Yes + │ │ + ▼ ▼ + Ignore Check for tombstone + │ + ┌───┴───────────┐ + │ │ + Regular delete Tombstone + │ │ + └───────┬───────┘ + │ + ▼ + Log: "node deleted: " + │ + ▼ + Spawn goroutine + │ + ▼ + ReconcilePacemakerConfig(lifecycleCtx) + │ + ▼ + (Drift detection & reconciliation) +``` + +**Tombstone Handling:** If informer cache is cleared before handler runs, we get a tombstone. Handler extracts the node from tombstone to log the deletion. + +**Why ReconcilePacemakerConfig?** Deleting a node creates drift (Pacemaker has node, K8s doesn't). + +## Lifecycle Context Management + +The controller uses context to manage goroutine lifecycle spawned by event handlers. + +### Context Creation + +```text +NewPacemakerLifecycleManager() + │ + ▼ +lifecycleCtx, lifecycleCtxCancel = context.WithCancel(parentCtx) + │ + ▼ +Store in struct: + - lifecycleCtx: context.Context + - lifecycleCtxCancel: context.CancelFunc +``` + +### Context Usage + +```text +Event handler fires (Add/Update/Delete) + │ + ▼ +Spawn goroutine + │ + ▼ +Pass lifecycleCtx to operation: + - StartJobControllers(c.lifecycleCtx) + - ReconcilePacemakerConfig(c.lifecycleCtx) + │ + ▼ +Operation uses context for: + - API calls (can be cancelled) + - Job creation (can be cancelled) + - Reconciliation (can be cancelled) +``` + +### Context Cancellation (Shutdown) + +```text +Controller shutdown initiated + │ + ▼ +lifecycleCtxCancel() called + │ + ▼ +All goroutines spawned by event handlers +receive cancellation signal + │ + ▼ +Operations abort gracefully: + - API calls return context.Canceled + - Goroutines exit + - No goroutine leaks +``` + +**Why Important?** Without proper context management, event handler goroutines would leak during operator restarts. + +## Informer Resync + +Both the PacemakerCluster informer and the controller factory use a 30-second resync interval. + +### What Resync Does + +```text +Every 30 seconds: + │ + ▼ +Informer re-lists resources from API server + │ + ▼ +Compares with local cache + │ + ┌───┴───────────┐ + │ │ + Changed Unchanged + │ │ + ▼ ▼ +Trigger sync() No action +``` + +**Why 30 Seconds?** +- Balance between responsiveness and API load +- Matches Pacemaker health check frequency +- Provides eventual consistency even if events are missed + +**What It Recovers:** +- Missed informer events (network blip, operator restart) +- Stale cache state (API server changes not propagated) +- Operator downtime (catches up on changes during restart) + +## Controller Framework Integration + +The lifecycle manager integrates with the OpenShift library-go controller framework. + +### Factory Controller + +```text +factory.Controller created with: + │ + ├─ Sync function: c.sync + │ + ├─ Informers: + │ - operatorClient (etcd operator status) + │ - pacemakerInformer (PacemakerCluster CR) + │ - nodeInformer (K8s nodes) + │ + └─ Resync interval: 30 seconds + │ + ▼ + Controller framework runs: + │ + ├─ Call sync() periodically + │ + ├─ Call sync() on informer updates + │ + ├─ Track degraded conditions + │ + └─ Implement exponential backoff on errors +``` + +### Degraded Condition Handling + +```text +sync() returns error + │ + ▼ +Controller framework checks error + │ + ┌───┴───────────┐ + │ │ + nil error Has error + │ │ + ▼ ▼ +Clear degraded Set degraded condition +condition in operator status + │ │ + ▼ ▼ +Available=True Degraded=True + Available=False +``` + +**Automatic Retry:** Framework retries failed sync operations with exponential backoff. + +## Shutdown Sequence + +```text +Operator shutdown signal received + │ + ▼ +Stop accepting new informer events + │ + ▼ +Cancel lifecycleCtx + │ + ▼ +Wait for in-flight goroutines to exit + │ + ▼ +Stop informers + │ + ▼ +Stop controller + │ + ▼ +Cleanup complete +``` + +**Graceful Shutdown:** In-flight operations receive cancellation signal and can clean up before exiting. + +## Concurrency Model + +The controller uses multiple synchronization primitives to handle concurrent operations safely. + +### Mutex Overview + +| Mutex | Scope | Protects | Location | +|-------|-------|----------|----------| +| recordedEventsMu | Instance | Event deduplication map | lifecycle_health.go | +| previousMu | Instance | Previous health status | lifecycle_health.go | +| startJobControllersMu | Instance | Job controller startup | lifecycle_job_controllers.go | +| reconcilePacemakerConfigMutex | Package | Reconciliation + generation counter | lifecycle_reconciliation.go | + +### Concurrent Callers + +```text +Periodic Sync (every 30s) + │ + ├─> StartJobControllers() + ├─> CleanupOrphanedJobs() + │ ├─> cleanupOrphanedJobs() + │ └─> cleanupOrphanedPods() + ├─> MonitorHealth() [if transition complete] + └─> ReconcilePacemakerConfig() [if transition complete] + +Node Add Event + │ + └─> ReconcilePacemakerConfig() + +Node Update Event (Ready) + │ + └─> StartJobControllers() + +Node Update Event (IP change) + │ + └─> ReconcilePacemakerConfig() + +Node Delete Event + │ + └─> ReconcilePacemakerConfig() +``` + +**All operations can run concurrently** - mutexes protect critical sections. + +## Health Monitoring + +Health monitoring checks the Pacemaker cluster status and reports it to the OpenShift operator framework. It runs every 30 seconds as part of the periodic sync loop and only operates after the external etcd transition completes. + +### When Health Monitoring Runs + +```text +sync() called (every 30 seconds) + │ + ▼ +Check ExternalEtcdTransitionCompleted + │ + ┌───┴───┐ + No Yes + │ │ + ▼ ▼ + Skip MonitorHealth() + (health monitoring) +``` + +**Why Only After Transition?** Before transition, Pacemaker cluster doesn't exist yet - nothing to monitor. + +### Health Monitoring Flow + +```text +┌──────────────────────────────────────────────────────────────┐ +│ MonitorHealth(ctx) │ +└──────────────────────────────────────────────────────────────┘ + │ + ▼ + ┌────────────────────────────────┐ + │ getPacemakerStatus() │ + │ - Read PacemakerCluster CR │ + │ - Check staleness │ + │ - Build health status │ + └────────────────────────────────┘ + │ + ▼ + ┌────────────────────────────────┐ + │ updateOperatorStatus() │ + │ - Set Degraded condition │ + │ - Set Progressing condition │ + │ - Set Available condition │ + └────────────────────────────────┘ + │ + ▼ + ┌────────────────────────────────┐ + │ recordHealthCheckEvents() │ + │ - Cleanup expired events │ + │ - Record warnings │ + │ - Record errors │ + └────────────────────────────────┘ + │ + ▼ + ┌────────────────────────────────┐ + │ recordHealthTransitionEvents() │ + │ - Detect state changes │ + │ - Record transition events │ + └────────────────────────────────┘ +``` + +### Get Pacemaker Status + +Status is retrieved from the PacemakerCluster custom resource: + +```text +┌──────────────────────────────────────────────────────────────┐ +│ getPacemakerStatus() │ +└──────────────────────────────────────────────────────────────┘ + │ + ▼ + ┌────────────────────────────────┐ + │ Read PacemakerCluster CR │ + │ from informer cache │ + └────────────────────────────────┘ + │ + ┌───────┴────────┐ + │ │ + CR exists CR not found + │ │ + ▼ ▼ + ┌────────────────┐ ┌──────────────────┐ + │ Check staleness│ │ Return Unknown │ + │ (5 min window) │ │ health status │ + └────────────────┘ └──────────────────┘ + │ + ┌────────┴────────┐ + │ │ + Not stale Stale + │ │ + ▼ ▼ +┌────────────────┐ ┌──────────────────┐ +│ Build health │ │ Return Unknown │ +│ status from CR │ │ health status │ +└────────────────┘ └──────────────────┘ +``` + +**Staleness Check:** CR status is considered stale if `lastUpdated` is more than 5 minutes old. Status collector runs every minute, so missing 5 consecutive updates indicates a problem. + +**Health States:** +- **Active:** Pacemaker cluster is healthy (all nodes online, quorum established, no errors) +- **Degraded:** Pacemaker cluster has issues (node offline, fencing failures, resource failures) +- **Unknown:** Cannot determine health (CR doesn't exist, status is stale, parsing error) + +### Update Operator Status + +Operator conditions are set based on health status: + +```text +Health Status Degraded Progressing Available +───────────────────────────────────────────────────────── +Active False False True +Degraded True False False +Unknown False True False +``` + +**Condition Messages:** +- Active: `"Pacemaker cluster is active with 2 nodes online"` +- Degraded: `"Node master-1 is offline; Fencing failure on node master-0"` + +### Event Recording with Deduplication + +Events are recorded for warnings and errors with deduplication to prevent spam. + +**Deduplication Windows:** + +| Event Type | Window | Reason | +|------------|--------|--------| +| Default | 5 minutes | Prevent log spam for transient issues | +| Fencing | 24 hours | Fencing events are critical - record daily until resolved | + +**Event Flow:** + +```text +Event occurs: "Node master-1 is offline" + │ + ▼ +Generate event key: "warning:Node master-1 is offline" + │ + ▼ +Check recordedEvents map + │ + ┌───┴────────────────┐ + │ │ +Key exists Key doesn't exist + │ │ + ▼ ▼ +Last recorded < 5min Record event + │ │ + ▼ ▼ +Skip (duplicate) Add to map with timestamp +``` + +**Event Cleanup:** Every `MonitorHealth()` call removes expired events from the deduplication map to prevent unbounded growth. + +### Health State Transitions + +State transitions between Active, Degraded, and Unknown are recorded as separate events: + +```text +┌──────────────────────────────────────────────────────────────┐ +│ recordHealthTransitionEvents(current) │ +└──────────────────────────────────────────────────────────────┘ + │ + ▼ + ┌────────────────────────────────┐ + │ Get previous health status │ + │ (protected by previousMu) │ + └────────────────────────────────┘ + │ + ┌───────┴────────┐ + │ │ + previous = nil previous exists + │ │ + ▼ ▼ + ┌────────────────┐ ┌──────────────────┐ + │ Record: │ │ Compare states │ + │ "Unknown→ │ └──────────────────┘ + │ " │ │ + └────────────────┘ ┌────┴────┐ + │ │ + Same state Different state + │ │ + ▼ ▼ + Skip Record transition + Update previous +``` + +**Example Transition Events:** +- `"Pacemaker health transitioned from Unknown to Active"` (Type: Normal) +- `"Pacemaker health transitioned from Active to Degraded"` (Type: Warning) +- `"Pacemaker health transitioned from Degraded to Active"` (Type: Normal) + +### Integration with PacemakerCluster CR + +The PacemakerCluster CR is populated by a status collector CronJob that runs every minute: + +```text +Every 1 minute: + │ + ▼ +┌────────────────────────────────┐ +│ Status Collector CronJob │ +│ (runs on pacemaker node) │ +└────────────────────────────────┘ + │ + ▼ +Run: pcs status xml + │ + ▼ +Parse XML into structured data + │ + ▼ +Update PacemakerCluster CR .status + │ + ▼ +Set .status.lastUpdated timestamp +``` + +**CR Status Fields:** `state`, `lastUpdated`, `nodes[]` (name, status, IP), `warnings[]`, `errors[]`, `clusterSummary` + +### Health vs Reconciliation + +Health monitoring and reconciliation are independent operations with different purposes: + +**Health Monitoring:** +- Purpose: Observability - report status to operators +- Actions: Read CR, update conditions, record events +- Does NOT: Modify Pacemaker, create jobs, trigger reconciliation + +**Reconciliation:** +- Purpose: Operational correctness - fix drift +- Actions: Compare K8s vs Pacemaker, create jobs, add/remove nodes +- Does NOT: Update operator conditions, record health events + +**Why Separate?** Health check failure shouldn't prevent drift correction, and vice versa. Both run independently in `sync()` for partial progress. + +## Observability + +The controller provides multiple observability mechanisms: + +### Logging + +- **klog.V(2):** Important state transitions (job startup, reconciliation) +- **klog.V(4):** Verbose debugging (sync start/end, early returns) +- **klog.Info:** Significant events (drift detected, jobs started) +- **klog.Warning:** Unexpected but recoverable conditions (stale CR, missing nodes) +- **klog.Error:** Errors requiring attention (API failures, job creation failures) + +### Operator Conditions + +Reported via StaticPodOperatorClient: + +- **TNFJobControllersDegraded:** Set during bootstrap if job startup fails after retry +- **PacemakerHealthDegraded:** Set if Pacemaker cluster is unhealthy +- **PacemakerHealthProgressing:** Set if Pacemaker is transitioning between states + +### Kubernetes Events + +Recorded via EventRecorder: + +- Health state transitions (Unknown→Active, Active→Degraded) +- Fencing failures (repeated every 24 hours if not resolved) +- Configuration drift detected +- Job completion/failure + +### Metrics + +Status exposed via PacemakerCluster CR: +- Cluster health (Active, Degraded, Unknown) +- Node membership list +- Last update timestamp +- Error messages diff --git a/manifests/0000_20_etcd-operator_06_deployment.yaml b/manifests/0000_20_etcd-operator_06_deployment.yaml index a1a7393f36..339e6c18d0 100644 --- a/manifests/0000_20_etcd-operator_06_deployment.yaml +++ b/manifests/0000_20_etcd-operator_06_deployment.yaml @@ -56,6 +56,7 @@ spec: - "--terminate-on-files=/var/run/secrets/etcd-client/tls.key" - "--terminate-on-files=/var/run/configmaps/etcd-ca/ca-bundle.crt" - "--terminate-on-files=/var/run/configmaps/etcd-service-ca/service-ca.crt" + - "-v=4" resources: requests: memory: 50Mi diff --git a/pkg/tnf/operator/lifecycle_cleanup.go b/pkg/tnf/operator/lifecycle_cleanup.go new file mode 100644 index 0000000000..1649ea7df0 --- /dev/null +++ b/pkg/tnf/operator/lifecycle_cleanup.go @@ -0,0 +1,201 @@ +package operator + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/klog/v2" + + "github.com/openshift/cluster-etcd-operator/pkg/operator/ceohelpers" + "github.com/openshift/cluster-etcd-operator/pkg/operator/operatorclient" +) + +const ( + // maxPodDeletionsPerSync limits orphaned pod deletions per sync to avoid API rate limiting. + // At 30s sync interval, 50 pods/sync = ~100 pods/minute cleanup rate. + maxPodDeletionsPerSync = 50 +) + +// CleanupOrphanedJobs cleans up TNF jobs for nodes that no longer exist in K8s. +// This is called periodically from sync() to catch missed delete events. +func (c *PacemakerLifecycleManager) CleanupOrphanedJobs(ctx context.Context) error { + // Check if node informer has synced + if c.nodeInformer == nil || !c.nodeInformer.HasSynced() { + klog.V(4).Infof("Skipping orphaned job cleanup - node informer not synced yet") + return nil + } + + // Get K8s control plane nodes + k8sNodes, err := ceohelpers.ListNodesFromInformer(c.nodeInformer) + if err != nil { + return fmt.Errorf("failed to list control plane nodes: %w", err) + } + + // Call helpers to perform cleanup + var errs []error + if err := c.cleanupOrphanedJobs(ctx, k8sNodes); err != nil { + errs = append(errs, err) + } + if err := c.cleanupOrphanedPods(ctx); err != nil { + errs = append(errs, err) + } + return errors.NewAggregate(errs) +} + +// cleanupOrphanedJobs deletes TNF jobs for nodes that no longer exist in K8s. +// This is called periodically during reconciliation to catch missed delete events. +func (c *PacemakerLifecycleManager) cleanupOrphanedJobs(ctx context.Context, k8sNodes []*corev1.Node) error { + // Build set of current node UIDs + currentNodeUIDs := make(map[string]bool) + for _, node := range k8sNodes { + currentNodeUIDs[string(node.UID)] = true + } + + // List all node-specific TNF jobs in openshift-etcd namespace + // Note: update-setup, setup, and fencing jobs are cluster-wide (no node label), so we don't clean them up here + jobList, err := c.kubeClient.BatchV1().Jobs(operatorclient.TargetNamespace).List(ctx, metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/name in (tnf-auth-job,tnf-after-setup-job)", + }) + if err != nil { + return fmt.Errorf("failed to list TNF jobs: %w", err) + } + + // Check each job to see if its node still exists + orphanedCount := 0 + var errs []error + for _, job := range jobList.Items { + var shouldDelete bool + var deleteReason string + + // Get node UID from job label (not node name - UIDs are stable across node replacements). + // If a node is deleted and re-added with the same name but different UID, + // jobs labeled with the old UID should be cleaned up. + jobNodeUID, ok := job.Labels["node"] + if !ok { + // Old job without node UID label (created before label was added). + // Delete it - jobs are idempotent and will be recreated with proper labels. + shouldDelete = true + deleteReason = "old job without node UID label (migration cleanup)" + } else if !currentNodeUIDs[jobNodeUID] { + // Node doesn't exist or was replaced - delete orphaned job + shouldDelete = true + deleteReason = fmt.Sprintf("node UID %s no longer exists", jobNodeUID) + } + + if shouldDelete { + klog.Infof("Deleting orphaned TNF job %s: %s", job.Name, deleteReason) + + // Delete the orphaned job + deletePolicy := metav1.DeletePropagationBackground + err := c.kubeClient.BatchV1().Jobs(operatorclient.TargetNamespace).Delete(ctx, job.Name, metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }) + if err != nil && !apierrors.IsNotFound(err) { + klog.Errorf("Failed to delete orphaned job %s: %v", job.Name, err) + errs = append(errs, fmt.Errorf("failed to delete job %s: %w", job.Name, err)) + continue + } + orphanedCount++ + } + } + + if orphanedCount > 0 { + klog.Infof("Cleaned up %d orphaned TNF jobs", orphanedCount) + } else { + klog.V(4).Infof("No orphaned TNF jobs found") + } + + return errors.NewAggregate(errs) +} + +// cleanupOrphanedPods deletes pods whose owner Jobs no longer exist. +// This handles cases where Job deletion with propagationPolicy:Background doesn't clean up pods, +// or where pods get stuck after Job deletion. +func (c *PacemakerLifecycleManager) cleanupOrphanedPods(ctx context.Context) error { + // List all TNF pods by their pod template label (app=tnf-job) + // Note: Pods inherit labels from Job.spec.template.metadata.labels, not Job.metadata.labels + podList, err := c.kubeClient.CoreV1().Pods(operatorclient.TargetNamespace).List(ctx, metav1.ListOptions{ + LabelSelector: "app=tnf-job", + }) + if err != nil { + return fmt.Errorf("failed to list TNF pods for cleanup: %w", err) + } + + // Build set of existing TNF jobs (use Job-level label) + jobList, err := c.kubeClient.BatchV1().Jobs(operatorclient.TargetNamespace).List(ctx, metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/component=two-node-fencing-setup", + }) + if err != nil { + return fmt.Errorf("failed to list TNF jobs for pod cleanup: %w", err) + } + + existingJobs := make(map[string]bool) + for _, job := range jobList.Items { + existingJobs[job.Name] = true + } + + // Delete pods whose owner Job no longer exists OR pods with no owner in terminal state + deletedCount := 0 + var errs []error + for _, pod := range podList.Items { + // Stop deleting if we've hit the per-sync limit to avoid API rate limiting + if deletedCount >= maxPodDeletionsPerSync { + klog.Infof("Reached max pod deletion limit (%d) for this sync - remaining orphaned pods will be cleaned in next sync", maxPodDeletionsPerSync) + break + } + + shouldDelete := false + var reason string + + // Case 1: Pod has owner reference but owner Job no longer exists + if len(pod.OwnerReferences) > 0 { + ownerJob := pod.OwnerReferences[0].Name + if !existingJobs[ownerJob] { + shouldDelete = true + reason = fmt.Sprintf("owner job %s no longer exists", ownerJob) + } + } else { + // Case 2: Pod has NO owner reference and is in terminal state (Succeeded/Failed) + // These are orphaned pods from deleted Jobs that didn't clean up properly + if pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed { + shouldDelete = true + reason = "no owner reference and pod in terminal state" + } else { + klog.V(4).Infof("Pod %s has no owner references but is not in terminal state (%s), skipping", pod.Name, pod.Status.Phase) + } + } + + if shouldDelete { + klog.V(2).Infof("Deleting orphaned TNF pod %s (%s)", pod.Name, reason) + + // Try normal deletion first + err := c.kubeClient.CoreV1().Pods(operatorclient.TargetNamespace).Delete(ctx, pod.Name, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + // Normal deletion failed - try force deletion with zero grace period + klog.Warningf("Normal delete failed for pod %s, attempting force delete: %v", pod.Name, err) + gracePeriod := int64(0) + forceDeleteErr := c.kubeClient.CoreV1().Pods(operatorclient.TargetNamespace).Delete(ctx, pod.Name, metav1.DeleteOptions{ + GracePeriodSeconds: &gracePeriod, + }) + if forceDeleteErr != nil && !apierrors.IsNotFound(forceDeleteErr) { + klog.Errorf("Failed to force delete orphaned pod %s: %v", pod.Name, forceDeleteErr) + errs = append(errs, fmt.Errorf("failed to delete pod %s: %w", pod.Name, forceDeleteErr)) + continue + } + } + deletedCount++ + } + } + + if deletedCount > 0 { + klog.Infof("Cleaned up %d orphaned TNF pods", deletedCount) + } else { + klog.V(4).Infof("No orphaned pods to clean up") + } + + return errors.NewAggregate(errs) +} diff --git a/pkg/tnf/operator/lifecycle_health.go b/pkg/tnf/operator/lifecycle_health.go new file mode 100644 index 0000000000..b1d723cf64 --- /dev/null +++ b/pkg/tnf/operator/lifecycle_health.go @@ -0,0 +1,448 @@ +package operator + +import ( + "context" + "fmt" + "strings" + "time" + + operatorv1 "github.com/openshift/api/operator/v1" + "github.com/openshift/library-go/pkg/operator/v1helpers" + "k8s.io/klog/v2" + + pacmkrv1 "github.com/openshift/api/etcd/v1" + "github.com/openshift/cluster-etcd-operator/pkg/tnf/pkg/pacemaker" +) + +const ( + // Degraded condition reason + reasonPacemakerUnhealthy = "PacemakerUnhealthy" + + // Warning message prefixes for categorizing events. + warningPrefixFencingEvent = "Recent fencing event:" + + // Operator condition types + conditionTypePacemakerDegraded = "PacemakerHealthCheckDegraded" + + // Event key prefixes + eventKeyPrefixWarning = "warning:" + eventKeyPrefixError = "error:" + + // Event message templates + msgPacemakerDegraded = "Pacemaker cluster is in degraded state" + msgPacemakerHealthy = "Pacemaker cluster is healthy - all nodes have healthy critical resources" + msgPacemakerWarningsCleared = "Pacemaker cluster warnings cleared" + msgDetectedFencing = "Pacemaker detected a recent fencing event: %s" + msgPacemakerWarning = "Pacemaker warning: %s" + msgPacemakerError = "Pacemaker error: %s" +) + +// MonitorHealth retrieves pacemaker health status, updates operator conditions, and records events. +// Returns nil if no status change or monitoring completed successfully. +func (c *PacemakerLifecycleManager) MonitorHealth(ctx context.Context) error { + // Get current pacemaker status from CR + currentStatus, previousStatus, err := c.getPacemakerStatus(ctx) + if err != nil { + // Note: getPacemakerStatus currently never returns error (always returns Unknown status instead) + // This is defensive coding in case that changes + return fmt.Errorf("failed to get pacemaker status: %w", err) + } + + if currentStatus == nil { + // nil status means CR hasn't changed since last sync - skip processing to avoid redundant work + klog.V(4).Infof("Pacemaker status unchanged since last sync") + return nil + } + + klog.V(2).Infof("Pacemaker health status: %s (errors: %d, warnings: %d)", + currentStatus.OverallStatus, len(currentStatus.Errors), len(currentStatus.Warnings)) + + // Update operator condition based on health status + var statusErr error + if err := c.updateOperatorStatus(ctx, currentStatus, previousStatus); err != nil { + // Don't block event recording on status update failure + klog.Warningf("Failed to update operator status (continuing with event recording): %v", err) + statusErr = err // Capture for return + } + + // Record health check events (with deduplication) - always run even if status update failed + c.recordHealthCheckEvents(currentStatus, previousStatus) + + return statusErr // Return error for aggregation so sync loop can observe failures +} + +// getPacemakerStatus retrieves the current pacemaker health status from the PacemakerCluster CR. +// Returns (current, previous, error). +// current is nil if CR timestamp hasn't changed (skip redundant processing). +// previous is the last valid (non-Unknown) status, used for grace period calculation. +// For Unknown status, previous is not updated (preserves last valid status for grace period). +func (c *PacemakerLifecycleManager) getPacemakerStatus(ctx context.Context) (*pacemaker.HealthStatus, *pacemaker.HealthStatus, error) { + klog.V(4).Infof("Retrieving pacemaker status from CR...") + + // Read previous status + c.previousMu.Lock() + previous := c.previous + c.previousMu.Unlock() + + // Get the PacemakerCluster CR from the informer cache + item, exists, err := c.pacemakerInformer.GetStore().GetByKey(pacemaker.PacemakerClusterResourceName) + if err != nil { + // Unknown status - don't update previous (preserves last valid for grace period) + return &pacemaker.HealthStatus{ + OverallStatus: pacemaker.StatusUnknown, + Warnings: []string{}, + Errors: []string{fmt.Sprintf("Failed to get PacemakerCluster CR from cache: %v", err)}, + }, previous, nil + } + if !exists { + // Unknown status - CR not found in cache + return &pacemaker.HealthStatus{ + OverallStatus: pacemaker.StatusUnknown, + Warnings: []string{}, + Errors: []string{"PacemakerCluster CR not found in cache"}, + }, previous, nil + } + + pacemakerCR, ok := item.(*pacmkrv1.PacemakerCluster) + if !ok { + return &pacemaker.HealthStatus{ + OverallStatus: pacemaker.StatusUnknown, + Warnings: []string{}, + Errors: []string{"Failed to convert cached item to PacemakerCluster"}, + }, previous, nil + } + + // Check if status is populated (LastUpdated is zero means status was never set) + if pacemakerCR.Status.LastUpdated.IsZero() { + // Unknown status - don't update previous + return &pacemaker.HealthStatus{ + OverallStatus: pacemaker.StatusUnknown, + Warnings: []string{}, + Errors: []string{"PacemakerCluster CR has no status populated"}, + }, previous, nil + } + + crLastUpdated := pacemakerCR.Status.LastUpdated.Time + + // Check staleness first - any update (even with errors) clears staleness. + timeSinceUpdate := time.Since(crLastUpdated) + if timeSinceUpdate > pacemaker.StatusStalenessThreshold { + // Unknown status - don't update previous + // Use absolute timestamp (stable) for event deduplication. + return &pacemaker.HealthStatus{ + OverallStatus: pacemaker.StatusUnknown, + Warnings: []string{}, + Errors: []string{fmt.Sprintf("Pacemaker status is stale (last updated: %s)", crLastUpdated.Format(time.RFC3339))}, + }, previous, nil + } + + // Check if lastUpdated timestamp has changed since last sync. + // If unchanged, skip processing to avoid redundant work on timer-triggered syncs. + // Use previous.CRLastUpdated for comparison (only set for non-Unknown status). + if previous != nil && !previous.CRLastUpdated.IsZero() && crLastUpdated.Equal(previous.CRLastUpdated) { + klog.V(4).Infof("Skipping sync: lastUpdated timestamp unchanged (%v)", crLastUpdated) + return nil, nil, nil + } + + // Build health status from the CRD status fields + currentStatus := pacemaker.BuildHealthStatusFromCR(pacemakerCR) + currentStatus.CRLastUpdated = crLastUpdated + + // Only update previous for non-Unknown status (preserves last valid for grace period) + // Concurrency safety: check timestamp when updating to prevent stale writes from overwriting fresh data. + // Two concurrent syncs can read c.previous, process different CR versions, then both try to update. + // We only update if currentStatus is fresher than what's already in c.previous. + if currentStatus.OverallStatus != pacemaker.StatusUnknown { + c.previousMu.Lock() + // Only update if we have fresher data (or c.previous is nil/uninitialized) + if c.previous == nil || c.previous.CRLastUpdated.IsZero() || currentStatus.CRLastUpdated.After(c.previous.CRLastUpdated) { + c.previous = currentStatus + } + c.previousMu.Unlock() + } + + return currentStatus, previous, nil +} + +// updateOperatorStatus processes the pacemaker.HealthStatus and conditionally updates the operator state. +// previous is the last non-Unknown health status (used for grace period calculation). +func (c *PacemakerLifecycleManager) updateOperatorStatus(ctx context.Context, status *pacemaker.HealthStatus, previous *pacemaker.HealthStatus) error { + klog.V(4).Infof("Updating operator availability based on pacemaker status: %s", status.OverallStatus) + + // Log warnings and errors for visibility + for _, warning := range status.Warnings { + klog.Warningf(msgPacemakerWarning, warning) + } + for _, err := range status.Errors { + klog.Errorf(msgPacemakerError, err) + } + + // Update operator conditions based on pacemaker status + switch status.OverallStatus { + case pacemaker.StatusError: + return c.setPacemakerDegradedCondition(ctx, status) + case pacemaker.StatusHealthy, pacemaker.StatusWarning: + // Both healthy and warning states should clear degraded condition + // Warnings are informational (e.g. recent fencing, node count mismatch) and don't indicate degradation + if status.OverallStatus == pacemaker.StatusWarning { + klog.V(2).Infof("Pacemaker health check has warnings but cluster is operational: %v", status.Warnings) + } + return c.clearPacemakerDegradedCondition(ctx, status) + case pacemaker.StatusUnknown: + // Unknown status means we cannot determine pacemaker health (CR not found, stale, no status, etc.) + // Only mark as degraded if we haven't received a valid status in a while (grace period). + // Use previous.CRLastUpdated which reflects when we last had valid cluster data. + // If previous is nil (first sync), don't degrade immediately. + if previous == nil || previous.CRLastUpdated.IsZero() { + klog.V(2).Infof("Pacemaker health check cannot determine status (no previous valid status), not marking degraded yet: %v", + status.Errors) + return nil + } + + timeSinceLastValid := time.Since(previous.CRLastUpdated) + if timeSinceLastValid > pacemaker.StatusUnknownDegradedThreshold { + klog.Warningf("Pacemaker health check cannot determine status for %v (threshold: %v), marking as degraded: %v", + timeSinceLastValid, pacemaker.StatusUnknownDegradedThreshold, status.Errors) + return c.setPacemakerDegradedCondition(ctx, status) + } + + // Still within grace period, just log + klog.V(2).Infof("Pacemaker health check cannot determine status for %v (threshold: %v), not marking degraded yet: %v", + timeSinceLastValid, pacemaker.StatusUnknownDegradedThreshold, status.Errors) + return nil + default: + // This should never happen, but log it if it does + klog.Errorf("Unexpected pacemaker health status: %s (errors: %v, warnings: %v)", + status.OverallStatus, status.Errors, status.Warnings) + return nil + } +} + +// setPacemakerDegradedCondition sets the PacemakerHealthCheckDegraded condition to True. +func (c *PacemakerLifecycleManager) setPacemakerDegradedCondition(ctx context.Context, status *pacemaker.HealthStatus) error { + message := strings.Join(status.Errors, "; ") + if message == "" { + message = msgPacemakerDegraded + } + + // Check if the condition is already set with the same message + currentCondition, err := c.getCurrentPacemakerCondition() + if err != nil { + return err + } + + // Only update if the condition is not already set to True with the same message + if currentCondition != nil && + currentCondition.Status == operatorv1.ConditionTrue && + currentCondition.Message == message { + klog.V(4).Infof("Pacemaker degraded condition already set with same message, skipping update") + return nil + } + + condition := operatorv1.OperatorCondition{ + Type: conditionTypePacemakerDegraded, + Status: operatorv1.ConditionTrue, + Reason: reasonPacemakerUnhealthy, + Message: message, + } + + return c.updateOperatorCondition(ctx, condition) +} + +// clearPacemakerDegradedCondition sets the PacemakerHealthCheckDegraded condition to False. +func (c *PacemakerLifecycleManager) clearPacemakerDegradedCondition(ctx context.Context, status *pacemaker.HealthStatus) error { + // Check the current condition state + currentCondition, err := c.getCurrentPacemakerCondition() + if err != nil { + return err + } + + // Skip if condition is already explicitly set to False (no change needed) + if currentCondition != nil && currentCondition.Status == operatorv1.ConditionFalse { + klog.V(4).Infof("Pacemaker degraded condition is already False, skipping update") + return nil + } + + // Set condition to False in two cases: + // 1. Condition doesn't exist yet (first healthy check - make health status explicit) + // 2. Condition is currently True (transitioning from degraded to healthy) + condition := operatorv1.OperatorCondition{ + Type: conditionTypePacemakerDegraded, + Status: operatorv1.ConditionFalse, + } + + err = c.updateOperatorCondition(ctx, condition) + if err != nil { + return err + } + + if currentCondition == nil { + klog.Infof("Pacemaker health check condition initialized: PacemakerHealthCheckDegraded=False") + } else { + klog.Infof("Pacemaker degraded condition cleared (transitioned from True to False)") + } + return nil +} + +// getCurrentPacemakerCondition retrieves the current PacemakerHealthCheckDegraded condition +func (c *PacemakerLifecycleManager) getCurrentPacemakerCondition() (*operatorv1.OperatorCondition, error) { + _, currentStatus, _, err := c.operatorClient.GetStaticPodOperatorState() + if err != nil { + return nil, fmt.Errorf("failed to get current operator status: %w", err) + } + + // Find the current pacemaker degraded condition + for i := range currentStatus.Conditions { + if currentStatus.Conditions[i].Type == conditionTypePacemakerDegraded { + return ¤tStatus.Conditions[i], nil + } + } + + return nil, nil +} + +// updateOperatorCondition updates the operator condition +func (c *PacemakerLifecycleManager) updateOperatorCondition(ctx context.Context, condition operatorv1.OperatorCondition) error { + _, _, err := v1helpers.UpdateStaticPodStatus(ctx, c.operatorClient, v1helpers.UpdateStaticPodConditionFn(condition)) + if err != nil { + klog.Errorf("Failed to update operator status: %v", err) + return err + } + + klog.V(2).Infof("Updated operator condition: %s=%s", condition.Type, condition.Status) + return nil +} + +// cleanupExpiredEvents removes events from the deduplication map that have exceeded their window. +// Fencing events are kept for 24 hours, other events for 5 minutes. +func (c *PacemakerLifecycleManager) cleanupExpiredEvents() { + c.recordedEventsMu.Lock() + defer c.recordedEventsMu.Unlock() + + now := time.Now() + + for key, timestamp := range c.recordedEvents { + // Determine the appropriate window based on event type + // Fencing events have longer window + window := pacemaker.EventDeduplicationWindowDefault + if strings.Contains(key, warningPrefixFencingEvent) { + window = pacemaker.EventDeduplicationWindowFencing + } + + // Remove if expired + if now.Sub(timestamp) > window { + delete(c.recordedEvents, key) + } + } +} + +// shouldRecordEvent checks if an event should be recorded based on deduplication logic. +// This should be called after cleanupExpiredEvents() has been called. +func (c *PacemakerLifecycleManager) shouldRecordEvent(eventKey string, deduplicationWindow time.Duration) bool { + c.recordedEventsMu.Lock() + defer c.recordedEventsMu.Unlock() + + now := time.Now() + + // Check if this event was recently recorded + if lastRecorded, exists := c.recordedEvents[eventKey]; exists { + if now.Sub(lastRecorded) < deduplicationWindow { + return false + } + } + + // Record this event + c.recordedEvents[eventKey] = now + return true +} + +// recordHealthCheckEvents records events for warnings, errors, and health transitions. +// current is the newly computed health status, previous is the last health status (may be nil on first sync). +func (c *PacemakerLifecycleManager) recordHealthCheckEvents(current *pacemaker.HealthStatus, previous *pacemaker.HealthStatus) { + // Clean up expired events before processing new ones + c.cleanupExpiredEvents() + + // Record events for warnings with appropriate deduplication window + for _, warning := range current.Warnings { + eventKey := fmt.Sprintf(eventKeyPrefixWarning+"%s", warning) + // Use longer deduplication window for fencing events + deduplicationWindow := pacemaker.EventDeduplicationWindowDefault + if strings.Contains(warning, warningPrefixFencingEvent) { + deduplicationWindow = pacemaker.EventDeduplicationWindowFencing + } + if c.shouldRecordEvent(eventKey, deduplicationWindow) { + c.recordWarningEvent(warning) + } + } + + // Record events for errors with default deduplication window + for _, err := range current.Errors { + eventKey := fmt.Sprintf(eventKeyPrefixError+"%s", err) + if c.shouldRecordEvent(eventKey, pacemaker.EventDeduplicationWindowDefault) { + c.recordErrorEvent(err) + } + } + + // Track and detect health transitions + c.recordHealthTransitionEvents(current, previous) +} + +// recordHealthTransitionEvents detects and records health state transitions. +// - PacemakerHealthy: fires on transition to operationally healthy from degraded OR unknown state +// - PacemakerWarningsCleared: fires when warnings are resolved, regardless of overall health +func (c *PacemakerLifecycleManager) recordHealthTransitionEvents(current *pacemaker.HealthStatus, previous *pacemaker.HealthStatus) { + // Determine previous state from the previous pacemaker.HealthStatus we computed. + // This is derived from the previous PacemakerCluster CR we processed. + // Note: previous.OverallStatus is never StatusUnknown (we only update previous for non-Unknown status). + // Instead, detect Unknown state by checking if previous is nil or if CRLastUpdated is too old (same threshold as degraded status). + // If CRLastUpdated is zero, treat as valid (test scenarios or in-memory status). + previousWasUnknown := previous == nil || (!previous.CRLastUpdated.IsZero() && time.Since(previous.CRLastUpdated) > pacemaker.StatusUnknownDegradedThreshold) + previousWasDegraded := previous != nil && previous.OverallStatus == pacemaker.StatusError + previousHadWarnings := previous != nil && len(previous.Warnings) > 0 + + // Determine current state + operationallyHealthy := current.OverallStatus == pacemaker.StatusHealthy || current.OverallStatus == pacemaker.StatusWarning + currentHasNoWarnings := len(current.Warnings) == 0 + + // Record PacemakerHealthy when transitioning to operationally healthy from: + // 1. Error state (previousWasDegraded) - recovering from known problems + // 2. Unknown state (previousWasUnknown) - initial healthy status or recovering from stale/missing CR + // Both Healthy and Warning are "operationally healthy" (cluster is functional). + if operationallyHealthy && (previousWasDegraded || previousWasUnknown) { + c.eventRecorder.Eventf(pacemaker.EventReasonHealthy, msgPacemakerHealthy) + klog.Infof("Pacemaker cluster transitioned to healthy (status: %s, fromDegraded: %v, fromUnknown: %v)", + current.OverallStatus, previousWasDegraded, previousWasUnknown) + } + + // Record PacemakerWarningsCleared when warnings are resolved. + // This fires whenever the previous status had warnings and the current one doesn't, + // regardless of overall cluster health. Warnings are independent of error state. + // Examples: + // - Warning → Healthy: warnings resolved, cluster fully healthy + // - Error+Warning → Error: warnings resolved, but cluster still degraded for other reasons + // - Error+Warning → Healthy: warnings resolved AND cluster recovered (fires both events) + if previousHadWarnings && currentHasNoWarnings { + c.eventRecorder.Eventf(pacemaker.EventReasonWarningsCleared, msgPacemakerWarningsCleared) + klog.Infof("Pacemaker cluster warnings cleared") + } +} + +// recordWarningEvent records appropriate warning events based on warning type. +// Note: Failed action events (warningPrefixFailedAction) are recorded by the status collector +// directly from pacemaker XML, not by the healthcheck controller. The PacemakerCluster CR +// only contains conditions, not raw failed action history. +func (c *PacemakerLifecycleManager) recordWarningEvent(warning string) { + switch { + case strings.Contains(warning, warningPrefixFencingEvent): + c.eventRecorder.Warningf(pacemaker.EventReasonFencingEvent, msgDetectedFencing, warning) + default: + c.eventRecorder.Warningf(pacemaker.EventReasonWarning, msgPacemakerWarning, warning) + } +} + +// recordErrorEvent records appropriate error events with specific reasons based on error content. +// This allows operators to filter and alert on specific types of degrading conditions. +func (c *PacemakerLifecycleManager) recordErrorEvent(errorMsg string) { + eventReason := pacemaker.GetEventReasonForError(errorMsg) + c.eventRecorder.Warningf(eventReason, msgPacemakerError, errorMsg) +} diff --git a/pkg/tnf/operator/lifecycle_manager.go b/pkg/tnf/operator/lifecycle_manager.go new file mode 100644 index 0000000000..b53bc2b44a --- /dev/null +++ b/pkg/tnf/operator/lifecycle_manager.go @@ -0,0 +1,344 @@ +package operator + +import ( + "context" + "fmt" + "sync" + "time" + + operatorv1informers "github.com/openshift/client-go/operator/informers/externalversions/operator/v1" + "github.com/openshift/library-go/pkg/controller/controllercmd" + "github.com/openshift/library-go/pkg/controller/factory" + "github.com/openshift/library-go/pkg/operator/events" + "github.com/openshift/library-go/pkg/operator/v1helpers" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" + + pacmkrv1 "github.com/openshift/api/etcd/v1" + "github.com/openshift/cluster-etcd-operator/pkg/operator/ceohelpers" + "github.com/openshift/cluster-etcd-operator/pkg/tnf/pkg/pacemaker" + "github.com/openshift/cluster-etcd-operator/pkg/tnf/pkg/tools" +) + +// Local constants for lifecycle controller +const ( + // Controller name + controllerNamePacemakerLifecycle = "PacemakerLifecycleManager" +) + +// PacemakerLifecycleManager monitors pacemaker status and manages node membership reconciliation +// in ExternalEtcd topology clusters. It handles health monitoring, drift detection, and +// triggers update-setup jobs when pacemaker membership diverges from Kubernetes. +type PacemakerLifecycleManager struct { + operatorClient v1helpers.StaticPodOperatorClient + kubeClient kubernetes.Interface + eventRecorder events.Recorder + pacemakerInformer cache.SharedIndexInformer + + // For node lifecycle management + nodeInformer cache.SharedIndexInformer + controllerContext *controllercmd.ControllerContext + kubeInformersForNamespaces v1helpers.KubeInformersForNamespaces + etcdInformer operatorv1informers.EtcdInformer + + // Event deduplication: tracks recently recorded events to avoid duplicates + recordedEventsMu sync.Mutex + recordedEvents map[string]time.Time + + // Previous health status for transition detection and grace period calculation. + // Only updated when status is non-Unknown, so CRLastUpdated reflects the last valid CR timestamp. + previousMu sync.Mutex + previous *pacemaker.HealthStatus + + // Job controller startup protection: prevents concurrent StartJobControllers calls + startJobControllersMu sync.Mutex + // Track if job controllers have been started (set once, never reset) + jobControllersStarted bool + + // Lifecycle context for goroutines spawned by event handlers + lifecycleCtx context.Context + lifecycleCtxCancel context.CancelFunc +} + +// NewPacemakerLifecycleManager creates a new PacemakerLifecycleManager for monitoring pacemaker status +// and managing node membership reconciliation in clusters that use ExternalEtcd. +// Returns the controller, the PacemakerLifecycleManager instance, and the PacemakerCluster informer +// (which must be started separately - see runPacemakerControllers in pkg/tnf/operator/starter.go). +func NewPacemakerLifecycleManager( + operatorClient v1helpers.StaticPodOperatorClient, + kubeClient kubernetes.Interface, + eventRecorder events.Recorder, + restConfig *rest.Config, + nodeInformer cache.SharedIndexInformer, + controllerContext *controllercmd.ControllerContext, + kubeInformersForNamespaces v1helpers.KubeInformersForNamespaces, + etcdInformer operatorv1informers.EtcdInformer, +) (factory.Controller, *PacemakerLifecycleManager, cache.SharedIndexInformer, error) { + // Create REST client for PacemakerStatus CRs + restClient, err := pacemaker.CreatePacemakerRESTClient(restConfig) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to create REST client: %w", err) + } + + // Create scheme for the parameter codec + scheme := runtime.NewScheme() + if err := pacmkrv1.AddToScheme(scheme); err != nil { + return nil, nil, nil, fmt.Errorf("failed to add scheme for informer: %w", err) + } + + // Create informer for PacemakerCluster + klog.Infof("Creating PacemakerCluster informer for group %s, resource %s", pacmkrv1.SchemeGroupVersion.String(), pacemaker.PacemakerResourceName) + informer := cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + klog.V(4).Infof("PacemakerCluster informer ListFunc called for resource %s", pacemaker.PacemakerResourceName) + // Sanitize ListOptions - remove fields not supported by all Kubernetes versions + sanitizedOptions := metav1.ListOptions{ + LabelSelector: options.LabelSelector, + FieldSelector: options.FieldSelector, + Watch: options.Watch, + AllowWatchBookmarks: options.AllowWatchBookmarks, + ResourceVersion: options.ResourceVersion, + TimeoutSeconds: options.TimeoutSeconds, + Limit: options.Limit, + Continue: options.Continue, + } + result := &pacmkrv1.PacemakerClusterList{} + err := restClient.Get(). + Resource(pacemaker.PacemakerResourceName). + VersionedParams(&sanitizedOptions, runtime.NewParameterCodec(scheme)). + Do(context.Background()). + Into(result) + if err != nil { + klog.Errorf("Failed to list PacemakerCluster resources (%s): %v", pacemaker.PacemakerResourceName, err) + } else { + klog.V(4).Infof("Successfully listed PacemakerCluster resources, found %d items", len(result.Items)) + } + return result, err + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + klog.V(4).Infof("PacemakerCluster informer WatchFunc called for resource %s", pacemaker.PacemakerResourceName) + // Sanitize ListOptions - remove fields not supported by all Kubernetes versions + // sendInitialEvents and resourceVersionMatch were added in 1.27+ and cause errors on older clusters + sanitizedOptions := metav1.ListOptions{ + LabelSelector: options.LabelSelector, + FieldSelector: options.FieldSelector, + Watch: options.Watch, + AllowWatchBookmarks: options.AllowWatchBookmarks, + ResourceVersion: options.ResourceVersion, + TimeoutSeconds: options.TimeoutSeconds, + Limit: options.Limit, + Continue: options.Continue, + } + watcher, err := restClient.Get(). + Resource(pacemaker.PacemakerResourceName). + VersionedParams(&sanitizedOptions, runtime.NewParameterCodec(scheme)). + Watch(context.Background()) + if err != nil { + klog.Errorf("Failed to watch PacemakerCluster resources (%s): %v", pacemaker.PacemakerResourceName, err) + } + return watcher, err + }, + }, + &pacmkrv1.PacemakerCluster{}, + pacemaker.HealthCheckResyncInterval, + cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, + ) + + // Create lifecycle context for goroutines spawned by event handlers + lifecycleCtx, lifecycleCtxCancel := context.WithCancel(context.Background()) + + c := &PacemakerLifecycleManager{ + operatorClient: operatorClient, + kubeClient: kubeClient, + eventRecorder: eventRecorder, + pacemakerInformer: informer, + nodeInformer: nodeInformer, + controllerContext: controllerContext, + kubeInformersForNamespaces: kubeInformersForNamespaces, + etcdInformer: etcdInformer, + recordedEvents: make(map[string]time.Time), + lifecycleCtx: lifecycleCtx, + lifecycleCtxCancel: lifecycleCtxCancel, + } + + // Initialize update-setup generation counter from existing ConfigMaps + // This prevents reusing stale ConfigMaps after operator restart + if err := c.initUpdateSetupGeneration(context.Background()); err != nil { + return nil, nil, nil, fmt.Errorf("failed to initialize update-setup generation counter: %w", err) + } + + syncCtx := factory.NewSyncContext(controllerNamePacemakerLifecycle, eventRecorder.WithComponentSuffix("pacemaker-lifecycle-manager")) + + klog.Infof("%s controller created, waiting for informers to sync before starting", controllerNamePacemakerLifecycle) + klog.Infof("PacemakerLifecycleManager will watch: operatorClient and %s/%s resource", pacmkrv1.SchemeGroupVersion.String(), pacemaker.PacemakerResourceName) + + // ResyncEvery ensures the sync function is called at regular intervals (30s) + // even if no informer events are detected. + controller := factory.New(). + WithSyncContext(syncCtx). + ResyncEvery(pacemaker.HealthCheckResyncInterval). + WithSync(c.sync). + WithInformers( + operatorClient.Informer(), + informer, + nodeInformer, + ).ToController(controllerNamePacemakerLifecycle, syncCtx.Recorder()) + + // PacemakerCluster informer is started in runPacemakerControllers (pkg/tnf/operator/starter.go) + // Node informer is started in RunOperator (pkg/operator/starter.go) + klog.Infof("PacemakerLifecycleManager controller created, will wait up to 10 minutes for informers to sync") + + // Register node event handlers for drift-triggered reconciliation + if err := c.RegisterNodeEventHandlers(); err != nil { + return nil, nil, nil, fmt.Errorf("failed to register node event handlers: %w", err) + } + + return controller, c, informer, nil +} + +// RegisterNodeEventHandlers registers Add/Update/Delete event handlers on the node informer. +// Returns an error if registration fails, as the lifecycle manager cannot function without +// receiving node events for drift detection and reconciliation. +// - UpdateFunc: handles Ready transitions for initial bootstrap and IP changes while Ready +// - AddFunc/DeleteFunc: trigger drift detection and reconciliation +func (c *PacemakerLifecycleManager) RegisterNodeEventHandlers() error { + if c.nodeInformer == nil { + return fmt.Errorf("nodeInformer is nil") + } + + _, err := c.nodeInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + UpdateFunc: func(oldObj, newObj any) { + oldNode, oldOk := oldObj.(*corev1.Node) + newNode, newOk := newObj.(*corev1.Node) + if !oldOk || !newOk { + klog.Warningf("failed to convert updated object to Node, old=%+v, new=%+v", oldObj, newObj) + return + } + + // Check for Ready transition (bootstrap path) + oldReady := tools.IsNodeReady(oldNode) + newReady := tools.IsNodeReady(newNode) + if !oldReady && newReady { + klog.Infof("node %s transitioned to ready state - triggering job controller startup", newNode.GetName()) + go func() { + if err := c.StartJobControllers(c.lifecycleCtx); err != nil { + klog.Errorf("Failed to start job controllers on node ready: %v", err) + } + }() + } + + // Check if node IP changed while Ready - trigger reconciliation + if newReady { + oldIP, oldErr := tools.GetNodeIPForPacemaker(*oldNode) + newIP, newErr := tools.GetNodeIPForPacemaker(*newNode) + if oldErr == nil && newErr == nil && oldIP != newIP { + klog.Infof("node %s IP changed from %s to %s - triggering reconciliation check", newNode.GetName(), oldIP, newIP) + go func() { + if err := c.ReconcilePacemakerConfig(c.lifecycleCtx); err != nil { + klog.Errorf("Failed to ensure reconciliation on IP change: %v", err) + } + }() + } + } + }, + AddFunc: func(obj any) { + node, ok := obj.(*corev1.Node) + if !ok { + klog.Warningf("failed to convert added object to Node %+v", obj) + return + } + klog.Infof("node added: %s - triggering reconciliation check", node.GetName()) + // Trigger reconciliation check (will no-op if transition not complete, no drift, or already running) + go func() { + if err := c.ReconcilePacemakerConfig(c.lifecycleCtx); err != nil { + klog.Errorf("Failed to ensure reconciliation on node add: %v", err) + } + }() + }, + DeleteFunc: func(obj any) { + node, ok := obj.(*corev1.Node) + if !ok { + // Handle tombstone case + tombstone, ok := obj.(cache.DeletedFinalStateUnknown) + if !ok { + klog.Warningf("failed to convert deleted object to Node or tombstone %+v", obj) + return + } + node, ok = tombstone.Obj.(*corev1.Node) + if !ok { + klog.Warningf("tombstone contained object that is not a Node %+v", obj) + return + } + } + klog.Infof("node deleted: %s - triggering reconciliation check", node.GetName()) + // Trigger reconciliation check (will no-op if transition not complete, no drift, or already running) + go func() { + if err := c.ReconcilePacemakerConfig(c.lifecycleCtx); err != nil { + klog.Errorf("Failed to ensure reconciliation on node delete: %v", err) + } + }() + }, + }) + + if err != nil { + return fmt.Errorf("failed to add event handler to node informer: %w", err) + } + + klog.Infof("Registered Add/Update/Delete event handlers for node lifecycle management") + return nil +} + +// sync is the main sync function that gets called periodically to check pacemaker status +func (c *PacemakerLifecycleManager) sync(ctx context.Context, syncCtx factory.SyncContext) error { + klog.V(4).Infof("PacemakerLifecycleManager sync started") + defer klog.V(4).Infof("PacemakerLifecycleManager sync completed") + + // Check if external etcd transition is complete + transitionComplete, err := ceohelpers.HasExternalEtcdCompletedTransition(ctx, c.operatorClient) + if err != nil { + return fmt.Errorf("failed to check external etcd transition status: %w", err) + } + + // Aggregate errors across independent operations + // Each operation continues even if others fail, ensuring partial progress + var errs []error + + // 1. Start job controllers (runs in both modes with different behavior) + if err := c.StartJobControllers(ctx); err != nil { + klog.Errorf("Failed to start job controllers: %v", err) + errs = append(errs, fmt.Errorf("failed to start job controllers: %w", err)) + } + + // 2. Cleanup orphaned and old jobs (always runs, even before transition) + // This prevents job/pod accumulation during bootstrap or unhealthy states + if err := c.CleanupOrphanedJobs(ctx); err != nil { + klog.Errorf("Failed to cleanup orphaned jobs: %v", err) + errs = append(errs, fmt.Errorf("failed to cleanup orphaned jobs: %w", err)) + } + + // Operations 3-4 only run after external etcd transition completes + if transitionComplete { + // 3. Monitor pacemaker health + if err := c.MonitorHealth(ctx); err != nil { + klog.Errorf("Failed to monitor health: %v", err) + errs = append(errs, fmt.Errorf("failed to monitor health: %w", err)) + } + + // 4. Reconcile pacemaker configuration (drift detection) + if err := c.ReconcilePacemakerConfig(ctx); err != nil { + klog.Errorf("Failed to reconcile pacemaker config: %v", err) + errs = append(errs, fmt.Errorf("failed to reconcile pacemaker config: %w", err)) + } + } + + // Return aggregated errors - controller framework will mark degraded and retry + return errors.NewAggregate(errs) +} diff --git a/pkg/tnf/operator/lifecycle_manager_test.go b/pkg/tnf/operator/lifecycle_manager_test.go new file mode 100644 index 0000000000..06482ff1d0 --- /dev/null +++ b/pkg/tnf/operator/lifecycle_manager_test.go @@ -0,0 +1,1564 @@ +package operator + +/* +TEST COVERAGE SUMMARY - lifecycle_manager_test.go +================================================== + +This file tests PacemakerLifecycleManager - health monitoring and node reconciliation +for Two-Node Fencing (TNF) clusters. + +WHAT'S TESTED +------------- + +Health Monitoring Path: +├── getPacemakerStatus() - CR status retrieval and staleness detection +│ ├── ✅ Healthy cluster +│ ├── ✅ Unhealthy kubelet +│ ├── ✅ Unhealthy etcd +│ ├── ✅ Insufficient nodes +│ ├── ✅ No nodes +│ ├── ✅ Unpopulated status +│ ├── ✅ Stale status detection +│ ├── ✅ CR not found +│ └── ✅ Unchanged timestamp (skip processing) +├── updateOperatorStatus() - Operator condition management +│ ├── ✅ Degraded condition set (pacemaker error) +│ ├── ✅ Available condition set (pacemaker healthy/warning) +│ ├── ✅ Unknown status with grace period +│ └── ✅ Unknown status grace period expired +├── recordHealthCheckEvents() - Event recording with deduplication +│ ├── ✅ Fencing event +│ ├── ✅ Resource/node/cluster unhealthy events +│ ├── ✅ Status transitions (degraded → healthy) +│ ├── ✅ Event deduplication (same event not re-recorded) +│ └── ✅ Old event cleanup (>1 hour) +└── detectDrift() - K8s vs pacemaker state comparison + ├── ✅ No drift + ├── ✅ Node count mismatch (add) + ├── ✅ Node count mismatch (delete) + ├── ✅ IP address mismatch + ├── ✅ Name mismatch (node replacement) + └── ✅ Empty states + +Reconciliation Path: +├── ReconcilePacemakerConfig() - Drift detection and reconciliation trigger +│ ├── ✅ No pacemaker CR - skip +│ ├── ✅ Node informer not synced - skip +│ ├── ✅ More than 2 nodes - skip +│ ├── ✅ Node not ready - skip +│ ├── ✅ No drift - no action +│ ├── ✅ Drift detected (count/IP mismatch) - trigger update-setup +│ ├── ✅ Update-setup already running - skip +│ ├── ✅ No intersection - error +│ └── ✅ Update-setup fails - return error +└── cleanupOrphanedJobs() - Delete jobs for deleted nodes + ├── ✅ No jobs to clean + ├── ✅ Orphaned job deleted + ├── ✅ Multiple orphaned jobs deleted + ├── ✅ Active job preserved + └── ✅ Non-TNF jobs ignored +*/ + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/tools/cache" + "k8s.io/utils/clock" + + pacmkrv1 "github.com/openshift/api/etcd/v1" + operatorv1 "github.com/openshift/api/operator/v1" + operatorv1informers "github.com/openshift/client-go/operator/informers/externalversions/operator/v1" + "github.com/openshift/cluster-etcd-operator/pkg/tnf/pkg/jobs" + "github.com/openshift/cluster-etcd-operator/pkg/tnf/pkg/pacemaker" + "github.com/openshift/library-go/pkg/controller/controllercmd" + "github.com/openshift/library-go/pkg/operator/events" + "github.com/openshift/library-go/pkg/operator/v1helpers" +) + +// Test helper functions + +// createFakeOperatorClient creates a fake static pod operator client for testing +func createFakeOperatorClient() v1helpers.StaticPodOperatorClient { + return v1helpers.NewFakeStaticPodOperatorClient( + &operatorv1.StaticPodOperatorSpec{ + OperatorSpec: operatorv1.OperatorSpec{ + ManagementState: operatorv1.Managed, + }, + }, + &operatorv1.StaticPodOperatorStatus{ + OperatorStatus: operatorv1.OperatorStatus{ + Conditions: []operatorv1.OperatorCondition{}, + }, + }, + nil, + nil, + ) +} + +func createTestHealthCheck() *PacemakerLifecycleManager { + return &PacemakerLifecycleManager{ + operatorClient: createFakeOperatorClient(), + kubeClient: fake.NewSimpleClientset(), + eventRecorder: events.NewInMemoryRecorder("test", clock.RealClock{}), + pacemakerInformer: createFakeInformer(nil), + recordedEvents: make(map[string]time.Time), + } +} + +// createFakeInformer creates a fake SharedIndexInformer with a store containing the given object. +// If obj is nil, the store will be empty. +func createFakeInformer(obj *pacmkrv1.PacemakerCluster) cache.SharedIndexInformer { + store := cache.NewStore(func(obj any) (string, error) { + if pc, ok := obj.(*pacmkrv1.PacemakerCluster); ok { + return pc.Name, nil + } + return "", fmt.Errorf("object is not a PacemakerCluster") + }) + if obj != nil { + if err := store.Add(obj); err != nil { + panic(fmt.Sprintf("failed to add object to fake informer store: %v", err)) + } + } + return &fakeInformer{store: store} +} + +// fakeInformer implements cache.SharedIndexInformer for testing +type fakeInformer struct { + store cache.Store +} + +func (f *fakeInformer) AddEventHandler(handler cache.ResourceEventHandler) (cache.ResourceEventHandlerRegistration, error) { + return nil, nil +} +func (f *fakeInformer) AddEventHandlerWithResyncPeriod(handler cache.ResourceEventHandler, resyncPeriod time.Duration) (cache.ResourceEventHandlerRegistration, error) { + return nil, nil +} +func (f *fakeInformer) AddEventHandlerWithOptions(handler cache.ResourceEventHandler, options cache.HandlerOptions) (cache.ResourceEventHandlerRegistration, error) { + return nil, nil +} +func (f *fakeInformer) RemoveEventHandler(handle cache.ResourceEventHandlerRegistration) error { + return nil +} +func (f *fakeInformer) GetStore() cache.Store { return f.store } +func (f *fakeInformer) GetController() cache.Controller { return nil } +func (f *fakeInformer) Run(stopCh <-chan struct{}) {} +func (f *fakeInformer) RunWithContext(ctx context.Context) { +} +func (f *fakeInformer) HasSynced() bool { return true } +func (f *fakeInformer) LastSyncResourceVersion() string { return "" } +func (f *fakeInformer) SetWatchErrorHandler(handler cache.WatchErrorHandler) error { return nil } +func (f *fakeInformer) SetWatchErrorHandlerWithContext(handler cache.WatchErrorHandlerWithContext) error { + return nil +} +func (f *fakeInformer) SetTransform(handler cache.TransformFunc) error { return nil } +func (f *fakeInformer) IsStopped() bool { return false } +func (f *fakeInformer) AddIndexers(indexers cache.Indexers) error { return nil } +func (f *fakeInformer) GetIndexer() cache.Indexer { return nil } + +// createTestHealthCheckWithMockStatus creates a HealthCheck with a mocked Pacemaker CR in the informer cache +func createTestHealthCheckWithMockStatus(t *testing.T, mockStatus *pacmkrv1.PacemakerCluster) *PacemakerLifecycleManager { + return &PacemakerLifecycleManager{ + operatorClient: createFakeOperatorClient(), + kubeClient: fake.NewSimpleClientset(), + eventRecorder: events.NewInMemoryRecorder("test", clock.RealClock{}), + pacemakerInformer: createFakeInformer(mockStatus), + recordedEvents: make(map[string]time.Time), + } +} + +// ============================================================================= +// Unit Tests +// ============================================================================= +// Note: Test fixtures are defined in test_fixtures_test.go + +func TestHealthCheck_getPacemakerStatus(t *testing.T) { + tests := []struct { + name string + mockStatus *pacmkrv1.PacemakerCluster + expectedStatus pacemaker.HealthStatusValue + expectErrors bool + expectWarnings bool + }{ + { + name: "healthy_status", + mockStatus: &pacmkrv1.PacemakerCluster{ + TypeMeta: metav1.TypeMeta{ + APIVersion: pacmkrv1.SchemeGroupVersion.String(), + Kind: "PacemakerCluster", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster", + }, + Status: pacmkrv1.PacemakerClusterStatus{ + Conditions: pacemaker.CreateHealthyClusterConditions(), + Nodes: &[]pacmkrv1.PacemakerClusterNodeStatus{ + pacemaker.CreateHealthyNodeStatus("master-0", []string{"192.168.1.10"}), + pacemaker.CreateHealthyNodeStatus("master-1", []string{"192.168.1.11"}), + }, + LastUpdated: metav1.Now(), + }, + }, + expectedStatus: pacemaker.StatusHealthy, + expectErrors: false, + expectWarnings: false, + }, + { + name: "stale_status", + mockStatus: &pacmkrv1.PacemakerCluster{ + TypeMeta: metav1.TypeMeta{ + APIVersion: pacmkrv1.SchemeGroupVersion.String(), + Kind: "PacemakerCluster", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster", + }, + Status: pacmkrv1.PacemakerClusterStatus{ + Conditions: pacemaker.CreateHealthyClusterConditions(), + Nodes: &[]pacmkrv1.PacemakerClusterNodeStatus{ + pacemaker.CreateHealthyNodeStatus("master-0", []string{"192.168.1.10"}), + pacemaker.CreateHealthyNodeStatus("master-1", []string{"192.168.1.11"}), + }, + LastUpdated: metav1.Time{Time: time.Now().Add(-10 * time.Minute)}, // > 5 min threshold + }, + }, + expectedStatus: pacemaker.StatusUnknown, + expectErrors: true, + expectWarnings: false, + }, + { + name: "nil_status", + mockStatus: &pacmkrv1.PacemakerCluster{ + TypeMeta: metav1.TypeMeta{ + APIVersion: pacmkrv1.SchemeGroupVersion.String(), + Kind: "PacemakerCluster", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster", + }, + Status: pacmkrv1.PacemakerClusterStatus{}, // Status not populated yet (zero LastUpdated) + }, + expectedStatus: pacemaker.StatusUnknown, + expectErrors: true, + expectWarnings: false, + }, + { + name: "insufficient_nodes", + mockStatus: &pacmkrv1.PacemakerCluster{ + TypeMeta: metav1.TypeMeta{ + APIVersion: pacmkrv1.SchemeGroupVersion.String(), + Kind: "PacemakerCluster", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster", + }, + Status: pacmkrv1.PacemakerClusterStatus{ + Conditions: pacemaker.CreateInsufficientNodesClusterConditions(), + Nodes: &[]pacmkrv1.PacemakerClusterNodeStatus{ + pacemaker.CreateHealthyNodeStatus("master-0", []string{"192.168.1.10"}), + }, + LastUpdated: metav1.Now(), + }, + }, + expectedStatus: pacemaker.StatusError, + expectErrors: true, + expectWarnings: false, + }, + { + name: "unhealthy_kubelet", + mockStatus: &pacmkrv1.PacemakerCluster{ + TypeMeta: metav1.TypeMeta{ + APIVersion: pacmkrv1.SchemeGroupVersion.String(), + Kind: "PacemakerCluster", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster", + }, + Status: pacmkrv1.PacemakerClusterStatus{ + Conditions: pacemaker.CreateHealthyClusterConditions(), + Nodes: &[]pacmkrv1.PacemakerClusterNodeStatus{ + pacemaker.CreateUnhealthyNodeStatus("master-0", []string{"192.168.1.10"}, pacmkrv1.PacemakerClusterResourceNameKubelet), + pacemaker.CreateHealthyNodeStatus("master-1", []string{"192.168.1.11"}), + }, + LastUpdated: metav1.Now(), + }, + }, + expectedStatus: pacemaker.StatusError, + expectErrors: true, + expectWarnings: false, + }, + { + name: "unhealthy_etcd", + mockStatus: &pacmkrv1.PacemakerCluster{ + TypeMeta: metav1.TypeMeta{ + APIVersion: pacmkrv1.SchemeGroupVersion.String(), + Kind: "PacemakerCluster", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster", + }, + Status: pacmkrv1.PacemakerClusterStatus{ + Conditions: pacemaker.CreateHealthyClusterConditions(), + Nodes: &[]pacmkrv1.PacemakerClusterNodeStatus{ + pacemaker.CreateHealthyNodeStatus("master-0", []string{"192.168.1.10"}), + pacemaker.CreateUnhealthyNodeStatus("master-1", []string{"192.168.1.11"}, pacmkrv1.PacemakerClusterResourceNameEtcd), + }, + LastUpdated: metav1.Now(), + }, + }, + expectedStatus: pacemaker.StatusError, + expectErrors: true, + expectWarnings: false, + }, + { + name: "unhealthy_etcd_first_node", + mockStatus: &pacmkrv1.PacemakerCluster{ + TypeMeta: metav1.TypeMeta{ + APIVersion: pacmkrv1.SchemeGroupVersion.String(), + Kind: "PacemakerCluster", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster", + }, + Status: pacmkrv1.PacemakerClusterStatus{ + Conditions: pacemaker.CreateHealthyClusterConditions(), + Nodes: &[]pacmkrv1.PacemakerClusterNodeStatus{ + pacemaker.CreateUnhealthyNodeStatus("master-0", []string{"192.168.1.10"}, pacmkrv1.PacemakerClusterResourceNameEtcd), + pacemaker.CreateHealthyNodeStatus("master-1", []string{"192.168.1.11"}), + }, + LastUpdated: metav1.Now(), + }, + }, + expectedStatus: pacemaker.StatusError, + expectErrors: true, + expectWarnings: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + controller := createTestHealthCheckWithMockStatus(t, tt.mockStatus) + ctx := context.TODO() + + current, _, err := controller.getPacemakerStatus(ctx) + + require.NoError(t, err, "getPacemakerStatus should not return an error") + require.NotNil(t, current, "pacemaker.HealthStatus should not be nil") + require.Equal(t, tt.expectedStatus, current.OverallStatus, "Status should match expected") + + if tt.expectErrors { + require.NotEmpty(t, current.Errors, "Should have errors") + } else { + require.Empty(t, current.Errors, "Should not have errors") + } + + if tt.expectWarnings { + require.NotEmpty(t, current.Warnings, "Should have warnings") + } else { + require.Empty(t, current.Warnings, "Should not have warnings") + } + }) + } +} + +func TestHealthCheck_getPacemakerStatus_UnchangedTimestamp(t *testing.T) { + // Create a fixed timestamp for the test + fixedTime := metav1.Now() + + mockStatus := &pacmkrv1.PacemakerCluster{ + TypeMeta: metav1.TypeMeta{ + APIVersion: pacmkrv1.SchemeGroupVersion.String(), + Kind: "PacemakerCluster", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster", + }, + Status: pacmkrv1.PacemakerClusterStatus{ + Conditions: pacemaker.CreateHealthyClusterConditions(), + Nodes: &[]pacmkrv1.PacemakerClusterNodeStatus{ + pacemaker.CreateHealthyNodeStatus("master-0", []string{"192.168.1.10"}), + pacemaker.CreateHealthyNodeStatus("master-1", []string{"192.168.1.11"}), + }, + LastUpdated: fixedTime, + }, + } + + controller := createTestHealthCheckWithMockStatus(t, mockStatus) + ctx := context.TODO() + + // First call should return a valid status + current1, _, err1 := controller.getPacemakerStatus(ctx) + require.NoError(t, err1, "First getPacemakerStatus should not return an error") + require.NotNil(t, current1, "First call should return a status") + require.Equal(t, pacemaker.StatusHealthy, current1.OverallStatus, "First call should return healthy status") + + // Second call with same timestamp should return nil (no change) + current2, _, err2 := controller.getPacemakerStatus(ctx) + require.NoError(t, err2, "Second getPacemakerStatus should not return an error") + require.Nil(t, current2, "Second call with unchanged timestamp should return nil") +} + +func TestHealthCheck_updateOperatorStatus(t *testing.T) { + tests := []struct { + name string + status *pacemaker.HealthStatus + previous *pacemaker.HealthStatus + }{ + { + name: "error_status_sets_degraded", + status: &pacemaker.HealthStatus{ + OverallStatus: pacemaker.StatusError, + Warnings: []string{"Test warning"}, + Errors: []string{"Test error"}, + }, + previous: &pacemaker.HealthStatus{ + OverallStatus: pacemaker.StatusHealthy, + CRLastUpdated: time.Now().Add(-1 * time.Minute), + }, + }, + { + name: "healthy_status_clears_degraded", + status: &pacemaker.HealthStatus{ + OverallStatus: pacemaker.StatusHealthy, + Warnings: []string{}, + Errors: []string{}, + }, + previous: &pacemaker.HealthStatus{ + OverallStatus: pacemaker.StatusHealthy, + CRLastUpdated: time.Now().Add(-1 * time.Minute), + }, + }, + { + name: "warning_status_clears_degraded", + status: &pacemaker.HealthStatus{ + OverallStatus: pacemaker.StatusWarning, + Warnings: []string{"Test warning"}, + Errors: []string{}, + }, + previous: &pacemaker.HealthStatus{ + OverallStatus: pacemaker.StatusHealthy, + CRLastUpdated: time.Now().Add(-1 * time.Minute), + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + controller := createTestHealthCheck() + ctx := context.TODO() + + err := controller.updateOperatorStatus(ctx, tt.status, tt.previous) + require.NoError(t, err, "updateOperatorStatus should not return an error") + }) + } +} + +func TestHealthCheck_recordHealthCheckEvents(t *testing.T) { + controller := createTestHealthCheck() + + // Test with warnings and errors - previous was healthy (not degraded, no warnings, known status) + previousHealthy := &pacemaker.HealthStatus{OverallStatus: pacemaker.StatusHealthy, Warnings: []string{}, Errors: []string{}} + currentWithWarnings := &pacemaker.HealthStatus{ + OverallStatus: pacemaker.StatusWarning, + Warnings: []string{ + "Recent failed resource action: kubelet monitor on master-0 failed", + "Recent fencing event: reboot of master-1 success", + "Some other warning", + }, + Errors: []string{"Test error"}, + } + + controller.recordHealthCheckEvents(currentWithWarnings, previousHealthy) + + // Test with healthy status - previous had warnings + currentHealthy := &pacemaker.HealthStatus{ + OverallStatus: pacemaker.StatusHealthy, + Warnings: []string{}, + Errors: []string{}, + } + + controller.recordHealthCheckEvents(currentHealthy, currentWithWarnings) +} + +func TestHealthCheck_eventDeduplication(t *testing.T) { + controller := createTestHealthCheck() + recorder := controller.eventRecorder.(events.InMemoryRecorder) + + // Track previous status through the test + var previousStatus *pacemaker.HealthStatus + + // First recording - Error state (start in Error to test transitions properly) + // previousStatus=nil because this is first sync + errorStatus := &pacemaker.HealthStatus{ + OverallStatus: pacemaker.StatusError, + Warnings: []string{}, + Errors: []string{"Test error"}, + } + controller.recordHealthCheckEvents(errorStatus, previousStatus) + previousStatus = errorStatus + + // Count events after first recording + initialEvents := len(recorder.Events()) + require.Equal(t, 1, initialEvents, "Should have recorded 1 error event") + + // Immediately record the same error again - should be deduplicated (5 min window) + // previousStatus is now errorStatus (degraded) + controller.recordHealthCheckEvents(errorStatus, previousStatus) + afterDuplicateEvents := len(recorder.Events()) + require.Equal(t, initialEvents, afterDuplicateEvents, "Duplicate error should not be recorded") + + // Transition to Warning (operationally healthy) - should record PacemakerHealthy event + // previousStatus is errorStatus (degraded), so healthy event should fire + warningStatus := &pacemaker.HealthStatus{ + OverallStatus: pacemaker.StatusWarning, + Warnings: []string{ + "Recent failed resource action: kubelet monitor on master-0 failed", + }, + Errors: []string{}, + } + controller.recordHealthCheckEvents(warningStatus, previousStatus) + previousStatus = warningStatus + afterWarningTransition := len(recorder.Events()) + require.Equal(t, 3, afterWarningTransition, "Should have recorded warning event + PacemakerHealthy transition event") + + // Stay in Warning - no transition event + controller.recordHealthCheckEvents(warningStatus, previousStatus) + afterWarningDuplicate := len(recorder.Events()) + require.Equal(t, afterWarningTransition, afterWarningDuplicate, "No transition event when staying in Warning") + + // Test fencing event deduplication (longer window) + fencingStatus := &pacemaker.HealthStatus{ + OverallStatus: pacemaker.StatusWarning, + Warnings: []string{ + "Recent fencing event: reboot of master-1 success", + }, + Errors: []string{}, + } + controller.recordHealthCheckEvents(fencingStatus, previousStatus) + previousStatus = fencingStatus + afterFencingEvents := len(recorder.Events()) + require.Equal(t, 4, afterFencingEvents, "Fencing event should be recorded") + + // Immediately record the same fencing event - should be deduplicated (24 hour window) + controller.recordHealthCheckEvents(fencingStatus, previousStatus) + afterFencingDuplicateEvents := len(recorder.Events()) + require.Equal(t, afterFencingEvents, afterFencingDuplicateEvents, "Duplicate fencing event should not be recorded") + + // Transition from Warning to Healthy - fires PacemakerWarningsCleared event + // previousStatus has warnings, so WarningsCleared should fire + healthyStatus := &pacemaker.HealthStatus{ + OverallStatus: pacemaker.StatusHealthy, + Warnings: []string{}, + Errors: []string{}, + } + controller.recordHealthCheckEvents(healthyStatus, previousStatus) + previousStatus = healthyStatus + afterHealthyTransition := len(recorder.Events()) + require.Equal(t, afterFencingDuplicateEvents+1, afterHealthyTransition, "PacemakerWarningsCleared event should fire on Warning to Healthy transition") + + // Stay healthy - no event + controller.recordHealthCheckEvents(healthyStatus, previousStatus) + afterHealthyDuplicate := len(recorder.Events()) + require.Equal(t, afterHealthyTransition, afterHealthyDuplicate, "No event when staying healthy") + + // Transition to Error + controller.recordHealthCheckEvents(errorStatus, previousStatus) + previousStatus = errorStatus + beforeRecovery := len(recorder.Events()) + + // Transition back to Healthy - should record PacemakerHealthy event + // previousStatus is errorStatus (degraded), so healthy event should fire + controller.recordHealthCheckEvents(healthyStatus, previousStatus) + afterRecovery := len(recorder.Events()) + require.Greater(t, afterRecovery, beforeRecovery, "PacemakerHealthy event should be recorded on transition from Error to Healthy") +} + +func TestHealthCheck_BuildHealthStatusFromCR(t *testing.T) { + tests := []struct { + name string + cr *pacmkrv1.PacemakerCluster + expectedStatus pacemaker.HealthStatusValue + expectErrors bool + errorContains string // Optional: check error contains this substring + }{ + { + name: "healthy_cluster", + cr: &pacmkrv1.PacemakerCluster{ + Status: pacmkrv1.PacemakerClusterStatus{ + LastUpdated: metav1.Now(), + Conditions: pacemaker.CreateHealthyClusterConditions(), + Nodes: &[]pacmkrv1.PacemakerClusterNodeStatus{ + pacemaker.CreateHealthyNodeStatus("master-0", []string{"192.168.1.10"}), + pacemaker.CreateHealthyNodeStatus("master-1", []string{"192.168.1.11"}), + }, + }, + }, + expectedStatus: pacemaker.StatusHealthy, + expectErrors: false, + }, + { + name: "unhealthy_kubelet", + cr: &pacmkrv1.PacemakerCluster{ + Status: pacmkrv1.PacemakerClusterStatus{ + LastUpdated: metav1.Now(), + Conditions: pacemaker.CreateHealthyClusterConditions(), + Nodes: &[]pacmkrv1.PacemakerClusterNodeStatus{ + pacemaker.CreateUnhealthyNodeStatus("master-0", []string{"192.168.1.10"}, pacmkrv1.PacemakerClusterResourceNameKubelet), + pacemaker.CreateHealthyNodeStatus("master-1", []string{"192.168.1.11"}), + }, + }, + }, + expectedStatus: pacemaker.StatusError, + expectErrors: true, + errorContains: string(pacmkrv1.PacemakerClusterResourceNameKubelet), + }, + { + name: "unhealthy_etcd", + cr: &pacmkrv1.PacemakerCluster{ + Status: pacmkrv1.PacemakerClusterStatus{ + LastUpdated: metav1.Now(), + Conditions: pacemaker.CreateHealthyClusterConditions(), + Nodes: &[]pacmkrv1.PacemakerClusterNodeStatus{ + pacemaker.CreateHealthyNodeStatus("master-0", []string{"192.168.1.10"}), + pacemaker.CreateUnhealthyNodeStatus("master-1", []string{"192.168.1.11"}, pacmkrv1.PacemakerClusterResourceNameEtcd), + }, + }, + }, + expectedStatus: pacemaker.StatusError, + expectErrors: true, + errorContains: string(pacmkrv1.PacemakerClusterResourceNameEtcd), + }, + { + name: "insufficient_nodes", + cr: &pacmkrv1.PacemakerCluster{ + Status: pacmkrv1.PacemakerClusterStatus{ + LastUpdated: metav1.Now(), + Conditions: pacemaker.CreateInsufficientNodesClusterConditions(), + Nodes: &[]pacmkrv1.PacemakerClusterNodeStatus{ + pacemaker.CreateHealthyNodeStatus("master-0", []string{"192.168.1.10"}), + }, + }, + }, + expectedStatus: pacemaker.StatusError, + expectErrors: true, + errorContains: "Insufficient nodes", + }, + { + name: "no_nodes", + cr: &pacmkrv1.PacemakerCluster{ + Status: pacmkrv1.PacemakerClusterStatus{ + LastUpdated: metav1.Now(), + Conditions: pacemaker.CreateHealthyClusterConditions(), + Nodes: &[]pacmkrv1.PacemakerClusterNodeStatus{}, + }, + }, + expectedStatus: pacemaker.StatusError, + expectErrors: true, + errorContains: "No nodes found", + }, + { + name: "unpopulated_status", + cr: &pacmkrv1.PacemakerCluster{ + Status: pacmkrv1.PacemakerClusterStatus{}, // Zero LastUpdated + }, + expectedStatus: pacemaker.StatusUnknown, + expectErrors: true, + errorContains: "Internal error", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + status := pacemaker.BuildHealthStatusFromCR(tt.cr) + + require.NotNil(t, status, "pacemaker.HealthStatus should not be nil") + require.Equal(t, tt.expectedStatus, status.OverallStatus) + + if tt.expectErrors { + require.NotEmpty(t, status.Errors, "Should have errors") + if tt.errorContains != "" { + found := false + for _, err := range status.Errors { + if strings.Contains(err, tt.errorContains) { + found = true + break + } + } + require.True(t, found, "Error should contain %q", tt.errorContains) + } + } else { + require.Empty(t, status.Errors, "Should have no errors") + require.Empty(t, status.Warnings, "Should have no warnings") + } + }) + } +} + +func TestHealthCheck_recordErrorEvent(t *testing.T) { + controller := createTestHealthCheck() + recorder := controller.eventRecorder.(events.InMemoryRecorder) + + // Test different error types get the correct event reason + tests := []struct { + name string + errorMsg string + expectedReason string + }{ + { + name: "resource_unhealthy", + errorMsg: "Kubelet resource is unhealthy on node master-0: resource has failed", + expectedReason: pacemaker.EventReasonResourceUnhealthy, + }, + { + name: "node_unhealthy", + errorMsg: "master-0 node is unhealthy: node has issues", + expectedReason: pacemaker.EventReasonNodeUnhealthy, + }, + { + name: "cluster_unhealthy", + errorMsg: "Cluster is unhealthy: Pacemaker cluster has issues that need investigation", + expectedReason: pacemaker.EventReasonClusterUnhealthy, + }, + { + name: "insufficient_nodes", + errorMsg: "Insufficient nodes in cluster (expected 2, found 1)", + expectedReason: pacemaker.EventReasonInsufficientNodes, + }, + { + name: "status_stale", + errorMsg: "Pacemaker status is stale", + expectedReason: pacemaker.EventReasonStatusStale, + }, + { + name: "cr_not_found", + errorMsg: "Failed to get PacemakerCluster CR: not found", + expectedReason: pacemaker.EventReasonCRNotFound, + }, + { + name: "cr_no_status", + errorMsg: "PacemakerCluster CR has no status populated", + expectedReason: pacemaker.EventReasonCRNotFound, + }, + { + name: "node_offline", + errorMsg: "Node master-0 is offline", + expectedReason: pacemaker.EventReasonNodeOffline, + }, + { + name: "excessive_nodes", + errorMsg: "Excessive nodes in cluster (expected 2, found 3)", + expectedReason: pacemaker.EventReasonInsufficientNodes, + }, + { + name: "cluster_maintenance", + errorMsg: "Cluster is in maintenance mode", + expectedReason: pacemaker.EventReasonClusterInMaintenance, + }, + { + name: "generic_error", + errorMsg: "Some unknown error", + expectedReason: pacemaker.EventReasonGenericError, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + initialCount := len(recorder.Events()) + controller.recordErrorEvent(tt.errorMsg) + + events := recorder.Events() + require.Len(t, events, initialCount+1, "Should have recorded one more event") + + // Get the last event + lastEvent := events[len(events)-1] + require.Equal(t, tt.expectedReason, lastEvent.Reason, "Event reason should match expected") + require.Contains(t, lastEvent.Message, tt.errorMsg, "Event message should contain error message") + }) + } +} + +func TestHealthCheck_eventDeduplication_StatusTransitions(t *testing.T) { + controller := createTestHealthCheck() + recorder := controller.eventRecorder.(events.InMemoryRecorder) + + // Track previous status through the test + var previousStatus *pacemaker.HealthStatus + + // Start with healthy status from Unknown state (first sync or recovering from stale) + // previousStatus=nil: PacemakerHealthy event fires on initial healthy status + healthyStatus := &pacemaker.HealthStatus{ + OverallStatus: pacemaker.StatusHealthy, + Warnings: []string{}, + Errors: []string{}, + } + controller.recordHealthCheckEvents(healthyStatus, previousStatus) // previous=nil (first sync) + previousStatus = healthyStatus + afterFirstHealthy := len(recorder.Events()) + require.Equal(t, 1, afterFirstHealthy, "Healthy event should fire on transition from Unknown") + + // Stay healthy - should not record + controller.recordHealthCheckEvents(healthyStatus, previousStatus) + require.Equal(t, afterFirstHealthy, len(recorder.Events()), "No event when staying healthy") + + // Transition to Warning - should record warning but not healthy event (already in healthy state) + warningStatus := &pacemaker.HealthStatus{ + OverallStatus: pacemaker.StatusWarning, + Warnings: []string{"Some warning"}, + Errors: []string{}, + } + controller.recordHealthCheckEvents(warningStatus, previousStatus) + previousStatus = warningStatus + afterWarning := len(recorder.Events()) + require.Equal(t, afterFirstHealthy+1, afterWarning, "Warning event should be recorded") + + // Transition from Warning to Healthy - should record PacemakerWarningsCleared event + // previousStatus has warnings, so WarningsCleared should fire + controller.recordHealthCheckEvents(healthyStatus, previousStatus) + previousStatus = healthyStatus + afterSecondHealthy := len(recorder.Events()) + require.Equal(t, afterWarning+1, afterSecondHealthy, "PacemakerWarningsCleared event should fire on Warning to Healthy transition") + + // Transition to Error + errorStatus := &pacemaker.HealthStatus{ + OverallStatus: pacemaker.StatusError, + Warnings: []string{}, + Errors: []string{"Critical error"}, + } + controller.recordHealthCheckEvents(errorStatus, previousStatus) + previousStatus = errorStatus + afterError := len(recorder.Events()) + require.Greater(t, afterError, afterSecondHealthy, "Error should be recorded") + + // Transition from Error to Healthy - should record PacemakerHealthy event + // previousStatus is errorStatus (degraded), so healthy event should fire + controller.recordHealthCheckEvents(healthyStatus, previousStatus) + afterThirdHealthy := len(recorder.Events()) + require.Equal(t, afterError+1, afterThirdHealthy, "Healthy event should be recorded on transition from Error") +} + +func TestHealthCheck_eventDeduplication_CleanupOldEntries(t *testing.T) { + controller := createTestHealthCheck() + + // Add a very old entry to the recorded events map + oldEventKey := "warning:Very old warning" + controller.recordedEventsMu.Lock() + controller.recordedEvents[oldEventKey] = time.Now().Add(-25 * time.Hour) // 25 hours ago (older than longest window of 24 hours) + controller.recordedEventsMu.Unlock() + + // Trigger cleanup by recording a new event + previousStatus := &pacemaker.HealthStatus{OverallStatus: pacemaker.StatusHealthy, Warnings: []string{}, Errors: []string{}} + currentStatus := &pacemaker.HealthStatus{ + OverallStatus: pacemaker.StatusWarning, + Warnings: []string{"New warning"}, + Errors: []string{}, + } + controller.recordHealthCheckEvents(currentStatus, previousStatus) + + // Verify old entry was cleaned up + controller.recordedEventsMu.Lock() + _, exists := controller.recordedEvents[oldEventKey] + require.False(t, exists, "Old entry should have been cleaned up") + controller.recordedEventsMu.Unlock() +} + +// TestDetectDrift tests drift detection between K8s and pacemaker nodes +func TestDetectDrift(t *testing.T) { + tests := []struct { + name string + k8sNodes []*corev1.Node + pacemakerNodes map[string]string + expectedDrift bool + }{ + { + name: "no drift - nodes match", + k8sNodes: []*corev1.Node{ + createReadyNodeWithIPLM("master-0", "192.168.1.10"), + createReadyNodeWithIPLM("master-1", "192.168.1.11"), + }, + pacemakerNodes: map[string]string{ + "master-0": "192.168.1.10", + "master-1": "192.168.1.11", + }, + expectedDrift: false, + }, + { + name: "drift - count mismatch - K8s has fewer nodes", + k8sNodes: []*corev1.Node{ + createReadyNodeWithIPLM("master-0", "192.168.1.10"), + }, + pacemakerNodes: map[string]string{ + "master-0": "192.168.1.10", + "master-1": "192.168.1.11", + }, + expectedDrift: true, + }, + { + name: "drift - count mismatch - K8s has more nodes", + k8sNodes: []*corev1.Node{ + createReadyNodeWithIPLM("master-0", "192.168.1.10"), + createReadyNodeWithIPLM("master-1", "192.168.1.11"), + }, + pacemakerNodes: map[string]string{ + "master-0": "192.168.1.10", + }, + expectedDrift: true, + }, + { + name: "drift - node name mismatch", + k8sNodes: []*corev1.Node{ + createReadyNodeWithIPLM("master-0", "192.168.1.10"), + createReadyNodeWithIPLM("master-1", "192.168.1.11"), + }, + pacemakerNodes: map[string]string{ + "master-0": "192.168.1.10", + "master-2": "192.168.1.12", + }, + expectedDrift: true, + }, + { + name: "drift - IP mismatch IPv4", + k8sNodes: []*corev1.Node{ + createReadyNodeWithIPLM("master-0", "192.168.1.10"), + createReadyNodeWithIPLM("master-1", "192.168.1.99"), // Changed IP + }, + pacemakerNodes: map[string]string{ + "master-0": "192.168.1.10", + "master-1": "192.168.1.11", + }, + expectedDrift: true, + }, + { + name: "drift - IP mismatch IPv6", + k8sNodes: []*corev1.Node{ + createReadyNodeWithIPLM("master-0", "fd00::1"), + createReadyNodeWithIPLM("master-1", "fd00::99"), // Changed IP + }, + pacemakerNodes: map[string]string{ + "master-0": "fd00::1", + "master-1": "fd00::2", + }, + expectedDrift: true, + }, + { + name: "no drift - IPv6 different format but same IP", + k8sNodes: []*corev1.Node{ + createReadyNodeWithIPLM("master-0", "fd00::1"), + createReadyNodeWithIPLM("master-1", "fd00:0000:0000:0000:0000:0000:0000:0002"), + }, + pacemakerNodes: map[string]string{ + "master-0": "fd00:0000:0000:0000:0000:0000:0000:0001", + "master-1": "fd00::2", + }, + expectedDrift: false, + }, + { + name: "drift - node in pacemaker but not in K8s", + k8sNodes: []*corev1.Node{ + createReadyNodeWithIPLM("master-0", "192.168.1.10"), + }, + pacemakerNodes: map[string]string{ + "master-0": "192.168.1.10", + "master-1": "192.168.1.11", + }, + expectedDrift: true, + }, + { + name: "no drift - both empty", + k8sNodes: []*corev1.Node{}, + pacemakerNodes: map[string]string{}, + expectedDrift: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create lifecycle manager with mock + controller := &PacemakerLifecycleManager{} + + // Execute + hasDrift := controller.detectDrift(tt.k8sNodes, tt.pacemakerNodes) + + // Verify + require.Equal(t, tt.expectedDrift, hasDrift, + "detectDrift() = %v, expected %v", hasDrift, tt.expectedDrift) + }) + } +} + +// TestCleanupOrphanedJobs tests job cleanup for deleted nodes +func TestCleanupOrphanedJobs(t *testing.T) { + tests := []struct { + name string + k8sNodes []*corev1.Node + existingJobs []runtime.Object + expectJobsDeleted []string + expectJobsKept []string + }{ + { + name: "orphaned jobs deleted - node replaced (same name, different UID)", + k8sNodes: []*corev1.Node{ + createReadyNodeLM("master-0"), + // master-1 was replaced - new node with same name but different UID + { + ObjectMeta: metav1.ObjectMeta{ + Name: "master-1", + UID: types.UID("master-1-new-uid"), // NEW UID after replacement + }, + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + {Type: corev1.NodeReady, Status: corev1.ConditionTrue}, + }, + }, + }, + }, + existingJobs: []runtime.Object{ + createTNFJobWithNodeLabelLM("tnf-auth-job-master-0", "master-0", "tnf-auth-job"), + // Jobs from old master-1 node (before replacement) - labeled with old UID + createTNFJobWithNodeLabelLM("tnf-auth-job-master-1", "master-1", "tnf-auth-job"), // Orphaned (old UID) + createTNFJobWithNodeLabelLM("tnf-after-setup-master-1", "master-1", "tnf-after-setup-job"), // Orphaned (old UID) + // Note: update-setup jobs don't have node labels and won't be cleaned up by node deletion + }, + expectJobsDeleted: []string{ + "tnf-auth-job-master-1", // Old UID "master-1" doesn't match new UID "master-1-new-uid" + "tnf-after-setup-master-1", // Old UID "master-1" doesn't match new UID "master-1-new-uid" + }, + expectJobsKept: []string{ + "tnf-auth-job-master-0", + }, + }, + { + name: "active node jobs preserved", + k8sNodes: []*corev1.Node{ + createReadyNodeLM("master-0"), + createReadyNodeLM("master-1"), + }, + existingJobs: []runtime.Object{ + createTNFJobWithNodeLabelLM("tnf-auth-job-master-0", "master-0", "tnf-auth-job"), + createTNFJobWithNodeLabelLM("tnf-auth-job-master-1", "master-1", "tnf-auth-job"), + createTNFJobWithNodeLabelLM("tnf-after-setup-master-0", "master-0", "tnf-after-setup-job"), + }, + expectJobsDeleted: []string{}, + expectJobsKept: []string{ + "tnf-auth-job-master-0", + "tnf-auth-job-master-1", + "tnf-after-setup-master-0", + }, + }, + { + name: "job without node label deleted - migration cleanup", + k8sNodes: []*corev1.Node{ + createReadyNodeLM("master-0"), + }, + existingJobs: []runtime.Object{ + createTNFJobWithNodeLabelLM("tnf-auth-job-master-0", "master-0", "tnf-auth-job"), + createTNFJobWithoutNodeLabelLM("tnf-legacy-job", "tnf-auth-job"), // Old job without label - will be deleted and recreated with label + }, + expectJobsDeleted: []string{ + "tnf-legacy-job", // Deleted (no node label - migration cleanup) + }, + expectJobsKept: []string{ + "tnf-auth-job-master-0", + }, + }, + { + name: "no jobs - no action", + k8sNodes: []*corev1.Node{createReadyNodeLM("master-0")}, + existingJobs: []runtime.Object{}, + expectJobsDeleted: []string{}, + expectJobsKept: []string{}, + }, + { + name: "label selector filters correctly - only TNF jobs", + k8sNodes: []*corev1.Node{ + createReadyNodeLM("master-0"), + }, + existingJobs: []runtime.Object{ + createTNFJobWithNodeLabelLM("tnf-auth-job-master-1", "master-1", "tnf-auth-job"), // TNF job - should be deleted + createNonTNFJobWithNodeLabelLM("other-job-master-1", "master-1"), // Not TNF job - should not be touched + }, + expectJobsDeleted: []string{ + "tnf-auth-job-master-1", + }, + expectJobsKept: []string{ + "other-job-master-1", // Not a TNF job, left alone + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + + // Create fake client with existing jobs + fakeKubeClient := fake.NewSimpleClientset(tt.existingJobs...) + + // Create lifecycle manager + controller := &PacemakerLifecycleManager{ + kubeClient: fakeKubeClient, + } + + // Execute + err := controller.cleanupOrphanedJobs(ctx, tt.k8sNodes) + + // Verify no error + require.NoError(t, err) + + // Verify deleted jobs are gone + for _, jobName := range tt.expectJobsDeleted { + _, err := fakeKubeClient.BatchV1().Jobs("openshift-etcd").Get(ctx, jobName, metav1.GetOptions{}) + require.True(t, apierrors.IsNotFound(err), "Expected job %s to be deleted", jobName) + } + + // Verify kept jobs still exist + for _, jobName := range tt.expectJobsKept { + _, err := fakeKubeClient.BatchV1().Jobs("openshift-etcd").Get(ctx, jobName, metav1.GetOptions{}) + require.NoError(t, err, "Expected job %s to still exist", jobName) + } + }) + } +} + +// TestReconcilePacemakerConfig tests the core reconciliation logic +func TestReconcilePacemakerConfig(t *testing.T) { + tests := []struct { + name string + transitionComplete bool + nodeInformerSynced bool + k8sNodes []*corev1.Node + pacemakerNodes map[string]string + updateSetupJobs []runtime.Object + mockUpdateSetupError error + expectError bool + errorContains string + expectUpdateSetupCalled bool + }{ + { + name: "no pacemaker CR - discovery reconciliation", + transitionComplete: true, // Note: In production, sync() would not call this before transition + nodeInformerSynced: true, + k8sNodes: []*corev1.Node{ + createReadyNodeWithIPLM("master-0", "192.168.1.10"), + }, + // pacemakerNodes is nil - will trigger discovery reconciliation (try all nodes) + expectError: false, + expectUpdateSetupCalled: true, + }, + { + name: "node informer not synced - skip", + transitionComplete: true, + nodeInformerSynced: false, + k8sNodes: []*corev1.Node{ + createReadyNodeWithIPLM("master-0", "192.168.1.10"), + }, + expectError: false, + expectUpdateSetupCalled: false, + }, + { + name: "more than 2 nodes - skip", + transitionComplete: true, + nodeInformerSynced: true, + k8sNodes: []*corev1.Node{ + createReadyNodeWithIPLM("master-0", "192.168.1.10"), + createReadyNodeWithIPLM("master-1", "192.168.1.11"), + createReadyNodeWithIPLM("master-2", "192.168.1.12"), + }, + expectError: false, + expectUpdateSetupCalled: false, + }, + { + name: "node not ready during bootstrap - skip", + transitionComplete: false, + nodeInformerSynced: true, + k8sNodes: []*corev1.Node{ + createReadyNodeWithIPLM("master-0", "192.168.1.10"), + createNotReadyNodeLM("master-1"), + }, + expectError: false, + expectUpdateSetupCalled: false, // Should skip during bootstrap until all nodes Ready + }, + { + name: "node not ready after transition - reconcile with ready nodes only", + transitionComplete: true, + nodeInformerSynced: true, + k8sNodes: []*corev1.Node{ + createReadyNodeWithIPLM("master-0", "192.168.1.10"), + createNotReadyNodeWithIPLM("master-1", "192.168.1.11"), + }, + pacemakerNodes: map[string]string{ + "master-0": "192.168.1.10", + "master-1": "192.168.1.11", + }, + expectError: false, + expectUpdateSetupCalled: false, // No drift - both nodes match, so no reconciliation needed + }, + { + name: "no drift - no action", + transitionComplete: true, + nodeInformerSynced: true, + k8sNodes: []*corev1.Node{ + createReadyNodeWithIPLM("master-0", "192.168.1.10"), + createReadyNodeWithIPLM("master-1", "192.168.1.11"), + }, + pacemakerNodes: map[string]string{ + "master-0": "192.168.1.10", + "master-1": "192.168.1.11", + }, + expectError: false, + expectUpdateSetupCalled: false, + }, + { + name: "drift - node count mismatch - trigger update-setup", + transitionComplete: true, + nodeInformerSynced: true, + k8sNodes: []*corev1.Node{ + createReadyNodeWithIPLM("master-0", "192.168.1.10"), + }, + pacemakerNodes: map[string]string{ + "master-0": "192.168.1.10", + "master-1": "192.168.1.11", + }, + expectError: false, + expectUpdateSetupCalled: true, + }, + { + name: "drift - IP mismatch - trigger update-setup", + transitionComplete: true, + nodeInformerSynced: true, + k8sNodes: []*corev1.Node{ + createReadyNodeWithIPLM("master-0", "192.168.1.10"), + createReadyNodeWithIPLM("master-1", "192.168.1.99"), + }, + pacemakerNodes: map[string]string{ + "master-0": "192.168.1.10", + "master-1": "192.168.1.11", + }, + expectError: false, + expectUpdateSetupCalled: true, + }, + { + name: "drift - update-setup already running - still reconcile", + transitionComplete: true, + nodeInformerSynced: true, + k8sNodes: []*corev1.Node{ + createReadyNodeWithIPLM("master-0", "192.168.1.10"), + }, + pacemakerNodes: map[string]string{ + "master-0": "192.168.1.10", + "master-1": "192.168.1.11", + }, + updateSetupJobs: []runtime.Object{ + createRunningUpdateSetupJobLM("tnf-update-setup-job"), + }, + expectError: false, + expectUpdateSetupCalled: true, + }, + { + name: "drift - no intersection - error", + transitionComplete: true, + nodeInformerSynced: true, + k8sNodes: []*corev1.Node{ + createReadyNodeWithIPLM("master-0", "192.168.1.10"), + }, + pacemakerNodes: map[string]string{ + "master-1": "192.168.1.11", + "master-2": "192.168.1.12", + }, + expectError: true, + errorContains: "no nodes in both K8s and pacemaker", + expectUpdateSetupCalled: false, + }, + { + name: "drift - update-setup fails - return error", + transitionComplete: true, + nodeInformerSynced: true, + k8sNodes: []*corev1.Node{ + createReadyNodeWithIPLM("master-0", "192.168.1.10"), + }, + pacemakerNodes: map[string]string{ + "master-0": "192.168.1.10", + "master-1": "192.168.1.11", + }, + mockUpdateSetupError: fmt.Errorf("update-setup failed"), + expectError: true, + errorContains: "update-setup failed", + expectUpdateSetupCalled: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + + // Create fake clients + fakeKubeClient := fake.NewSimpleClientset(tt.updateSetupJobs...) + + // Create fake operator client + var conditions []operatorv1.OperatorCondition + if tt.transitionComplete { + conditions = append(conditions, operatorv1.OperatorCondition{ + Type: "ExternalEtcdHasCompletedTransition", + Status: operatorv1.ConditionTrue, + }) + } + fakeOperatorClient := v1helpers.NewFakeStaticPodOperatorClient( + &operatorv1.StaticPodOperatorSpec{}, + &operatorv1.StaticPodOperatorStatus{ + OperatorStatus: operatorv1.OperatorStatus{ + Conditions: conditions, + }, + }, + nil, + nil, + ) + + // Create node informer + nodeIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{}) + for _, node := range tt.k8sNodes { + require.NoError(t, nodeIndexer.Add(node), "failed to add node to indexer") + } + nodeInformer := &mockNodeInformerLM{ + indexer: nodeIndexer, + synced: tt.nodeInformerSynced, + } + + // Create pacemaker informer + pacemakerInformer := createPacemakerInformerWithNodesLM(tt.pacemakerNodes) + + // Create lifecycle manager + controller := &PacemakerLifecycleManager{ + operatorClient: fakeOperatorClient, + kubeClient: fakeKubeClient, + nodeInformer: nodeInformer, + pacemakerInformer: pacemakerInformer, + } + + // Mock startTnfJobcontrollers (called from ReconcilePacemakerConfig before transition check) + originalStartFunc := startTnfJobcontrollersFunc + startTnfJobcontrollersFunc = func(ctx context.Context, nodes []*corev1.Node, controllerContext *controllercmd.ControllerContext, operatorClient v1helpers.StaticPodOperatorClient, kubeClient kubernetes.Interface, kubeInformersForNamespaces v1helpers.KubeInformersForNamespaces, etcdInformer operatorv1informers.EtcdInformer, lifecycleManager *PacemakerLifecycleManager) error { + // No-op for tests - just return success + return nil + } + defer func() { startTnfJobcontrollersFunc = originalStartFunc }() + + // Mock updateSetup + updateSetupCalled := false + originalUpdateSetup := updateSetupFunc + updateSetupFunc = func(validTargetNodes []*corev1.Node, validNodeFunc jobs.TargetNodesFunc, allK8sNodes []*corev1.Node, pacemakerNodes map[string]string, ctx context.Context, controllerContext *controllercmd.ControllerContext, operatorClient v1helpers.StaticPodOperatorClient, kubeClient kubernetes.Interface, kubeInformersForNamespaces v1helpers.KubeInformersForNamespaces) error { + updateSetupCalled = true + return tt.mockUpdateSetupError + } + defer func() { updateSetupFunc = originalUpdateSetup }() + + // Execute + err := controller.ReconcilePacemakerConfig(ctx) + + // Verify error expectations + if tt.expectError { + require.Error(t, err) + if tt.errorContains != "" { + require.Contains(t, err.Error(), tt.errorContains) + } + } else { + require.NoError(t, err) + } + + // Verify updateSetup was called as expected + require.Equal(t, tt.expectUpdateSetupCalled, updateSetupCalled, + "updateSetup called=%v, expected=%v", updateSetupCalled, tt.expectUpdateSetupCalled) + }) + } +} + +// Helper functions for test setup + +func createReadyNodeLM(name string) *corev1.Node { + return &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(name), // Use name as UID for test simplicity + }, + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + { + Type: corev1.NodeReady, + Status: corev1.ConditionTrue, + }, + }, + }, + } +} + +func createNotReadyNodeLM(name string) *corev1.Node { + return &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + { + Type: corev1.NodeReady, + Status: corev1.ConditionFalse, + }, + }, + }, + } +} + +func createNotReadyNodeWithIPLM(name, ip string) *corev1.Node { + return &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Status: corev1.NodeStatus{ + Addresses: []corev1.NodeAddress{ + {Type: corev1.NodeInternalIP, Address: ip}, + }, + Conditions: []corev1.NodeCondition{ + { + Type: corev1.NodeReady, + Status: corev1.ConditionFalse, + }, + }, + }, + } +} + +func createReadyNodeWithIPLM(name, ip string) *corev1.Node { + node := createReadyNodeLM(name) + node.Status.Addresses = []corev1.NodeAddress{ + {Type: corev1.NodeInternalIP, Address: ip}, + } + return node +} + +func createTNFJobWithNodeLabelLM(name, nodeName, jobType string) *batchv1.Job { + return &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "openshift-etcd", + Labels: map[string]string{ + "app.kubernetes.io/name": jobType, + "node": nodeName, + }, + }, + } +} + +func createTNFJobWithoutNodeLabelLM(name, jobType string) *batchv1.Job { + return &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "openshift-etcd", + Labels: map[string]string{ + "app.kubernetes.io/name": jobType, + // No "node" label + }, + }, + } +} + +func createNonTNFJobWithNodeLabelLM(name, nodeName string) *batchv1.Job { + return &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "openshift-etcd", + Labels: map[string]string{ + "app.kubernetes.io/name": "some-other-job", // Not a TNF job + "node": nodeName, + }, + }, + } +} + +func createRunningUpdateSetupJobLM(name string) *batchv1.Job { + return &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "openshift-etcd", + Labels: map[string]string{ + "app.kubernetes.io/name": "tnf-update-setup-job", + "app.kubernetes.io/component": "two-node-fencing-setup", + }, + }, + Status: batchv1.JobStatus{ + // No Complete or Failed condition = still running + }, + } +} + +func createPacemakerInformerWithNodesLM(nodes map[string]string) cache.SharedIndexInformer { + if nodes == nil { + // No CR exists + return createFakeInformer(nil) + } + + // Create PacemakerCluster CR with the given nodes + nodeStatuses := []pacmkrv1.PacemakerClusterNodeStatus{} + for name, ip := range nodes { + nodeStatuses = append(nodeStatuses, pacmkrv1.PacemakerClusterNodeStatus{ + NodeName: name, + Addresses: []pacmkrv1.PacemakerNodeAddress{ + {Address: ip}, + }, + }) + } + + pacemakerCR := &pacmkrv1.PacemakerCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster", + }, + Status: pacmkrv1.PacemakerClusterStatus{ + Nodes: &nodeStatuses, + LastUpdated: metav1.Now(), + }, + } + + return createFakeInformer(pacemakerCR) +} + +// mockNodeInformerLM is a mock SharedIndexInformer for testing +type mockNodeInformerLM struct { + indexer cache.Indexer + synced bool +} + +func (m *mockNodeInformerLM) AddEventHandler(handler cache.ResourceEventHandler) (cache.ResourceEventHandlerRegistration, error) { + return nil, nil +} +func (m *mockNodeInformerLM) AddEventHandlerWithResyncPeriod(handler cache.ResourceEventHandler, resyncPeriod time.Duration) (cache.ResourceEventHandlerRegistration, error) { + return nil, nil +} +func (m *mockNodeInformerLM) AddEventHandlerWithOptions(handler cache.ResourceEventHandler, options cache.HandlerOptions) (cache.ResourceEventHandlerRegistration, error) { + return nil, nil +} +func (m *mockNodeInformerLM) RemoveEventHandler(handle cache.ResourceEventHandlerRegistration) error { + return nil +} +func (m *mockNodeInformerLM) GetStore() cache.Store { return m.indexer } +func (m *mockNodeInformerLM) GetIndexer() cache.Indexer { return m.indexer } +func (m *mockNodeInformerLM) GetController() cache.Controller { return nil } +func (m *mockNodeInformerLM) Run(stopCh <-chan struct{}) {} +func (m *mockNodeInformerLM) RunWithContext(ctx context.Context) {} +func (m *mockNodeInformerLM) HasSynced() bool { return m.synced } +func (m *mockNodeInformerLM) LastSyncResourceVersion() string { return "" } +func (m *mockNodeInformerLM) SetWatchErrorHandler(handler cache.WatchErrorHandler) error { + return nil +} +func (m *mockNodeInformerLM) SetWatchErrorHandlerWithContext(handler cache.WatchErrorHandlerWithContext) error { + return nil +} +func (m *mockNodeInformerLM) SetTransform(handler cache.TransformFunc) error { return nil } +func (m *mockNodeInformerLM) IsStopped() bool { return false } +func (m *mockNodeInformerLM) AddIndexers(indexers cache.Indexers) error { return nil } diff --git a/pkg/tnf/operator/lifecycle_status_collector.go b/pkg/tnf/operator/lifecycle_status_collector.go new file mode 100644 index 0000000000..fe8caa83dd --- /dev/null +++ b/pkg/tnf/operator/lifecycle_status_collector.go @@ -0,0 +1,222 @@ +package operator + +import ( + "context" + "fmt" + "os" + "sort" + + operatorv1 "github.com/openshift/api/operator/v1" + "github.com/openshift/cluster-etcd-operator/bindata" + "github.com/openshift/cluster-etcd-operator/pkg/operator/operatorclient" + "github.com/openshift/cluster-etcd-operator/pkg/tnf/pkg/jobs" + "github.com/openshift/library-go/pkg/controller/controllercmd" + "github.com/openshift/library-go/pkg/operator/v1helpers" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" +) + +const ( + // pacemakerStatusCollectorName is the name of the Pacemaker status collector CronJob + pacemakerStatusCollectorName = "pacemaker-status-collector" +) + +// runPacemakerStatusCollectorCronJob starts the CronJob controller for periodic pacemaker status collection. +// The CronJob runs "tnf-monitor collect" which executes "sudo -n pcs status xml" and updates the PacemakerCluster CR. +// validNodeFunc is called on each sync to determine which nodes are valid targets for the collector. +func runPacemakerStatusCollectorCronJob( + ctx context.Context, + controllerContext *controllercmd.ControllerContext, + operatorClient v1helpers.StaticPodOperatorClient, + kubeClient kubernetes.Interface, + validNodeFunc jobs.TargetNodesFunc, +) { + // Start the cronjob controller to create a CronJob for periodic status collection. + statusCronJobController := jobs.NewCronJobController( + pacemakerStatusCollectorName, + bindata.MustAsset("tnfdeployment/cronjob.yaml"), + operatorClient, + kubeClient, + controllerContext.EventRecorder, + func(_ *operatorv1.OperatorSpec, cronJob *batchv1.CronJob) error { + // Set the name and namespace + cronJob.SetName(pacemakerStatusCollectorName) + cronJob.SetNamespace(operatorclient.TargetNamespace) + + // Set the schedule - run every minute + cronJob.Spec.Schedule = "* * * * *" + + // Initialize labels maps if nil and set labels at all levels + if cronJob.Labels == nil { + cronJob.Labels = make(map[string]string) + } + cronJob.Labels["app.kubernetes.io/name"] = pacemakerStatusCollectorName + + if cronJob.Spec.JobTemplate.Labels == nil { + cronJob.Spec.JobTemplate.Labels = make(map[string]string) + } + cronJob.Spec.JobTemplate.Labels["app.kubernetes.io/name"] = pacemakerStatusCollectorName + + if cronJob.Spec.JobTemplate.Spec.Template.Labels == nil { + cronJob.Spec.JobTemplate.Spec.Template.Labels = make(map[string]string) + } + cronJob.Spec.JobTemplate.Spec.Template.Labels["app.kubernetes.io/name"] = pacemakerStatusCollectorName + + // Configure the container + cronJob.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Image = os.Getenv("OPERATOR_IMAGE") + cronJob.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Command = []string{"tnf-monitor", "collect", "-v=4"} + + // Get valid nodes using the provided function (same logic as update-setup job) + validNodes, err := validNodeFunc() + if err != nil || len(validNodes) == 0 { + klog.Warningf("Failed to determine valid nodes for status collector: %v - falling back to nodeSelector from manifest", err) + // On error, rely on existing nodeSelector from manifest + return nil + } + + // Sort nodes for deterministic ordering + sort.Slice(validNodes, func(i, j int) bool { + return validNodes[i].Name < validNodes[j].Name + }) + + // Determine target node based on last job status + targetNode, err := determineStatusCollectorNode(ctx, kubeClient, validNodes) + if err != nil { + klog.Warningf("Failed to determine target node for status collector: %v - using first node", err) + targetNode = validNodes[0].Name + } + + // Pin to specific node (rotate on failure, stick with success) + cronJob.Spec.JobTemplate.Spec.Template.Spec.NodeName = targetNode + // Clear affinity to ensure NodeName takes precedence + cronJob.Spec.JobTemplate.Spec.Template.Spec.Affinity = nil + + klog.V(2).Infof("Status collector pinned to node: %s", targetNode) + return nil + }, + ) + go statusCronJobController.Run(ctx, 1) +} + +// determineStatusCollectorNode selects the target node for the status collector job. +// Strategy: +// - No job exists: use first node in sorted list +// - Last node not in valid list: use first node (node was removed) +// - Job failed: rotate to next node in list +// - Job succeeded: keep same node (stick with success) +func determineStatusCollectorNode(ctx context.Context, kubeClient kubernetes.Interface, validNodes []*corev1.Node) (string, error) { + if len(validNodes) == 0 { + return "", fmt.Errorf("no valid nodes provided") + } + + // Query recent Jobs created by the CronJob + jobs, err := kubeClient.BatchV1().Jobs(operatorclient.TargetNamespace).List(ctx, metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/name=" + pacemakerStatusCollectorName, + }) + if err != nil { + return "", fmt.Errorf("failed to list jobs: %w", err) + } + + // Find most recent job + lastJob := getMostRecentJob(jobs.Items) + if lastJob == nil { + // No job exists yet - start with first node + klog.V(2).Infof("Status collector: no prior jobs, starting with first node %s", validNodes[0].Name) + return validNodes[0].Name, nil + } + + // Get the node the last job ran on (or was scheduled to) + lastNode := getNodeFromJob(lastJob) + if lastNode == "" { + klog.Warningf("Status collector: could not determine node from last job, using first node") + return validNodes[0].Name, nil + } + + // Check if last node is still in valid list + nodeIndex := findNodeIndex(validNodes, lastNode) + if nodeIndex == -1 { + // Last node no longer valid (removed from cluster) - start over with first node + klog.V(2).Infof("Status collector: last node %s no longer valid, starting over with first node %s", lastNode, validNodes[0].Name) + return validNodes[0].Name, nil + } + + // Check if last job failed + if isJobFailed(lastJob) { + // Job failed - rotate to next node + nextNode := getNextNodeInRotation(validNodes, nodeIndex) + klog.V(2).Infof("Status collector: job failed on %s, rotating to %s", lastNode, nextNode) + return nextNode, nil + } + + // Job succeeded - keep same node + klog.V(2).Infof("Status collector: job succeeded on %s, keeping same node", lastNode) + return lastNode, nil +} + +// getMostRecentJob returns the job with the latest creation timestamp +func getMostRecentJob(jobs []batchv1.Job) *batchv1.Job { + if len(jobs) == 0 { + return nil + } + + var mostRecent *batchv1.Job + for i := range jobs { + if mostRecent == nil || jobs[i].CreationTimestamp.After(mostRecent.CreationTimestamp.Time) { + mostRecent = &jobs[i] + } + } + return mostRecent +} + +// isJobFailed returns true if the job has failed +func isJobFailed(job *batchv1.Job) bool { + // Check conditions for Failed type + for _, condition := range job.Status.Conditions { + if condition.Type == batchv1.JobFailed && condition.Status == corev1.ConditionTrue { + return true + } + } + // Also check if failed count is non-zero + return job.Status.Failed > 0 +} + +// getNodeFromJob extracts the node name from the job spec +func getNodeFromJob(job *batchv1.Job) string { + // Check NodeName first (most reliable) + if job.Spec.Template.Spec.NodeName != "" { + return job.Spec.Template.Spec.NodeName + } + + // Fallback: check node affinity if NodeName not set + affinity := job.Spec.Template.Spec.Affinity + if affinity != nil && affinity.NodeAffinity != nil && affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution != nil { + for _, term := range affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms { + for _, expr := range term.MatchExpressions { + if expr.Key == "kubernetes.io/hostname" && expr.Operator == corev1.NodeSelectorOpIn && len(expr.Values) > 0 { + return expr.Values[0] + } + } + } + } + + return "" +} + +// findNodeIndex returns the index of the node in the list, or -1 if not found +func findNodeIndex(nodes []*corev1.Node, nodeName string) int { + for i, node := range nodes { + if node.Name == nodeName { + return i + } + } + return -1 +} + +// getNextNodeInRotation returns the next node in the sorted list (wraps around) +func getNextNodeInRotation(nodes []*corev1.Node, currentIndex int) string { + nextIndex := (currentIndex + 1) % len(nodes) + return nodes[nextIndex].Name +} diff --git a/pkg/tnf/operator/nodehandler.go b/pkg/tnf/operator/nodehandler.go deleted file mode 100644 index ad72bba084..0000000000 --- a/pkg/tnf/operator/nodehandler.go +++ /dev/null @@ -1,320 +0,0 @@ -package operator - -import ( - "context" - "fmt" - "sync" - "time" - - operatorv1 "github.com/openshift/api/operator/v1" - operatorv1informers "github.com/openshift/client-go/operator/informers/externalversions/operator/v1" - "github.com/openshift/library-go/pkg/controller/controllercmd" - "github.com/openshift/library-go/pkg/operator/v1helpers" - corev1 "k8s.io/api/core/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/kubernetes" - corev1listers "k8s.io/client-go/listers/core/v1" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/cache" - "k8s.io/klog/v2" - - "github.com/openshift/cluster-etcd-operator/pkg/operator/bootstrapteardown" - "github.com/openshift/cluster-etcd-operator/pkg/operator/ceohelpers" - "github.com/openshift/cluster-etcd-operator/pkg/operator/operatorclient" - "github.com/openshift/cluster-etcd-operator/pkg/tnf/pkg/etcd" - "github.com/openshift/cluster-etcd-operator/pkg/tnf/pkg/jobs" - "github.com/openshift/cluster-etcd-operator/pkg/tnf/pkg/tools" -) - -var ( - // handleNodesMutex ensures that node handling code doesn't run concurrently - handleNodesMutex sync.Mutex - - // handleNodesFunc is a variable to allow mocking in tests - handleNodesFunc = handleNodes - - // startTnfJobcontrollersFunc is a variable to allow mocking in tests - startTnfJobcontrollersFunc = startTnfJobcontrollers - - // updateSetupFunc is a variable to allow mocking in tests - updateSetupFunc = updateSetup - - // retryBackoffConfig allows customizing retry behavior for tests - retryBackoffConfig = wait.Backoff{ - Duration: 5 * time.Second, - Factor: 2.0, - Steps: 9, // ~10 minutes total: 5s + 10s + 20s + 40s + 80s + 120s + 120s + 120s + 120s - Cap: 2 * time.Minute, - } -) - -func handleNodesWithRetry( - controllerContext *controllercmd.ControllerContext, - controlPlaneNodeLister corev1listers.NodeLister, - ctx context.Context, - operatorClient v1helpers.StaticPodOperatorClient, - kubeClient kubernetes.Interface, - kubeInformersForNamespaces v1helpers.KubeInformersForNamespaces, - etcdInformer operatorv1informers.EtcdInformer, -) { - - // Ensure only one execution at a time - don't run in parallel - handleNodesMutex.Lock() - defer handleNodesMutex.Unlock() - - // Retry with exponential backoff to handle transient failures - var setupErr error - err := wait.ExponentialBackoffWithContext(ctx, retryBackoffConfig, func(ctx context.Context) (bool, error) { - setupErr = handleNodesFunc(controllerContext, controlPlaneNodeLister, ctx, operatorClient, kubeClient, kubeInformersForNamespaces, etcdInformer) - if setupErr != nil { - klog.Warningf("failed to setup TNF job controllers, will retry: %v", setupErr) - return false, nil - } - return true, nil - }) - - if err != nil || setupErr != nil { - klog.Errorf("failed to setup TNF job controllers after 10 minutes of retries: %v", setupErr) - - // Degrade the operator to indicate TNF job controller setup failed - _, _, updateErr := v1helpers.UpdateStatus(ctx, operatorClient, v1helpers.UpdateConditionFn(operatorv1.OperatorCondition{ - Type: "TNFJobControllersDegraded", - Status: operatorv1.ConditionTrue, - Reason: "SetupFailed", - Message: fmt.Sprintf("Failed to setup TNF job controllers after retries: %v", setupErr), - })) - if updateErr != nil { - klog.Errorf("failed to update operator status to degraded: %v", updateErr) - } - } else { - // Clear any previous degraded condition on success - _, _, updateErr := v1helpers.UpdateStatus(ctx, operatorClient, v1helpers.UpdateConditionFn(operatorv1.OperatorCondition{ - Type: "TNFJobControllersDegraded", - Status: operatorv1.ConditionFalse, - Reason: "AsExpected", - Message: "TNF job controllers setup completed successfully", - })) - if updateErr != nil { - klog.Errorf("failed to update operator status: %v", updateErr) - } - } -} - -func handleNodes( - controllerContext *controllercmd.ControllerContext, - controlPlaneNodeLister corev1listers.NodeLister, - ctx context.Context, - operatorClient v1helpers.StaticPodOperatorClient, - kubeClient kubernetes.Interface, - kubeInformersForNamespaces v1helpers.KubeInformersForNamespaces, - etcdInformer operatorv1informers.EtcdInformer, -) error { - - // ensure we have 2 control plane nodes before doing anything - nodeList, err := controlPlaneNodeLister.List(labels.Everything()) - if err != nil { - return fmt.Errorf("failed to list control plane nodes: %w", err) - } - if len(nodeList) > 2 { - klog.Warningf("found more than 2 control plane nodes (%d), unsupported use case, no further steps are taken for now", len(nodeList)) - // don't retry - return nil - } - if len(nodeList) < 2 { - klog.Warningf("found a single control plane node only, waiting for the second one") - // don't retry - return nil - } - bothReady := true - for _, node := range nodeList { - if !tools.IsNodeReady(node) { - klog.Warningf("node %q is not ready yet, waiting for it to become ready", node.GetName()) - bothReady = false - } - } - if !bothReady { - // a node condition change will retrigger node handling automatically, no need to trigger a retry here - return nil - } - - klog.Infof("found 2 control plane nodes (%q, %q)", nodeList[0].GetName(), nodeList[1].GetName()) - - // check if TNF was already set up, by looking for existing jobs - jobsExist, err := tnfSetupJobsExist(ctx, kubeClient) - if err != nil { - return fmt.Errorf("failed to check for existing TNF jobs: %w", err) - } - - // always start job controllers, otherwise jobs won't be recreated after a CEO restart - err = startTnfJobcontrollersFunc(nodeList, ctx, controllerContext, operatorClient, kubeClient, kubeInformersForNamespaces, etcdInformer) - if err != nil { - return fmt.Errorf("failed to start TNF job controllers: %w", err) - } - - // in case TNF was not setup yet, we are done here - if !jobsExist { - return nil - } - - // if TNF was already set up, we might need to update - err = updateSetupFunc(nodeList, ctx, controllerContext, operatorClient, kubeClient, kubeInformersForNamespaces) - if err != nil { - return fmt.Errorf("failed to update pacemaker setup: %w", err) - } - - return nil -} - -func startTnfJobcontrollers( - nodeList []*corev1.Node, - ctx context.Context, - controllerContext *controllercmd.ControllerContext, - operatorClient v1helpers.StaticPodOperatorClient, - kubeClient kubernetes.Interface, - kubeInformersForNamespaces v1helpers.KubeInformersForNamespaces, - etcdInformer operatorv1informers.EtcdInformer) error { - - klog.Infof("Running TNF setup procedure. Waiting for etcd bootstrap to complete") - - // Wait for the etcd informer to sync before checking bootstrap status - // This ensures operatorClient.GetStaticPodOperatorState() has data to work with - klog.Infof("waiting for etcd informer to sync...") - if !cache.WaitForCacheSync(ctx.Done(), etcdInformer.Informer().HasSynced) { - return fmt.Errorf("failed to sync etcd informer") - } - klog.Infof("etcd informer synced") - - if err := waitForEtcdBootstrapCompleted(ctx, operatorClient); err != nil { - return fmt.Errorf("failed to wait for etcd bootstrap: %w", err) - } - - // Wait for all nodes to have their installers complete (creates /var/lib/etcd) - klog.Infof("bootstrap completed, waiting for all nodes to reach latest revision") - if err := etcd.WaitForStableRevision(ctx, operatorClient); err != nil { - return fmt.Errorf("failed to wait for all nodes at latest revision: %w", err) - } - - klog.Infof("all nodes at latest revision, creating TNF job controllers") - - // the order of job creation does not matter, the jobs wait on each other as needed - for _, node := range nodeList { - jobs.RunTNFJobController(ctx, tools.JobTypeAuth, &node.Name, controllerContext, operatorClient, kubeClient, kubeInformersForNamespaces, jobs.DefaultConditions) - jobs.RunTNFJobController(ctx, tools.JobTypeAfterSetup, &node.Name, controllerContext, operatorClient, kubeClient, kubeInformersForNamespaces, jobs.DefaultConditions) - } - - jobs.RunTNFJobController(ctx, tools.JobTypeSetup, nil, controllerContext, operatorClient, kubeClient, kubeInformersForNamespaces, jobs.AllConditions) - jobs.RunTNFJobController(ctx, tools.JobTypeFencing, nil, controllerContext, operatorClient, kubeClient, kubeInformersForNamespaces, jobs.DefaultConditions) - - // wait until the after-setup jobs finished, - // in order to avoid races with update jobs - return waitForTnfAfterSetupJobsCompletion(ctx, kubeClient, nodeList) -} - -func waitForEtcdBootstrapCompleted(ctx context.Context, operatorClient v1helpers.StaticPodOperatorClient) error { - isEtcdRunningInCluster, err := ceohelpers.IsEtcdRunningInCluster(ctx, operatorClient) - if err != nil { - return fmt.Errorf("failed to check if bootstrap is completed: %w", err) - } - if !isEtcdRunningInCluster { - klog.Infof("waiting for bootstrap to complete with etcd running in cluster") - clientConfig, err := rest.InClusterConfig() - if err != nil { - return fmt.Errorf("failed to get in-cluster config: %w", err) - } - err = bootstrapteardown.WaitForEtcdBootstrap(ctx, clientConfig) - if err != nil { - return fmt.Errorf("failed to wait for bootstrap to complete: %w", err) - } - } - return nil -} - -func updateSetup( - nodeList []*corev1.Node, - ctx context.Context, - controllerContext *controllercmd.ControllerContext, - operatorClient v1helpers.StaticPodOperatorClient, - kubeClient kubernetes.Interface, - kubeInformersForNamespaces v1helpers.KubeInformersForNamespaces, -) error { - - klog.Info("TNF was already setup, checking for needed changes for modified nodes") - - // re-run auth job on both nodes - for _, node := range nodeList { - klog.Infof("(Re-)running auth job on node %s", node.GetName()) - err := jobs.RestartJobOrRunController(ctx, tools.JobTypeAuth, &node.Name, controllerContext, operatorClient, kubeClient, kubeInformersForNamespaces, jobs.DefaultConditions, tools.AuthJobCompletedTimeout) - if err != nil { - return fmt.Errorf("failed to (re-)start auth job on node %s: %w", node.GetName(), err) - } - } - // wait for completion - for _, node := range nodeList { - err := jobs.WaitForCompletion(ctx, kubeClient, tools.JobTypeAuth.GetJobName(&node.Name), operatorclient.TargetNamespace, tools.AuthJobCompletedTimeout) - if err != nil { - return fmt.Errorf("failed to wait for auth job on node %s to complete: %w", node.Name, err) - } - } - - // run update-setup job on both nodes, it will detect on which node it needs to do something - for _, node := range nodeList { - klog.Infof("(Re-)running update setup job on node %s", node.GetName()) - err := jobs.RestartJobOrRunController(ctx, tools.JobTypeUpdateSetup, &node.Name, controllerContext, operatorClient, kubeClient, kubeInformersForNamespaces, jobs.DefaultConditions, tools.SetupJobCompletedTimeout) - if err != nil { - return fmt.Errorf("failed to (re-)start update setup job on node %s: %w", node.GetName(), err) - } - - } - // wait for completion - for _, node := range nodeList { - err := jobs.WaitForCompletion(ctx, kubeClient, tools.JobTypeUpdateSetup.GetJobName(&node.Name), operatorclient.TargetNamespace, tools.SetupJobCompletedTimeout) - if err != nil { - return fmt.Errorf("failed to wait for update setup job on node %s to complete: %w", node.GetName(), err) - } - } - - // run after-setup job on both nodes - for _, node := range nodeList { - klog.Infof("(Re-)running after setup job on node %s", node.GetName()) - err := jobs.RestartJobOrRunController(ctx, tools.JobTypeAfterSetup, &node.Name, controllerContext, operatorClient, kubeClient, kubeInformersForNamespaces, jobs.DefaultConditions, tools.AfterSetupJobCompletedTimeout) - if err != nil { - return fmt.Errorf("failed to (re-)start after setup job on node %s: %w", node.GetName(), err) - } - - } - // wait for completion - for _, node := range nodeList { - err := jobs.WaitForCompletion(ctx, kubeClient, tools.JobTypeAfterSetup.GetJobName(&node.Name), operatorclient.TargetNamespace, tools.AfterSetupJobCompletedTimeout) - if err != nil { - return fmt.Errorf("failed to wait for after setup job on node %s to complete: %w", node.GetName(), err) - } - } - - return nil -} - -// tnfSetupJobsExist checks if TNF was already set up by checking for any existing TNF jobs -func tnfSetupJobsExist(ctx context.Context, kubeClient kubernetes.Interface) (bool, error) { - // Check if any TNF jobs exist in the target namespace - jobsClient := kubeClient.BatchV1().Jobs(operatorclient.TargetNamespace) - list, err := jobsClient.List(ctx, v1.ListOptions{ - LabelSelector: "app.kubernetes.io/component=two-node-fencing-setup", - }) - if err != nil { - return false, err - } - return len(list.Items) > 0, nil -} - -func waitForTnfAfterSetupJobsCompletion(ctx context.Context, kubeClient kubernetes.Interface, nodeList []*corev1.Node) error { - for _, node := range nodeList { - jobName := tools.JobTypeAfterSetup.GetJobName(&node.Name) - klog.Infof("Waiting for after-setup job %s to complete", jobName) - if err := jobs.WaitForCompletion(ctx, kubeClient, jobName, operatorclient.TargetNamespace, tools.AllCompletedTimeout); err != nil { - return fmt.Errorf("failed to wait for after-setup job %s to complete: %w", jobName, err) - } - } - return nil -} diff --git a/pkg/tnf/operator/nodehandler_test.go b/pkg/tnf/operator/nodehandler_test.go deleted file mode 100644 index 0e510cd7ce..0000000000 --- a/pkg/tnf/operator/nodehandler_test.go +++ /dev/null @@ -1,343 +0,0 @@ -package operator - -import ( - "context" - "errors" - "testing" - "time" - - "github.com/stretchr/testify/require" - batchv1 "k8s.io/api/batch/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/informers" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/kubernetes/fake" - "k8s.io/client-go/tools/cache" - - operatorv1 "github.com/openshift/api/operator/v1" - operatorversionedclientfake "github.com/openshift/client-go/operator/clientset/versioned/fake" - extinfops "github.com/openshift/client-go/operator/informers/externalversions" - operatorv1informers "github.com/openshift/client-go/operator/informers/externalversions/operator/v1" - "github.com/openshift/library-go/pkg/controller/controllercmd" - "github.com/openshift/library-go/pkg/operator/events" - "github.com/openshift/library-go/pkg/operator/v1helpers" - corev1listers "k8s.io/client-go/listers/core/v1" - "k8s.io/utils/clock" - - "github.com/openshift/cluster-etcd-operator/pkg/operator/ceohelpers" - "github.com/openshift/cluster-etcd-operator/pkg/operator/operatorclient" - u "github.com/openshift/cluster-etcd-operator/pkg/testutils" -) - -func TestHandleNodes(t *testing.T) { - tests := []struct { - name string - nodes []*corev1.Node - existingJobs []runtime.Object - mockStartControllers func() error - mockUpdateSetup func() error - expectError bool - expectStartControllers bool - expectUpdateSetup bool - errorContains string - }{ - { - name: "Less than 2 nodes - returns nil without action", - nodes: []*corev1.Node{ - createReadyNode("master-0"), - }, - existingJobs: []runtime.Object{}, - expectError: false, - expectStartControllers: false, - expectUpdateSetup: false, - }, - { - name: "More than 2 nodes - returns nil without action", - nodes: []*corev1.Node{ - createReadyNode("master-0"), - createReadyNode("master-1"), - createReadyNode("master-2"), - }, - existingJobs: []runtime.Object{}, - expectError: false, - expectStartControllers: false, - expectUpdateSetup: false, - }, - { - name: "2 nodes but first not ready - returns nil without action", - nodes: []*corev1.Node{ - createNotReadyNode("master-0"), - createReadyNode("master-1"), - }, - existingJobs: []runtime.Object{}, - expectError: false, - expectStartControllers: false, - expectUpdateSetup: false, - }, - { - name: "2 nodes but second not ready - returns nil without action", - nodes: []*corev1.Node{ - createReadyNode("master-0"), - createNotReadyNode("master-1"), - }, - existingJobs: []runtime.Object{}, - expectError: false, - expectStartControllers: false, - expectUpdateSetup: false, - }, - { - name: "2 ready nodes, no existing jobs - starts controllers only", - nodes: []*corev1.Node{ - createReadyNode("master-0"), - createReadyNode("master-1"), - }, - existingJobs: []runtime.Object{}, - mockStartControllers: func() error { - return nil - }, - expectError: false, - expectStartControllers: true, - expectUpdateSetup: false, - }, - { - name: "2 ready nodes, existing jobs - starts controllers and updates setup", - nodes: []*corev1.Node{ - createReadyNode("master-0"), - createReadyNode("master-1"), - }, - existingJobs: []runtime.Object{ - createTNFJob("tnf-setup"), - }, - mockStartControllers: func() error { - return nil - }, - mockUpdateSetup: func() error { - return nil - }, - expectError: false, - expectStartControllers: true, - expectUpdateSetup: true, - }, - { - name: "2 ready nodes - error starting controllers", - nodes: []*corev1.Node{ - createReadyNode("master-0"), - createReadyNode("master-1"), - }, - existingJobs: []runtime.Object{}, - mockStartControllers: func() error { - return errors.New("failed to start controllers") - }, - expectError: true, - expectStartControllers: true, - expectUpdateSetup: false, - errorContains: "failed to start TNF job controllers", - }, - { - name: "2 ready nodes, existing jobs - error updating setup", - nodes: []*corev1.Node{ - createReadyNode("master-0"), - createReadyNode("master-1"), - }, - existingJobs: []runtime.Object{ - createTNFJob("tnf-setup"), - }, - mockStartControllers: func() error { - return nil - }, - mockUpdateSetup: func() error { - return errors.New("failed to update") - }, - expectError: true, - expectStartControllers: true, - expectUpdateSetup: true, - errorContains: "failed to update pacemaker setup", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Setup - ctx := context.Background() - - // Create fake kubernetes client with jobs - fakeKubeClient := fake.NewSimpleClientset(tt.existingJobs...) - - // Create fake operator client - fakeOperatorClient := v1helpers.NewFakeStaticPodOperatorClient( - &operatorv1.StaticPodOperatorSpec{}, - u.StaticPodOperatorStatus( - u.WithLatestRevision(1), - u.WithNodeStatusAtCurrentRevision(1), - u.WithNodeStatusAtCurrentRevision(1), - ), - nil, - nil, - ) - - // Create controller context - eventRecorder := events.NewRecorder( - fakeKubeClient.CoreV1().Events(operatorclient.TargetNamespace), - "test-nodehandler", - &corev1.ObjectReference{}, - clock.RealClock{}, - ) - controllerContext := &controllercmd.ControllerContext{ - EventRecorder: eventRecorder, - } - - // Create node informer and lister - nodeInformer := informers.NewSharedInformerFactory(fakeKubeClient, 0).Core().V1().Nodes() - for _, node := range tt.nodes { - err := nodeInformer.Informer().GetIndexer().Add(node) - require.NoError(t, err) - } - controlPlaneNodeLister := corev1listers.NewNodeLister(nodeInformer.Informer().GetIndexer()) - - // Create etcd informer - operatorClientFake := operatorversionedclientfake.NewClientset() - etcdInformers := extinfops.NewSharedInformerFactory(operatorClientFake, 10*time.Minute) - etcdIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) - require.NoError(t, etcdIndexer.Add(&operatorv1.Etcd{ - ObjectMeta: metav1.ObjectMeta{ - Name: ceohelpers.InfrastructureClusterName, - }, - })) - etcdInformers.Operator().V1().Etcds().Informer().AddIndexers(etcdIndexer.GetIndexers()) - ctx2 := t.Context() - etcdInformers.Start(ctx2.Done()) - synced := etcdInformers.WaitForCacheSync(ctx2.Done()) - for v, ok := range synced { - require.True(t, ok, "cache failed to sync: %v", v) - } - - // Create kubeInformersForNamespaces - kubeInformersForNamespaces := v1helpers.NewKubeInformersForNamespaces( - fakeKubeClient, - "", - operatorclient.GlobalUserSpecifiedConfigNamespace, - operatorclient.GlobalMachineSpecifiedConfigNamespace, - operatorclient.TargetNamespace, - operatorclient.OperatorNamespace, - "kube-system", - ) - - // Track function calls - startControllersCalled := false - updateSetupCalled := false - - // Mock startTnfJobcontrollers - originalStartFunc := startTnfJobcontrollersFunc - if tt.mockStartControllers != nil { - startTnfJobcontrollersFunc = func( - nodeList []*corev1.Node, - ctx context.Context, - controllerContext *controllercmd.ControllerContext, - operatorClient v1helpers.StaticPodOperatorClient, - kubeClient kubernetes.Interface, - kubeInformersForNamespaces v1helpers.KubeInformersForNamespaces, - etcdInformer operatorv1informers.EtcdInformer, - ) error { - startControllersCalled = true - return tt.mockStartControllers() - } - } - defer func() { startTnfJobcontrollersFunc = originalStartFunc }() - - // Mock updateSetup - originalUpdateFunc := updateSetupFunc - if tt.mockUpdateSetup != nil { - updateSetupFunc = func( - nodeList []*corev1.Node, - ctx context.Context, - controllerContext *controllercmd.ControllerContext, - operatorClient v1helpers.StaticPodOperatorClient, - kubeClient kubernetes.Interface, - kubeInformersForNamespaces v1helpers.KubeInformersForNamespaces, - ) error { - updateSetupCalled = true - return tt.mockUpdateSetup() - } - } - defer func() { updateSetupFunc = originalUpdateFunc }() - - // Execute - err := handleNodes( - controllerContext, - controlPlaneNodeLister, - ctx, - fakeOperatorClient, - fakeKubeClient, - kubeInformersForNamespaces, - etcdInformers.Operator().V1().Etcds(), - ) - - // Verify - if tt.expectError { - require.Error(t, err, "Expected error but got none") - if tt.errorContains != "" { - require.Contains(t, err.Error(), tt.errorContains, - "Expected error to contain %q but got: %v", tt.errorContains, err) - } - } else { - require.NoError(t, err, "Expected no error but got: %v", err) - } - - require.Equal(t, tt.expectStartControllers, startControllersCalled, - "Expected startTnfJobcontrollers called=%v, but got=%v", - tt.expectStartControllers, startControllersCalled) - - require.Equal(t, tt.expectUpdateSetup, updateSetupCalled, - "Expected updateSetup called=%v, but got=%v", - tt.expectUpdateSetup, updateSetupCalled) - }) - } -} - -// Helper functions - -func createReadyNode(name string) *corev1.Node { - return &corev1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - }, - Status: corev1.NodeStatus{ - Conditions: []corev1.NodeCondition{ - { - Type: corev1.NodeReady, - Status: corev1.ConditionTrue, - }, - }, - }, - } -} - -func createNotReadyNode(name string) *corev1.Node { - return &corev1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - }, - Status: corev1.NodeStatus{ - Conditions: []corev1.NodeCondition{ - { - Type: corev1.NodeReady, - Status: corev1.ConditionFalse, - }, - }, - }, - } -} - -func createTNFJob(name string) *batchv1.Job { - return &batchv1.Job{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: operatorclient.TargetNamespace, - Labels: map[string]string{ - "app.kubernetes.io/component": "two-node-fencing-setup", - }, - }, - } -} diff --git a/pkg/tnf/pkg/pacemaker/client_helpers.go b/pkg/tnf/pkg/pacemaker/client_helpers.go index 0e99eb1718..b0b272a92a 100644 --- a/pkg/tnf/pkg/pacemaker/client_helpers.go +++ b/pkg/tnf/pkg/pacemaker/client_helpers.go @@ -22,10 +22,10 @@ func getKubeConfig() (*rest.Config, error) { return config, nil } -// createPacemakerRESTClient creates a REST client configured for PacemakerStatus CRs. +// CreatePacemakerRESTClient creates a REST client configured for PacemakerStatus CRs. // It takes a base Kubernetes REST config and configures it with the necessary // scheme, group version, and serializer for the PacemakerStatus CRD. -func createPacemakerRESTClient(baseConfig *rest.Config) (rest.Interface, error) { +func CreatePacemakerRESTClient(baseConfig *rest.Config) (rest.Interface, error) { if baseConfig == nil { return nil, fmt.Errorf("baseConfig cannot be nil") } diff --git a/pkg/tnf/pkg/pacemaker/healthcheck.go b/pkg/tnf/pkg/pacemaker/healthcheck.go index 684f9ee540..a223dd8984 100644 --- a/pkg/tnf/pkg/pacemaker/healthcheck.go +++ b/pkg/tnf/pkg/pacemaker/healthcheck.go @@ -1,22 +1,11 @@ package pacemaker import ( - "context" "fmt" "strings" - "sync" "time" - operatorv1 "github.com/openshift/api/operator/v1" - "github.com/openshift/library-go/pkg/controller/factory" - "github.com/openshift/library-go/pkg/operator/events" - "github.com/openshift/library-go/pkg/operator/v1helpers" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/watch" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/cache" "k8s.io/klog/v2" pacmkrv1 "github.com/openshift/api/etcd/v1" @@ -25,46 +14,26 @@ import ( // HealthStatusValue represents the overall health status of the pacemaker cluster. type HealthStatusValue string -// Local constants for healthcheck controller +// Health status constants const ( // Status values for health assessment - statusHealthy HealthStatusValue = "Healthy" - statusWarning HealthStatusValue = "Warning" - statusError HealthStatusValue = "Error" - statusUnknown HealthStatusValue = "Unknown" - - // Degraded condition reason - reasonPacemakerUnhealthy = "PacemakerUnhealthy" + StatusHealthy HealthStatusValue = "Healthy" + StatusWarning HealthStatusValue = "Warning" + StatusError HealthStatusValue = "Error" + StatusUnknown HealthStatusValue = "Unknown" // Warning message prefixes for categorizing events. - // Note: warningPrefixFailedAction was removed - failed action events are recorded - // by the status collector directly from pacemaker XML, not the healthcheck controller. warningPrefixFencingEvent = "Recent fencing event:" - // Operator condition types - conditionTypePacemakerDegraded = "PacemakerHealthCheckDegraded" - - // Event key prefixes - eventKeyPrefixWarning = "warning:" - eventKeyPrefixError = "error:" - // Error messages - msgNoNodesFound = "No nodes found in cluster" - msgNodeUnhealthy = "%s node is unhealthy: %s" - msgNodeOffline = "Node %s is offline" - msgInsufficientNodes = "Insufficient nodes in cluster (expected %d, found %d)" - msgExcessiveNodes = "Excessive nodes in cluster (expected %d, found %d)" - msgClusterInMaintenance = "Cluster is in maintenance mode" - msgClusterUnhealthy = "Cluster is unhealthy: %s" - msgPacemakerDegraded = "Pacemaker cluster is in degraded state" - msgPacemakerHealthy = "Pacemaker cluster is healthy - all nodes have healthy critical resources" - msgPacemakerWarningsCleared = "Pacemaker cluster warnings cleared" - msgFencingRedundancyLost = "fencing at risk (agent running but not managed for recovery)" - - // Event message templates - msgDetectedFencing = "Pacemaker detected a recent fencing event: %s" - msgPacemakerWarning = "Pacemaker warning: %s" - msgPacemakerError = "Pacemaker error: %s" + msgNoNodesFound = "No nodes found in cluster" + msgNodeUnhealthy = "%s node is unhealthy: %s" + msgNodeOffline = "Node %s is offline" + msgInsufficientNodes = "Insufficient nodes in cluster (expected %d, found %d)" + msgExcessiveNodes = "Excessive nodes in cluster (expected %d, found %d)" + msgClusterInMaintenance = "Cluster is in maintenance mode" + msgClusterUnhealthy = "Cluster is unhealthy: %s" + msgFencingRedundancyLost = "fencing at risk (agent running but not managed for recovery)" ) // errorPatternsByPriority maps error message substrings to event reasons. @@ -111,259 +80,37 @@ type HealthStatus struct { CRLastUpdated time.Time } -// HealthCheck monitors pacemaker status in ExternalEtcd topology clusters -type HealthCheck struct { - operatorClient v1helpers.StaticPodOperatorClient - kubeClient kubernetes.Interface - eventRecorder events.Recorder - pacemakerInformer cache.SharedIndexInformer - - // Event deduplication: tracks recently recorded events to avoid duplicates - recordedEventsMu sync.Mutex - recordedEvents map[string]time.Time - - // Previous health status for transition detection and grace period calculation. - // Only updated when status is non-Unknown, so CRLastUpdated reflects the last valid CR timestamp. - previousMu sync.Mutex - previous *HealthStatus -} - -// NewHealthCheck creates a new HealthCheck for monitoring pacemaker status -// in clusters that use ExternalEtcd clusters. -// Returns the controller and the PacemakerCluster informer (which must be started separately). -func NewHealthCheck( - operatorClient v1helpers.StaticPodOperatorClient, - kubeClient kubernetes.Interface, - eventRecorder events.Recorder, - restConfig *rest.Config, -) (factory.Controller, cache.SharedIndexInformer, error) { - // Create REST client for PacemakerStatus CRs - restClient, err := createPacemakerRESTClient(restConfig) - if err != nil { - return nil, nil, fmt.Errorf("failed to create REST client: %w", err) - } - - // Create scheme for the parameter codec - scheme := runtime.NewScheme() - if err := pacmkrv1.AddToScheme(scheme); err != nil { - return nil, nil, fmt.Errorf("failed to add scheme for informer: %w", err) - } - - // Create informer for PacemakerCluster - klog.Infof("Creating PacemakerCluster informer for group %s, resource %s", pacmkrv1.SchemeGroupVersion.String(), PacemakerResourceName) - informer := cache.NewSharedIndexInformer( - &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { - klog.V(4).Infof("PacemakerCluster informer ListFunc called for resource %s", PacemakerResourceName) - result := &pacmkrv1.PacemakerClusterList{} - err := restClient.Get(). - Resource(PacemakerResourceName). - VersionedParams(&options, runtime.NewParameterCodec(scheme)). - Do(context.Background()). - Into(result) - if err != nil { - klog.Errorf("Failed to list PacemakerCluster resources (%s): %v", PacemakerResourceName, err) - } else { - klog.V(4).Infof("Successfully listed PacemakerCluster resources, found %d items", len(result.Items)) - } - return result, err - }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { - klog.V(4).Infof("PacemakerCluster informer WatchFunc called for resource %s", PacemakerResourceName) - watcher, err := restClient.Get(). - Resource(PacemakerResourceName). - VersionedParams(&options, runtime.NewParameterCodec(scheme)). - Watch(context.Background()) - if err != nil { - klog.Errorf("Failed to watch PacemakerCluster resources (%s): %v", PacemakerResourceName, err) - } - return watcher, err - }, - }, - &pacmkrv1.PacemakerCluster{}, - HealthCheckResyncInterval, - cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, - ) - - c := &HealthCheck{ - operatorClient: operatorClient, - kubeClient: kubeClient, - eventRecorder: eventRecorder, - pacemakerInformer: informer, - recordedEvents: make(map[string]time.Time), - // previous starts as nil - first sync will be treated as "from Unknown" - } - - syncCtx := factory.NewSyncContext("PacemakerHealthCheck", eventRecorder.WithComponentSuffix("pacemaker-health-check")) - - klog.Infof("PacemakerHealthCheck controller created, waiting for informers to sync before starting") - klog.Infof("PacemakerHealthCheck will watch: operatorClient and %s/%s resource", pacmkrv1.SchemeGroupVersion.String(), PacemakerResourceName) - - // ResyncEvery ensures the sync function is called at regular intervals (30s) - // even if no informer events are detected. - controller := factory.New(). - WithSyncContext(syncCtx). - ResyncEvery(HealthCheckResyncInterval). - WithSync(c.sync). - WithInformers( - operatorClient.Informer(), - informer, - ).ToController("PacemakerHealthCheck", syncCtx.Recorder()) - - klog.Infof("PacemakerHealthCheck controller successfully created and ready to start") - klog.Infof("PacemakerHealthCheck informers to sync: 1) operatorClient.Informer(), 2) PacemakerCluster informer") - klog.Infof("PacemakerCluster informer must be started separately before controller.Run() is called") - klog.Infof("factory.Controller will wait up to 10 minutes for informers to sync, then exit if they fail to sync") - return controller, informer, nil -} - -// sync is the main sync function that gets called periodically to check pacemaker status -func (c *HealthCheck) sync(ctx context.Context, syncCtx factory.SyncContext) error { - klog.V(4).Infof("PacemakerHealthCheck sync started") - defer klog.V(4).Infof("PacemakerHealthCheck sync completed") - - // Get current and previous health status for transition detection - currentStatus, previousStatus, err := c.getPacemakerStatus(ctx) - if err != nil { - klog.Errorf("Failed to get pacemaker status: %v", err) - return err - } - - // nil status means the CR hasn't changed since last sync (same lastUpdated). - // Skip processing to avoid redundant operator status updates and event recording. - if currentStatus == nil { - return nil - } - - // Log the determined status for visibility - klog.V(2).Infof("Pacemaker health status: %s (errors: %d, warnings: %d)", - currentStatus.OverallStatus, len(currentStatus.Errors), len(currentStatus.Warnings)) - - if err := c.updateOperatorStatus(ctx, currentStatus, previousStatus); err != nil { - return err - } - - c.recordHealthCheckEvents(currentStatus, previousStatus) - - return nil -} - -// getPacemakerStatus retrieves pacemaker status from the PacemakerCluster CR. -// Returns (currentStatus, previousStatus, nil) on success. -// Returns (nil, nil, nil) if the CR hasn't changed since last sync (same lastUpdated timestamp). -// For Unknown status, previous is not updated (preserves last valid status for grace period). -func (c *HealthCheck) getPacemakerStatus(ctx context.Context) (*HealthStatus, *HealthStatus, error) { - klog.V(4).Infof("Retrieving pacemaker status from CR...") - - // Read previous status - c.previousMu.Lock() - previous := c.previous - c.previousMu.Unlock() - - // Get the PacemakerCluster CR from the informer cache - item, exists, err := c.pacemakerInformer.GetStore().GetByKey(PacemakerClusterResourceName) - if err != nil { - // Unknown status - don't update previous (preserves last valid for grace period) - return &HealthStatus{ - OverallStatus: statusUnknown, - Warnings: []string{}, - Errors: []string{fmt.Sprintf("Failed to get PacemakerCluster CR from cache: %v", err)}, - }, previous, nil - } - if !exists { - // Unknown status - CR not found in cache - return &HealthStatus{ - OverallStatus: statusUnknown, - Warnings: []string{}, - Errors: []string{"PacemakerCluster CR not found in cache"}, - }, previous, nil - } - - pacemakerCR, ok := item.(*pacmkrv1.PacemakerCluster) - if !ok { - return &HealthStatus{ - OverallStatus: statusUnknown, - Warnings: []string{}, - Errors: []string{"Failed to convert cached item to PacemakerCluster"}, - }, previous, nil - } - - // Check if status is populated (LastUpdated is zero means status was never set) - if pacemakerCR.Status.LastUpdated.IsZero() { - // Unknown status - don't update previous - return &HealthStatus{ - OverallStatus: statusUnknown, - Warnings: []string{}, - Errors: []string{"PacemakerCluster CR has no status populated"}, - }, previous, nil - } - - crLastUpdated := pacemakerCR.Status.LastUpdated.Time - - // Check staleness first - any update (even with errors) clears staleness. - timeSinceUpdate := time.Since(crLastUpdated) - if timeSinceUpdate > StatusStalenessThreshold { - // Unknown status - don't update previous - // Use absolute timestamp (stable) for event deduplication. - return &HealthStatus{ - OverallStatus: statusUnknown, - Warnings: []string{}, - Errors: []string{fmt.Sprintf("Pacemaker status is stale (last updated: %s)", crLastUpdated.Format(time.RFC3339))}, - }, previous, nil - } - - // Check if lastUpdated timestamp has changed since last sync. - // If unchanged, skip processing to avoid redundant work on timer-triggered syncs. - // Use previous.CRLastUpdated for comparison (only set for non-Unknown status). - if previous != nil && !previous.CRLastUpdated.IsZero() && crLastUpdated.Equal(previous.CRLastUpdated) { - klog.V(4).Infof("Skipping sync: lastUpdated timestamp unchanged (%v)", crLastUpdated) - return nil, nil, nil - } - - // Build health status from the CRD status fields - currentStatus := c.buildHealthStatusFromCR(pacemakerCR) - currentStatus.CRLastUpdated = crLastUpdated - - // Only update previous for non-Unknown status (preserves last valid for grace period) - if currentStatus.OverallStatus != statusUnknown { - c.previousMu.Lock() - c.previous = currentStatus - c.previousMu.Unlock() - } - - return currentStatus, previous, nil -} - -// buildHealthStatusFromCR builds a HealthStatus from the PacemakerCluster CR status fields -// Note: This function assumes Status is not nil (checked by caller in getPacemakerStatus) -func (c *HealthCheck) buildHealthStatusFromCR(pacemakerStatus *pacmkrv1.PacemakerCluster) *HealthStatus { +// BuildHealthStatusFromCR builds a HealthStatus from the PacemakerCluster CR status fields. +// Returns nil if the CR has no status populated (indicates CR was never updated by status collector). +func BuildHealthStatusFromCR(pacemakerStatus *pacmkrv1.PacemakerCluster) *HealthStatus { status := &HealthStatus{ - OverallStatus: statusUnknown, + OverallStatus: StatusUnknown, Warnings: []string{}, Errors: []string{}, } - // Defensive check: this should not happen as getPacemakerStatus checks for unpopulated Status + // Defensive check: this should not happen as caller should check for unpopulated Status if pacemakerStatus == nil || pacemakerStatus.Status.LastUpdated.IsZero() { - klog.Errorf("buildHealthStatusFromCR called with nil PacemakerCluster or unpopulated Status") - status.Errors = append(status.Errors, "Internal error: unpopulated Status in buildHealthStatusFromCR") + status.Errors = append(status.Errors, "Internal error: unpopulated Status in BuildHealthStatusFromCR") return status } + status.CRLastUpdated = pacemakerStatus.Status.LastUpdated.Time + // Check cluster-level configuration issues FIRST (node count, maintenance mode) // These are often root causes (e.g., "excessive nodes" causes resource failures on extra node) - c.checkClusterConditions(pacemakerStatus, status) + checkClusterConditions(pacemakerStatus, status) // Check node statuses (node-level conditions and resource health) - c.checkNodeStatuses(pacemakerStatus, status) + checkNodeStatuses(pacemakerStatus, status) // Determine overall status: errors > warnings > healthy if len(status.Errors) > 0 { - status.OverallStatus = statusError + status.OverallStatus = StatusError } else if len(status.Warnings) > 0 { - status.OverallStatus = statusWarning + status.OverallStatus = StatusWarning } else { - status.OverallStatus = statusHealthy + status.OverallStatus = StatusHealthy } return status @@ -372,17 +119,16 @@ func (c *HealthCheck) buildHealthStatusFromCR(pacemakerStatus *pacmkrv1.Pacemake // checkClusterConditions checks cluster-level configuration issues (node count, maintenance mode). // These are ALWAYS reported regardless of node-level errors, as they often represent root causes. // For example, "excessive nodes" causes resource failures on the extra node. -func (c *HealthCheck) checkClusterConditions(pacemakerStatus *pacmkrv1.PacemakerCluster, status *HealthStatus) { +func checkClusterConditions(pacemakerStatus *pacmkrv1.PacemakerCluster, status *HealthStatus) { conditions := pacemakerStatus.Status.Conditions if len(conditions) == 0 { // Missing cluster conditions means we can't verify cluster health configuration - klog.V(2).Infof("No cluster conditions present in status") status.Errors = append(status.Errors, "No cluster conditions available") return } // Always check cluster-level configuration issues - these are root causes - clusterIssues := c.getClusterConditionIssues(conditions, pacemakerStatus) + clusterIssues := getClusterConditionIssues(conditions, pacemakerStatus) // Add cluster-level error if there are configuration issues if len(clusterIssues) > 0 { @@ -391,7 +137,7 @@ func (c *HealthCheck) checkClusterConditions(pacemakerStatus *pacmkrv1.Pacemaker } // getClusterConditionIssues returns specific issues from cluster-level conditions (non-summary conditions) -func (c *HealthCheck) getClusterConditionIssues(conditions []metav1.Condition, pacemakerStatus *pacmkrv1.PacemakerCluster) []string { +func getClusterConditionIssues(conditions []metav1.Condition, pacemakerStatus *pacmkrv1.PacemakerCluster) []string { var issues []string // Check NodeCountAsExpected condition @@ -419,10 +165,9 @@ func (c *HealthCheck) getClusterConditionIssues(conditions []metav1.Condition, p } // checkNodeStatuses checks if all nodes have healthy conditions and resources -func (c *HealthCheck) checkNodeStatuses(pacemakerStatus *pacmkrv1.PacemakerCluster, status *HealthStatus) { +func checkNodeStatuses(pacemakerStatus *pacmkrv1.PacemakerCluster, status *HealthStatus) { // Nil-guard for Nodes field - missing node data is an error (cannot verify cluster health) if pacemakerStatus.Status.Nodes == nil { - klog.V(2).Infof("Pacemaker.Status.Nodes is nil, cannot determine node status") status.Errors = append(status.Errors, msgNoNodesFound) return } @@ -431,24 +176,23 @@ func (c *HealthCheck) checkNodeStatuses(pacemakerStatus *pacmkrv1.PacemakerClust // Empty nodes list is also an error - cannot verify cluster health without node data if len(nodes) == 0 { - klog.V(2).Infof("Pacemaker has no node information") status.Errors = append(status.Errors, msgNoNodesFound) return } // Check each node's conditions and resource health (consolidated into single error per node) for _, node := range nodes { - c.checkNodeConditions(node, status) + checkNodeConditions(node, status) } } // checkNodeConditions checks the conditions of a single node and its resources, // routing issues to errors or warnings based on severity. // Errors degrade the operator; warnings are informational (e.g., fencing redundancy lost). -func (c *HealthCheck) checkNodeConditions(node pacmkrv1.PacemakerClusterNodeStatus, status *HealthStatus) { +func checkNodeConditions(node pacmkrv1.PacemakerClusterNodeStatus, status *HealthStatus) { conditions := node.Conditions if len(conditions) == 0 { - klog.V(2).Infof("Node %s has no conditions", node.NodeName) + status.Warnings = append(status.Warnings, fmt.Sprintf("Node %s: missing condition metadata", node.NodeName)) return } @@ -462,22 +206,26 @@ func (c *HealthCheck) checkNodeConditions(node pacmkrv1.PacemakerClusterNodeStat // Always check for fencing warnings - these are independent of overall node health. // Fencing redundancy degraded (FencingHealthy=False but FencingAvailable=True) is a warning // that should be reported even when the node is otherwise healthy. - fencingWarnings := c.getFencingWarnings(conditions) + fencingWarnings := getFencingWarnings(conditions) for _, warning := range fencingWarnings { status.Warnings = append(status.Warnings, fmt.Sprintf("%s: %s", node.NodeName, warning)) } // Check overall node Healthy condition healthyCondition := FindCondition(conditions, pacmkrv1.NodeHealthyConditionType) - if healthyCondition == nil || healthyCondition.Status == metav1.ConditionTrue { + if healthyCondition == nil { + status.Warnings = append(status.Warnings, fmt.Sprintf("Node %s: missing Healthy condition", node.NodeName)) + return + } + if healthyCondition.Status == metav1.ConditionTrue { return // Node is healthy (except for warnings already captured above) } // Node is unhealthy - collect errors (warnings already captured above) - nodeErrors := c.getNodeConditionErrors(conditions) + nodeErrors := getNodeConditionErrors(conditions) // Collect resource errors (all resource issues are currently errors) - resourceErrors := c.getNodeResourceSummaries(node) + resourceErrors := getNodeResourceSummaries(node) // Combine all errors allErrors := append(nodeErrors, resourceErrors...) @@ -497,7 +245,7 @@ func (c *HealthCheck) checkNodeConditions(node pacmkrv1.PacemakerClusterNodeStat // getFencingWarnings returns warnings about degraded fencing redundancy. // This is checked independently of overall node health because fencing redundancy // degradation should be reported even when the node is otherwise healthy. -func (c *HealthCheck) getFencingWarnings(conditions []metav1.Condition) []string { +func getFencingWarnings(conditions []metav1.Condition) []string { var warnings []string // Check FencingHealthy - if false but FencingAvailable is true, fencing redundancy is degraded (warning) @@ -538,7 +286,7 @@ var nodeConditionChecks = []nodeConditionCheck{ // getNodeConditionErrors returns errors from node-level conditions. // Errors require immediate action and cause the operator to degrade. -func (c *HealthCheck) getNodeConditionErrors(conditions []metav1.Condition) []string { +func getNodeConditionErrors(conditions []metav1.Condition) []string { var errors []string for _, check := range nodeConditionChecks { @@ -547,7 +295,7 @@ func (c *HealthCheck) getNodeConditionErrors(conditions []metav1.Condition) []st } } - // Ready/Pending is logged but not reported - it's a temporary transitional state + // Ready/Pending is logged but not reported as an error - it's a temporary transitional state if cond := FindCondition(conditions, pacmkrv1.NodeReadyConditionType); cond != nil && cond.Status != metav1.ConditionTrue { klog.V(2).Infof("Node is pending (temporary state)") } @@ -557,14 +305,14 @@ func (c *HealthCheck) getNodeConditionErrors(conditions []metav1.Condition) []st // getNodeResourceSummaries returns summaries of unhealthy resources on a node. // Each summary includes the resource name and its specific issue. -func (c *HealthCheck) getNodeResourceSummaries(node pacmkrv1.PacemakerClusterNodeStatus) []string { +func getNodeResourceSummaries(node pacmkrv1.PacemakerClusterNodeStatus) []string { var summaries []string for _, resource := range node.Resources { healthyCondition := FindCondition(resource.Conditions, pacmkrv1.ResourceHealthyConditionType) if healthyCondition != nil && healthyCondition.Status != metav1.ConditionTrue { // Get specific reason for this resource - reason := c.getResourceIssue(resource.Conditions) + reason := getResourceIssue(resource.Conditions) summaries = append(summaries, fmt.Sprintf("%s %s", resource.Name, reason)) } } @@ -573,13 +321,12 @@ func (c *HealthCheck) getNodeResourceSummaries(node pacmkrv1.PacemakerClusterNod } // getResourceIssue returns a specific issue description for an unhealthy resource. -// // All resource-level issues are treated as errors. The Active=True with Started=False state // (anomalous transitional state) could theoretically be a warning since it might self-resolve, // but routing resource issues to warnings vs errors would require significant refactoring. // Additionally: (1) this state is rare and brief and (2) etcd not running causes API server // degradation anyway, so treating it as an error is appropriate. -func (c *HealthCheck) getResourceIssue(conditions []metav1.Condition) string { +func getResourceIssue(conditions []metav1.Condition) string { // Check for specific failure conditions (prioritized by severity) operationalCondition := FindCondition(conditions, pacmkrv1.ResourceOperationalConditionType) if operationalCondition != nil && operationalCondition.Status != metav1.ConditionTrue { @@ -604,280 +351,9 @@ func (c *HealthCheck) getResourceIssue(conditions []metav1.Condition) string { return "is unhealthy" } -// updateOperatorStatus processes the HealthStatus and conditionally updates the operator state. -// previous is the last non-Unknown health status (used for grace period calculation). -func (c *HealthCheck) updateOperatorStatus(ctx context.Context, status *HealthStatus, previous *HealthStatus) error { - klog.V(4).Infof("Updating operator availability based on pacemaker status: %s", status.OverallStatus) - - // Log warnings and errors for visibility - for _, warning := range status.Warnings { - klog.Warningf(msgPacemakerWarning, warning) - } - for _, err := range status.Errors { - klog.Errorf(msgPacemakerError, err) - } - - // Update operator conditions based on pacemaker status - switch status.OverallStatus { - case statusError: - return c.setPacemakerDegradedCondition(ctx, status) - case statusHealthy, statusWarning: - // Both healthy and warning states should clear degraded condition - // Warnings are informational (e.g. recent fencing, node count mismatch) and don't indicate degradation - if status.OverallStatus == statusWarning { - klog.V(2).Infof("Pacemaker health check has warnings but cluster is operational: %v", status.Warnings) - } - return c.clearPacemakerDegradedCondition(ctx, status) - case statusUnknown: - // Unknown status means we cannot determine pacemaker health (CR not found, stale, no status, etc.) - // Only mark as degraded if we haven't received a valid status in a while (grace period). - // Use previous.CRLastUpdated which reflects when we last had valid cluster data. - // If previous is nil (first sync), don't degrade immediately. - if previous == nil || previous.CRLastUpdated.IsZero() { - klog.V(2).Infof("Pacemaker health check cannot determine status (no previous valid status), not marking degraded yet: %v", - status.Errors) - return nil - } - - timeSinceLastValid := time.Since(previous.CRLastUpdated) - if timeSinceLastValid > StatusUnknownDegradedThreshold { - klog.Warningf("Pacemaker health check cannot determine status for %v (threshold: %v), marking as degraded: %v", - timeSinceLastValid, StatusUnknownDegradedThreshold, status.Errors) - return c.setPacemakerDegradedCondition(ctx, status) - } - - // Still within grace period, just log - klog.V(2).Infof("Pacemaker health check cannot determine status for %v (threshold: %v), not marking degraded yet: %v", - timeSinceLastValid, StatusUnknownDegradedThreshold, status.Errors) - return nil - default: - // This should never happen, but log it if it does - klog.Errorf("Unexpected pacemaker health status: %s (errors: %v, warnings: %v)", - status.OverallStatus, status.Errors, status.Warnings) - return nil - } -} - -func (c *HealthCheck) setPacemakerDegradedCondition(ctx context.Context, status *HealthStatus) error { - message := strings.Join(status.Errors, "; ") - if message == "" { - message = msgPacemakerDegraded - } - - // Check if the condition is already set with the same message - currentCondition, err := c.getCurrentPacemakerCondition() - if err != nil { - return err - } - - // Only update if the condition is not already set to True with the same message - if currentCondition != nil && - currentCondition.Status == operatorv1.ConditionTrue && - currentCondition.Message == message { - klog.V(4).Infof("Pacemaker degraded condition already set with same message, skipping update") - return nil - } - - condition := operatorv1.OperatorCondition{ - Type: conditionTypePacemakerDegraded, - Status: operatorv1.ConditionTrue, - Reason: reasonPacemakerUnhealthy, - Message: message, - } - - return c.updateOperatorCondition(ctx, condition) -} - -func (c *HealthCheck) clearPacemakerDegradedCondition(ctx context.Context, status *HealthStatus) error { - // Check the current condition state - currentCondition, err := c.getCurrentPacemakerCondition() - if err != nil { - return err - } - - // Skip if condition is already explicitly set to False (no change needed) - if currentCondition != nil && currentCondition.Status == operatorv1.ConditionFalse { - klog.V(4).Infof("Pacemaker degraded condition is already False, skipping update") - return nil - } - - // Set condition to False in two cases: - // 1. Condition doesn't exist yet (first healthy check - make health status explicit) - // 2. Condition is currently True (transitioning from degraded to healthy) - condition := operatorv1.OperatorCondition{ - Type: conditionTypePacemakerDegraded, - Status: operatorv1.ConditionFalse, - } - - err = c.updateOperatorCondition(ctx, condition) - if err != nil { - return err - } - - if currentCondition == nil { - klog.Infof("Pacemaker health check condition initialized: PacemakerHealthCheckDegraded=False") - } else { - klog.Infof("Pacemaker degraded condition cleared (transitioned from True to False)") - } - return nil -} - -// getCurrentPacemakerCondition retrieves the current PacemakerHealthCheckDegraded condition -func (c *HealthCheck) getCurrentPacemakerCondition() (*operatorv1.OperatorCondition, error) { - _, currentStatus, _, err := c.operatorClient.GetStaticPodOperatorState() - if err != nil { - return nil, fmt.Errorf("failed to get current operator status: %w", err) - } - - // Find the current pacemaker degraded condition - for i := range currentStatus.Conditions { - if currentStatus.Conditions[i].Type == conditionTypePacemakerDegraded { - return ¤tStatus.Conditions[i], nil - } - } - - return nil, nil -} - -// updateOperatorCondition updates the operator condition -func (c *HealthCheck) updateOperatorCondition(ctx context.Context, condition operatorv1.OperatorCondition) error { - _, _, err := v1helpers.UpdateStaticPodStatus(ctx, c.operatorClient, v1helpers.UpdateStaticPodConditionFn(condition)) - if err != nil { - klog.Errorf("Failed to update operator status: %v", err) - return err - } - - klog.V(2).Infof("Updated operator condition: %s=%s", condition.Type, condition.Status) - return nil -} - -// cleanupExpiredEvents removes events from the deduplication map that have exceeded their window. -// Fencing events are kept for 24 hours, other events for 5 minutes. -func (c *HealthCheck) cleanupExpiredEvents() { - c.recordedEventsMu.Lock() - defer c.recordedEventsMu.Unlock() - - now := time.Now() - - for key, timestamp := range c.recordedEvents { - // Determine the appropriate window based on event type - // Fencing events have longer window - window := EventDeduplicationWindowDefault - if strings.Contains(key, warningPrefixFencingEvent) { - window = EventDeduplicationWindowFencing - } - - // Remove if expired - if now.Sub(timestamp) > window { - delete(c.recordedEvents, key) - } - } -} - -// shouldRecordEvent checks if an event should be recorded based on deduplication logic. -// This should be called after cleanupExpiredEvents() has been called. -func (c *HealthCheck) shouldRecordEvent(eventKey string, deduplicationWindow time.Duration) bool { - c.recordedEventsMu.Lock() - defer c.recordedEventsMu.Unlock() - - now := time.Now() - - // Check if this event was recently recorded - if lastRecorded, exists := c.recordedEvents[eventKey]; exists { - if now.Sub(lastRecorded) < deduplicationWindow { - return false - } - } - - // Record this event - c.recordedEvents[eventKey] = now - return true -} - -// recordHealthCheckEvents records events for warnings, errors, and health transitions. -// current is the newly computed health status, previous is the last health status (may be nil on first sync). -func (c *HealthCheck) recordHealthCheckEvents(current *HealthStatus, previous *HealthStatus) { - // Clean up expired events before processing new ones - c.cleanupExpiredEvents() - - // Record events for warnings with appropriate deduplication window - for _, warning := range current.Warnings { - eventKey := fmt.Sprintf(eventKeyPrefixWarning+"%s", warning) - // Use longer deduplication window for fencing events - deduplicationWindow := EventDeduplicationWindowDefault - if strings.Contains(warning, warningPrefixFencingEvent) { - deduplicationWindow = EventDeduplicationWindowFencing - } - if c.shouldRecordEvent(eventKey, deduplicationWindow) { - c.recordWarningEvent(warning) - } - } - - // Record events for errors with default deduplication window - for _, err := range current.Errors { - eventKey := fmt.Sprintf(eventKeyPrefixError+"%s", err) - if c.shouldRecordEvent(eventKey, EventDeduplicationWindowDefault) { - c.recordErrorEvent(err) - } - } - - // Track and detect health transitions - c.recordHealthTransitionEvents(current, previous) -} - -// recordHealthTransitionEvents detects and records health state transitions. -// - PacemakerHealthy: fires on transition to operationally healthy from degraded OR unknown state -// - PacemakerWarningsCleared: fires when warnings are resolved, regardless of overall health -func (c *HealthCheck) recordHealthTransitionEvents(current *HealthStatus, previous *HealthStatus) { - // Determine previous state from the previous HealthStatus we computed. - // This is derived from the previous PacemakerCluster CR we processed. - previousWasUnknown := previous == nil || previous.OverallStatus == statusUnknown - previousWasDegraded := previous != nil && previous.OverallStatus == statusError - previousHadWarnings := previous != nil && len(previous.Warnings) > 0 - - // Determine current state - operationallyHealthy := current.OverallStatus == statusHealthy || current.OverallStatus == statusWarning - currentHasNoWarnings := len(current.Warnings) == 0 - - // Record PacemakerHealthy when transitioning to operationally healthy from: - // 1. Error state (previousWasDegraded) - recovering from known problems - // 2. Unknown state (previousWasUnknown) - initial healthy status or recovering from stale/missing CR - // Both Healthy and Warning are "operationally healthy" (cluster is functional). - if operationallyHealthy && (previousWasDegraded || previousWasUnknown) { - c.eventRecorder.Eventf(EventReasonHealthy, msgPacemakerHealthy) - klog.Infof("Pacemaker cluster transitioned to healthy (status: %s, fromDegraded: %v, fromUnknown: %v)", - current.OverallStatus, previousWasDegraded, previousWasUnknown) - } - - // Record PacemakerWarningsCleared when warnings are resolved. - // This fires whenever the previous status had warnings and the current one doesn't, - // regardless of overall cluster health. Warnings are independent of error state. - // Examples: - // - Warning → Healthy: warnings resolved, cluster fully healthy - // - Error+Warning → Error: warnings resolved, but cluster still degraded for other reasons - // - Error+Warning → Healthy: warnings resolved AND cluster recovered (fires both events) - if previousHadWarnings && currentHasNoWarnings { - c.eventRecorder.Eventf(EventReasonWarningsCleared, msgPacemakerWarningsCleared) - klog.Infof("Pacemaker cluster warnings cleared") - } -} - -// recordWarningEvent records appropriate warning events based on warning type. -// Note: Failed action events (warningPrefixFailedAction) are recorded by the status collector -// directly from pacemaker XML, not by the healthcheck controller. The PacemakerCluster CR -// only contains conditions, not raw failed action history. -func (c *HealthCheck) recordWarningEvent(warning string) { - switch { - case strings.Contains(warning, warningPrefixFencingEvent): - c.eventRecorder.Warningf(EventReasonFencingEvent, msgDetectedFencing, warning) - default: - c.eventRecorder.Warningf(EventReasonWarning, msgPacemakerWarning, warning) - } -} - -// getEventReasonForError determines the appropriate event reason based on error content. +// GetEventReasonForError determines the appropriate event reason based on error content. // Used for both deduplication (key) and event recording. -func getEventReasonForError(errorMsg string) string { +func GetEventReasonForError(errorMsg string) string { for _, entry := range errorPatternsByPriority { if strings.Contains(errorMsg, entry.pattern) { return entry.reason @@ -885,10 +361,3 @@ func getEventReasonForError(errorMsg string) string { } return EventReasonGenericError } - -// recordErrorEvent records appropriate error events with specific reasons based on error content. -// This allows operators to filter and alert on specific types of degrading conditions. -func (c *HealthCheck) recordErrorEvent(errorMsg string) { - eventReason := getEventReasonForError(errorMsg) - c.eventRecorder.Warningf(eventReason, msgPacemakerError, errorMsg) -} diff --git a/pkg/tnf/pkg/pacemaker/healthcheck_test.go b/pkg/tnf/pkg/pacemaker/healthcheck_test.go deleted file mode 100644 index 3d87e7183a..0000000000 --- a/pkg/tnf/pkg/pacemaker/healthcheck_test.go +++ /dev/null @@ -1,1388 +0,0 @@ -package pacemaker - -import ( - "context" - "fmt" - "strings" - "testing" - "time" - - operatorv1 "github.com/openshift/api/operator/v1" - "github.com/openshift/library-go/pkg/operator/events" - "github.com/openshift/library-go/pkg/operator/v1helpers" - "github.com/stretchr/testify/require" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes/fake" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/cache" - "k8s.io/utils/clock" - - pacmkrv1 "github.com/openshift/api/etcd/v1" -) - -// Test helper functions - -// createFakeOperatorClient creates a fake static pod operator client for testing -func createFakeOperatorClient() v1helpers.StaticPodOperatorClient { - return v1helpers.NewFakeStaticPodOperatorClient( - &operatorv1.StaticPodOperatorSpec{ - OperatorSpec: operatorv1.OperatorSpec{ - ManagementState: operatorv1.Managed, - }, - }, - &operatorv1.StaticPodOperatorStatus{ - OperatorStatus: operatorv1.OperatorStatus{ - Conditions: []operatorv1.OperatorCondition{}, - }, - }, - nil, - nil, - ) -} - -func createTestHealthCheck() *HealthCheck { - return &HealthCheck{ - operatorClient: createFakeOperatorClient(), - kubeClient: fake.NewSimpleClientset(), - eventRecorder: events.NewInMemoryRecorder("test", clock.RealClock{}), - pacemakerInformer: createFakeInformer(nil), - recordedEvents: make(map[string]time.Time), - } -} - -// createFakeInformer creates a fake SharedIndexInformer with a store containing the given object. -// If obj is nil, the store will be empty. -func createFakeInformer(obj *pacmkrv1.PacemakerCluster) cache.SharedIndexInformer { - store := cache.NewStore(func(obj any) (string, error) { - if pc, ok := obj.(*pacmkrv1.PacemakerCluster); ok { - return pc.Name, nil - } - return "", fmt.Errorf("object is not a PacemakerCluster") - }) - if obj != nil { - _ = store.Add(obj) - } - return &fakeInformer{store: store} -} - -// fakeInformer implements cache.SharedIndexInformer for testing -type fakeInformer struct { - store cache.Store -} - -func (f *fakeInformer) AddEventHandler(handler cache.ResourceEventHandler) (cache.ResourceEventHandlerRegistration, error) { - return nil, nil -} -func (f *fakeInformer) AddEventHandlerWithResyncPeriod(handler cache.ResourceEventHandler, resyncPeriod time.Duration) (cache.ResourceEventHandlerRegistration, error) { - return nil, nil -} -func (f *fakeInformer) AddEventHandlerWithOptions(handler cache.ResourceEventHandler, options cache.HandlerOptions) (cache.ResourceEventHandlerRegistration, error) { - return nil, nil -} -func (f *fakeInformer) RemoveEventHandler(handle cache.ResourceEventHandlerRegistration) error { - return nil -} -func (f *fakeInformer) GetStore() cache.Store { return f.store } -func (f *fakeInformer) GetController() cache.Controller { return nil } -func (f *fakeInformer) Run(stopCh <-chan struct{}) {} -func (f *fakeInformer) RunWithContext(ctx context.Context) { -} -func (f *fakeInformer) HasSynced() bool { return true } -func (f *fakeInformer) LastSyncResourceVersion() string { return "" } -func (f *fakeInformer) SetWatchErrorHandler(handler cache.WatchErrorHandler) error { return nil } -func (f *fakeInformer) SetWatchErrorHandlerWithContext(handler cache.WatchErrorHandlerWithContext) error { - return nil -} -func (f *fakeInformer) SetTransform(handler cache.TransformFunc) error { return nil } -func (f *fakeInformer) IsStopped() bool { return false } -func (f *fakeInformer) AddIndexers(indexers cache.Indexers) error { return nil } -func (f *fakeInformer) GetIndexer() cache.Indexer { return nil } - -// createTestHealthCheckWithMockStatus creates a HealthCheck with a mocked Pacemaker CR in the informer cache -func createTestHealthCheckWithMockStatus(t *testing.T, mockStatus *pacmkrv1.PacemakerCluster) *HealthCheck { - return &HealthCheck{ - operatorClient: createFakeOperatorClient(), - kubeClient: fake.NewSimpleClientset(), - eventRecorder: events.NewInMemoryRecorder("test", clock.RealClock{}), - pacemakerInformer: createFakeInformer(mockStatus), - recordedEvents: make(map[string]time.Time), - } -} - -// ============================================================================= -// Unit Tests -// ============================================================================= -// Note: Test fixtures are defined in test_fixtures_test.go - -func TestNewHealthCheck(t *testing.T) { - controller, informer, err := NewHealthCheck( - createFakeOperatorClient(), - fake.NewSimpleClientset(), - events.NewInMemoryRecorder("test", clock.RealClock{}), - &rest.Config{Host: "https://localhost:6443"}, - ) - - require.NoError(t, err, "NewHealthCheck should not return an error") - require.NotNil(t, controller, "Controller should not be nil") - require.NotNil(t, informer, "Informer should not be nil") -} - -func TestHealthCheck_getPacemakerStatus(t *testing.T) { - tests := []struct { - name string - mockStatus *pacmkrv1.PacemakerCluster - expectedStatus HealthStatusValue - expectErrors bool - expectWarnings bool - }{ - { - name: "healthy_status", - mockStatus: &pacmkrv1.PacemakerCluster{ - TypeMeta: metav1.TypeMeta{ - APIVersion: pacmkrv1.SchemeGroupVersion.String(), - Kind: "PacemakerCluster", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "cluster", - }, - Status: pacmkrv1.PacemakerClusterStatus{ - Conditions: createHealthyClusterConditions(), - Nodes: &[]pacmkrv1.PacemakerClusterNodeStatus{ - createHealthyNodeStatus("master-0", []string{"192.168.1.10"}), - createHealthyNodeStatus("master-1", []string{"192.168.1.11"}), - }, - LastUpdated: metav1.Now(), - }, - }, - expectedStatus: statusHealthy, - expectErrors: false, - expectWarnings: false, - }, - { - name: "stale_status", - mockStatus: &pacmkrv1.PacemakerCluster{ - TypeMeta: metav1.TypeMeta{ - APIVersion: pacmkrv1.SchemeGroupVersion.String(), - Kind: "PacemakerCluster", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "cluster", - }, - Status: pacmkrv1.PacemakerClusterStatus{ - Conditions: createHealthyClusterConditions(), - Nodes: &[]pacmkrv1.PacemakerClusterNodeStatus{ - createHealthyNodeStatus("master-0", []string{"192.168.1.10"}), - createHealthyNodeStatus("master-1", []string{"192.168.1.11"}), - }, - LastUpdated: metav1.Time{Time: time.Now().Add(-10 * time.Minute)}, // > 5 min threshold - }, - }, - expectedStatus: statusUnknown, - expectErrors: true, - expectWarnings: false, - }, - { - name: "nil_status", - mockStatus: &pacmkrv1.PacemakerCluster{ - TypeMeta: metav1.TypeMeta{ - APIVersion: pacmkrv1.SchemeGroupVersion.String(), - Kind: "PacemakerCluster", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "cluster", - }, - Status: pacmkrv1.PacemakerClusterStatus{}, // Status not populated yet (zero LastUpdated) - }, - expectedStatus: statusUnknown, - expectErrors: true, - expectWarnings: false, - }, - { - name: "insufficient_nodes", - mockStatus: &pacmkrv1.PacemakerCluster{ - TypeMeta: metav1.TypeMeta{ - APIVersion: pacmkrv1.SchemeGroupVersion.String(), - Kind: "PacemakerCluster", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "cluster", - }, - Status: pacmkrv1.PacemakerClusterStatus{ - Conditions: createInsufficientNodesClusterConditions(), - Nodes: &[]pacmkrv1.PacemakerClusterNodeStatus{ - createHealthyNodeStatus("master-0", []string{"192.168.1.10"}), - }, - LastUpdated: metav1.Now(), - }, - }, - expectedStatus: statusError, - expectErrors: true, - expectWarnings: false, - }, - { - name: "unhealthy_kubelet", - mockStatus: &pacmkrv1.PacemakerCluster{ - TypeMeta: metav1.TypeMeta{ - APIVersion: pacmkrv1.SchemeGroupVersion.String(), - Kind: "PacemakerCluster", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "cluster", - }, - Status: pacmkrv1.PacemakerClusterStatus{ - Conditions: createHealthyClusterConditions(), - Nodes: &[]pacmkrv1.PacemakerClusterNodeStatus{ - createUnhealthyNodeStatus("master-0", []string{"192.168.1.10"}, pacmkrv1.PacemakerClusterResourceNameKubelet), - createHealthyNodeStatus("master-1", []string{"192.168.1.11"}), - }, - LastUpdated: metav1.Now(), - }, - }, - expectedStatus: statusError, - expectErrors: true, - expectWarnings: false, - }, - { - name: "unhealthy_etcd", - mockStatus: &pacmkrv1.PacemakerCluster{ - TypeMeta: metav1.TypeMeta{ - APIVersion: pacmkrv1.SchemeGroupVersion.String(), - Kind: "PacemakerCluster", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "cluster", - }, - Status: pacmkrv1.PacemakerClusterStatus{ - Conditions: createHealthyClusterConditions(), - Nodes: &[]pacmkrv1.PacemakerClusterNodeStatus{ - createHealthyNodeStatus("master-0", []string{"192.168.1.10"}), - createUnhealthyNodeStatus("master-1", []string{"192.168.1.11"}, pacmkrv1.PacemakerClusterResourceNameEtcd), - }, - LastUpdated: metav1.Now(), - }, - }, - expectedStatus: statusError, - expectErrors: true, - expectWarnings: false, - }, - { - name: "unhealthy_etcd_first_node", - mockStatus: &pacmkrv1.PacemakerCluster{ - TypeMeta: metav1.TypeMeta{ - APIVersion: pacmkrv1.SchemeGroupVersion.String(), - Kind: "PacemakerCluster", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "cluster", - }, - Status: pacmkrv1.PacemakerClusterStatus{ - Conditions: createHealthyClusterConditions(), - Nodes: &[]pacmkrv1.PacemakerClusterNodeStatus{ - createUnhealthyNodeStatus("master-0", []string{"192.168.1.10"}, pacmkrv1.PacemakerClusterResourceNameEtcd), - createHealthyNodeStatus("master-1", []string{"192.168.1.11"}), - }, - LastUpdated: metav1.Now(), - }, - }, - expectedStatus: statusError, - expectErrors: true, - expectWarnings: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - controller := createTestHealthCheckWithMockStatus(t, tt.mockStatus) - ctx := context.TODO() - - current, _, err := controller.getPacemakerStatus(ctx) - - require.NoError(t, err, "getPacemakerStatus should not return an error") - require.NotNil(t, current, "HealthStatus should not be nil") - require.Equal(t, tt.expectedStatus, current.OverallStatus, "Status should match expected") - - if tt.expectErrors { - require.NotEmpty(t, current.Errors, "Should have errors") - } else { - require.Empty(t, current.Errors, "Should not have errors") - } - - if tt.expectWarnings { - require.NotEmpty(t, current.Warnings, "Should have warnings") - } else { - require.Empty(t, current.Warnings, "Should not have warnings") - } - }) - } -} - -func TestHealthCheck_getPacemakerStatus_UnchangedTimestamp(t *testing.T) { - // Create a fixed timestamp for the test - fixedTime := metav1.Now() - - mockStatus := &pacmkrv1.PacemakerCluster{ - TypeMeta: metav1.TypeMeta{ - APIVersion: pacmkrv1.SchemeGroupVersion.String(), - Kind: "PacemakerCluster", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "cluster", - }, - Status: pacmkrv1.PacemakerClusterStatus{ - Conditions: createHealthyClusterConditions(), - Nodes: &[]pacmkrv1.PacemakerClusterNodeStatus{ - createHealthyNodeStatus("master-0", []string{"192.168.1.10"}), - createHealthyNodeStatus("master-1", []string{"192.168.1.11"}), - }, - LastUpdated: fixedTime, - }, - } - - controller := createTestHealthCheckWithMockStatus(t, mockStatus) - ctx := context.TODO() - - // First call should return a valid status - current1, _, err1 := controller.getPacemakerStatus(ctx) - require.NoError(t, err1, "First getPacemakerStatus should not return an error") - require.NotNil(t, current1, "First call should return a status") - require.Equal(t, statusHealthy, current1.OverallStatus, "First call should return healthy status") - - // Second call with same timestamp should return nil (no change) - current2, _, err2 := controller.getPacemakerStatus(ctx) - require.NoError(t, err2, "Second getPacemakerStatus should not return an error") - require.Nil(t, current2, "Second call with unchanged timestamp should return nil") -} - -func TestHealthCheck_updateOperatorStatus(t *testing.T) { - tests := []struct { - name string - status *HealthStatus - previous *HealthStatus - }{ - { - name: "error_status_sets_degraded", - status: &HealthStatus{ - OverallStatus: statusError, - Warnings: []string{"Test warning"}, - Errors: []string{"Test error"}, - }, - previous: &HealthStatus{ - OverallStatus: statusHealthy, - CRLastUpdated: time.Now().Add(-1 * time.Minute), - }, - }, - { - name: "healthy_status_clears_degraded", - status: &HealthStatus{ - OverallStatus: statusHealthy, - Warnings: []string{}, - Errors: []string{}, - }, - previous: &HealthStatus{ - OverallStatus: statusHealthy, - CRLastUpdated: time.Now().Add(-1 * time.Minute), - }, - }, - { - name: "warning_status_clears_degraded", - status: &HealthStatus{ - OverallStatus: statusWarning, - Warnings: []string{"Test warning"}, - Errors: []string{}, - }, - previous: &HealthStatus{ - OverallStatus: statusHealthy, - CRLastUpdated: time.Now().Add(-1 * time.Minute), - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - controller := createTestHealthCheck() - ctx := context.TODO() - - err := controller.updateOperatorStatus(ctx, tt.status, tt.previous) - require.NoError(t, err, "updateOperatorStatus should not return an error") - }) - } -} - -func TestHealthCheck_recordHealthCheckEvents(t *testing.T) { - controller := createTestHealthCheck() - - // Test with warnings and errors - previous was healthy (not degraded, no warnings, known status) - previousHealthy := &HealthStatus{OverallStatus: statusHealthy, Warnings: []string{}, Errors: []string{}} - currentWithWarnings := &HealthStatus{ - OverallStatus: statusWarning, - Warnings: []string{ - "Recent failed resource action: kubelet monitor on master-0 failed", - "Recent fencing event: reboot of master-1 success", - "Some other warning", - }, - Errors: []string{"Test error"}, - } - - controller.recordHealthCheckEvents(currentWithWarnings, previousHealthy) - - // Test with healthy status - previous had warnings - currentHealthy := &HealthStatus{ - OverallStatus: statusHealthy, - Warnings: []string{}, - Errors: []string{}, - } - - controller.recordHealthCheckEvents(currentHealthy, currentWithWarnings) -} - -func TestHealthCheck_eventDeduplication(t *testing.T) { - controller := createTestHealthCheck() - recorder := controller.eventRecorder.(events.InMemoryRecorder) - - // Track previous status through the test - var previousStatus *HealthStatus - - // First recording - Error state (start in Error to test transitions properly) - // previousStatus=nil because this is first sync - errorStatus := &HealthStatus{ - OverallStatus: statusError, - Warnings: []string{}, - Errors: []string{"Test error"}, - } - controller.recordHealthCheckEvents(errorStatus, previousStatus) - previousStatus = errorStatus - - // Count events after first recording - initialEvents := len(recorder.Events()) - require.Equal(t, 1, initialEvents, "Should have recorded 1 error event") - - // Immediately record the same error again - should be deduplicated (5 min window) - // previousStatus is now errorStatus (degraded) - controller.recordHealthCheckEvents(errorStatus, previousStatus) - afterDuplicateEvents := len(recorder.Events()) - require.Equal(t, initialEvents, afterDuplicateEvents, "Duplicate error should not be recorded") - - // Transition to Warning (operationally healthy) - should record PacemakerHealthy event - // previousStatus is errorStatus (degraded), so healthy event should fire - warningStatus := &HealthStatus{ - OverallStatus: statusWarning, - Warnings: []string{ - "Recent failed resource action: kubelet monitor on master-0 failed", - }, - Errors: []string{}, - } - controller.recordHealthCheckEvents(warningStatus, previousStatus) - previousStatus = warningStatus - afterWarningTransition := len(recorder.Events()) - require.Equal(t, 3, afterWarningTransition, "Should have recorded warning event + PacemakerHealthy transition event") - - // Stay in Warning - no transition event - controller.recordHealthCheckEvents(warningStatus, previousStatus) - afterWarningDuplicate := len(recorder.Events()) - require.Equal(t, afterWarningTransition, afterWarningDuplicate, "No transition event when staying in Warning") - - // Test fencing event deduplication (longer window) - fencingStatus := &HealthStatus{ - OverallStatus: statusWarning, - Warnings: []string{ - "Recent fencing event: reboot of master-1 success", - }, - Errors: []string{}, - } - controller.recordHealthCheckEvents(fencingStatus, previousStatus) - previousStatus = fencingStatus - afterFencingEvents := len(recorder.Events()) - require.Equal(t, 4, afterFencingEvents, "Fencing event should be recorded") - - // Immediately record the same fencing event - should be deduplicated (24 hour window) - controller.recordHealthCheckEvents(fencingStatus, previousStatus) - afterFencingDuplicateEvents := len(recorder.Events()) - require.Equal(t, afterFencingEvents, afterFencingDuplicateEvents, "Duplicate fencing event should not be recorded") - - // Transition from Warning to Healthy - fires PacemakerWarningsCleared event - // previousStatus has warnings, so WarningsCleared should fire - healthyStatus := &HealthStatus{ - OverallStatus: statusHealthy, - Warnings: []string{}, - Errors: []string{}, - } - controller.recordHealthCheckEvents(healthyStatus, previousStatus) - previousStatus = healthyStatus - afterHealthyTransition := len(recorder.Events()) - require.Equal(t, afterFencingDuplicateEvents+1, afterHealthyTransition, "PacemakerWarningsCleared event should fire on Warning to Healthy transition") - - // Stay healthy - no event - controller.recordHealthCheckEvents(healthyStatus, previousStatus) - afterHealthyDuplicate := len(recorder.Events()) - require.Equal(t, afterHealthyTransition, afterHealthyDuplicate, "No event when staying healthy") - - // Transition to Error - controller.recordHealthCheckEvents(errorStatus, previousStatus) - previousStatus = errorStatus - beforeRecovery := len(recorder.Events()) - - // Transition back to Healthy - should record PacemakerHealthy event - // previousStatus is errorStatus (degraded), so healthy event should fire - controller.recordHealthCheckEvents(healthyStatus, previousStatus) - afterRecovery := len(recorder.Events()) - require.Greater(t, afterRecovery, beforeRecovery, "PacemakerHealthy event should be recorded on transition from Error to Healthy") -} - -func TestHealthCheck_buildHealthStatusFromCR(t *testing.T) { - tests := []struct { - name string - cr *pacmkrv1.PacemakerCluster - expectedStatus HealthStatusValue - expectErrors bool - errorContains string // Optional: check error contains this substring - }{ - { - name: "healthy_cluster", - cr: &pacmkrv1.PacemakerCluster{ - Status: pacmkrv1.PacemakerClusterStatus{ - LastUpdated: metav1.Now(), - Conditions: createHealthyClusterConditions(), - Nodes: &[]pacmkrv1.PacemakerClusterNodeStatus{ - createHealthyNodeStatus("master-0", []string{"192.168.1.10"}), - createHealthyNodeStatus("master-1", []string{"192.168.1.11"}), - }, - }, - }, - expectedStatus: statusHealthy, - expectErrors: false, - }, - { - name: "unhealthy_kubelet", - cr: &pacmkrv1.PacemakerCluster{ - Status: pacmkrv1.PacemakerClusterStatus{ - LastUpdated: metav1.Now(), - Conditions: createHealthyClusterConditions(), - Nodes: &[]pacmkrv1.PacemakerClusterNodeStatus{ - createUnhealthyNodeStatus("master-0", []string{"192.168.1.10"}, pacmkrv1.PacemakerClusterResourceNameKubelet), - createHealthyNodeStatus("master-1", []string{"192.168.1.11"}), - }, - }, - }, - expectedStatus: statusError, - expectErrors: true, - errorContains: string(pacmkrv1.PacemakerClusterResourceNameKubelet), - }, - { - name: "unhealthy_etcd", - cr: &pacmkrv1.PacemakerCluster{ - Status: pacmkrv1.PacemakerClusterStatus{ - LastUpdated: metav1.Now(), - Conditions: createHealthyClusterConditions(), - Nodes: &[]pacmkrv1.PacemakerClusterNodeStatus{ - createHealthyNodeStatus("master-0", []string{"192.168.1.10"}), - createUnhealthyNodeStatus("master-1", []string{"192.168.1.11"}, pacmkrv1.PacemakerClusterResourceNameEtcd), - }, - }, - }, - expectedStatus: statusError, - expectErrors: true, - errorContains: string(pacmkrv1.PacemakerClusterResourceNameEtcd), - }, - { - name: "insufficient_nodes", - cr: &pacmkrv1.PacemakerCluster{ - Status: pacmkrv1.PacemakerClusterStatus{ - LastUpdated: metav1.Now(), - Conditions: createInsufficientNodesClusterConditions(), - Nodes: &[]pacmkrv1.PacemakerClusterNodeStatus{ - createHealthyNodeStatus("master-0", []string{"192.168.1.10"}), - }, - }, - }, - expectedStatus: statusError, - expectErrors: true, - errorContains: "Insufficient nodes", - }, - { - name: "no_nodes", - cr: &pacmkrv1.PacemakerCluster{ - Status: pacmkrv1.PacemakerClusterStatus{ - LastUpdated: metav1.Now(), - Conditions: createHealthyClusterConditions(), - Nodes: &[]pacmkrv1.PacemakerClusterNodeStatus{}, - }, - }, - expectedStatus: statusError, - expectErrors: true, - errorContains: "No nodes found", - }, - { - name: "unpopulated_status", - cr: &pacmkrv1.PacemakerCluster{ - Status: pacmkrv1.PacemakerClusterStatus{}, // Zero LastUpdated - }, - expectedStatus: statusUnknown, - expectErrors: true, - errorContains: "Internal error", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - controller := &HealthCheck{} - status := controller.buildHealthStatusFromCR(tt.cr) - - require.NotNil(t, status, "HealthStatus should not be nil") - require.Equal(t, tt.expectedStatus, status.OverallStatus) - - if tt.expectErrors { - require.NotEmpty(t, status.Errors, "Should have errors") - if tt.errorContains != "" { - found := false - for _, err := range status.Errors { - if strings.Contains(err, tt.errorContains) { - found = true - break - } - } - require.True(t, found, "Error should contain %q", tt.errorContains) - } - } else { - require.Empty(t, status.Errors, "Should have no errors") - require.Empty(t, status.Warnings, "Should have no warnings") - } - }) - } -} - -func TestHealthCheck_checkClusterConditions(t *testing.T) { - tests := []struct { - name string - conditions []metav1.Condition - nodes *[]pacmkrv1.PacemakerClusterNodeStatus - expectErrors bool - errorContains []string // Multiple strings to check - }{ - { - name: "healthy_cluster", - conditions: createHealthyClusterConditions(), - nodes: &[]pacmkrv1.PacemakerClusterNodeStatus{ - createHealthyNodeStatus("master-0", []string{"192.168.1.10"}), - createHealthyNodeStatus("master-1", []string{"192.168.1.11"}), - }, - expectErrors: false, - }, - { - name: "insufficient_nodes", - conditions: createInsufficientNodesClusterConditions(), - nodes: &[]pacmkrv1.PacemakerClusterNodeStatus{ - createHealthyNodeStatus("master-0", []string{"192.168.1.10"}), - }, - expectErrors: true, - errorContains: []string{"Insufficient nodes"}, - }, - { - name: "excessive_nodes", - conditions: createExcessiveNodesClusterConditions(), - nodes: &[]pacmkrv1.PacemakerClusterNodeStatus{ - createHealthyNodeStatus("master-0", []string{"192.168.1.10"}), - createHealthyNodeStatus("master-1", []string{"192.168.1.11"}), - createHealthyNodeStatus("master-2", []string{"192.168.1.12"}), - }, - expectErrors: true, - errorContains: []string{"Excessive nodes", "expected 2, found 3"}, - }, - { - name: "maintenance_mode", - conditions: createMaintenanceModeClusterConditions(), - nodes: &[]pacmkrv1.PacemakerClusterNodeStatus{ - createHealthyNodeStatus("master-0", []string{"192.168.1.10"}), - createHealthyNodeStatus("master-1", []string{"192.168.1.11"}), - }, - expectErrors: true, - errorContains: []string{"maintenance mode"}, - }, - { - name: "empty_conditions", - conditions: []metav1.Condition{}, - nodes: &[]pacmkrv1.PacemakerClusterNodeStatus{ - createHealthyNodeStatus("master-0", []string{"192.168.1.10"}), - }, - expectErrors: true, // Empty conditions means we can't verify cluster health - errorContains: []string{"No cluster conditions available"}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - controller := &HealthCheck{} - pacemakerStatus := &pacmkrv1.PacemakerCluster{ - Status: pacmkrv1.PacemakerClusterStatus{ - Conditions: tt.conditions, - Nodes: tt.nodes, - }, - } - - status := &HealthStatus{Warnings: []string{}, Errors: []string{}} - controller.checkClusterConditions(pacemakerStatus, status) - - if tt.expectErrors { - require.NotEmpty(t, status.Errors, "Should have errors") - for _, expected := range tt.errorContains { - require.Contains(t, status.Errors[0], expected, "Error should contain: %s", expected) - } - } else { - require.Empty(t, status.Errors, "Should have no errors") - } - }) - } -} - -func TestHealthCheck_checkNodeStatuses(t *testing.T) { - tests := []struct { - name string - nodes *[]pacmkrv1.PacemakerClusterNodeStatus - expectErrors bool - errorContains string - }{ - { - name: "healthy_nodes", - nodes: &[]pacmkrv1.PacemakerClusterNodeStatus{ - createHealthyNodeStatus("master-0", []string{"192.168.1.10"}), - createHealthyNodeStatus("master-1", []string{"192.168.1.11"}), - }, - expectErrors: false, - }, - { - name: "unhealthy_kubelet", - nodes: &[]pacmkrv1.PacemakerClusterNodeStatus{ - createUnhealthyNodeStatus("master-0", []string{"192.168.1.10"}, pacmkrv1.PacemakerClusterResourceNameKubelet), - createHealthyNodeStatus("master-1", []string{"192.168.1.11"}), - }, - expectErrors: true, - errorContains: string(pacmkrv1.PacemakerClusterResourceNameKubelet), - }, - { - name: "unhealthy_etcd", - nodes: &[]pacmkrv1.PacemakerClusterNodeStatus{ - createUnhealthyNodeStatus("master-1", []string{"192.168.1.11"}, pacmkrv1.PacemakerClusterResourceNameEtcd), - createHealthyNodeStatus("master-0", []string{"192.168.1.10"}), - }, - expectErrors: true, - errorContains: string(pacmkrv1.PacemakerClusterResourceNameEtcd), - }, - { - name: "nil_nodes", - nodes: nil, - expectErrors: true, // Nil nodes means we can't verify cluster health - errorContains: "No nodes found", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - controller := &HealthCheck{} - pacemakerStatus := &pacmkrv1.PacemakerCluster{ - Status: pacmkrv1.PacemakerClusterStatus{ - Conditions: createHealthyClusterConditions(), - Nodes: tt.nodes, - }, - } - - status := &HealthStatus{Warnings: []string{}, Errors: []string{}} - controller.checkNodeStatuses(pacemakerStatus, status) - - if tt.expectErrors { - require.NotEmpty(t, status.Errors, "Should have errors") - if tt.errorContains != "" { - found := false - for _, err := range status.Errors { - if strings.Contains(err, tt.errorContains) { - found = true - break - } - } - require.True(t, found, "Should have error containing: %s", tt.errorContains) - } - } else { - require.Empty(t, status.Errors, "Should have no errors") - } - }) - } -} - -func TestHealthCheck_checkNodeStatuses_MultipleUnhealthyResources(t *testing.T) { - controller := &HealthCheck{} - - // Create a node with multiple unhealthy resources - node := createHealthyNodeStatus("master-0", []string{"192.168.1.10"}) - // Mark node as unhealthy - for i := range node.Conditions { - if node.Conditions[i].Type == pacmkrv1.NodeHealthyConditionType { - node.Conditions[i].Status = metav1.ConditionFalse - node.Conditions[i].Reason = pacmkrv1.NodeHealthyReasonUnhealthy - } - } - // Mark kubelet and etcd as unhealthy - for i := range node.Resources { - if node.Resources[i].Name == pacmkrv1.PacemakerClusterResourceNameKubelet || - node.Resources[i].Name == pacmkrv1.PacemakerClusterResourceNameEtcd { - node.Resources[i].Conditions = createUnhealthyResourceConditions() - } - } - - pacemakerStatus := &pacmkrv1.PacemakerCluster{ - Status: pacmkrv1.PacemakerClusterStatus{ - Nodes: &[]pacmkrv1.PacemakerClusterNodeStatus{ - node, - createHealthyNodeStatus("master-1", []string{"192.168.1.11"}), - }, - }, - } - - status := &HealthStatus{Warnings: []string{}, Errors: []string{}} - controller.checkNodeStatuses(pacemakerStatus, status) - - // Should have 1 consolidated error for the unhealthy node (includes all resource issues) - require.Len(t, status.Errors, 1, "Should have 1 consolidated error for unhealthy node") - - // Verify the single error mentions both resources - nodeError := status.Errors[0] - require.Contains(t, nodeError, "master-0", "Error should mention the node name") - require.Contains(t, nodeError, "node is unhealthy", "Error should indicate node is unhealthy") - require.Contains(t, nodeError, string(pacmkrv1.PacemakerClusterResourceNameKubelet), "Error should mention Kubelet") - require.Contains(t, nodeError, string(pacmkrv1.PacemakerClusterResourceNameEtcd), "Error should mention Etcd") -} - -func TestHealthCheck_getResourceIssue(t *testing.T) { - controller := &HealthCheck{} - - tests := []struct { - name string - conditions []metav1.Condition - wantIssue string - }{ - { - name: "failed takes priority", - conditions: []metav1.Condition{{Type: pacmkrv1.ResourceOperationalConditionType, Status: metav1.ConditionFalse}}, - wantIssue: "has failed", - }, - { - name: "stopped", - conditions: []metav1.Condition{{Type: pacmkrv1.ResourceStartedConditionType, Status: metav1.ConditionFalse}}, - wantIssue: "is stopped", - }, - { - name: "inactive", - conditions: []metav1.Condition{{Type: pacmkrv1.ResourceActiveConditionType, Status: metav1.ConditionFalse}}, - wantIssue: "is not active", - }, - { - name: "unmanaged", - conditions: []metav1.Condition{{Type: pacmkrv1.ResourceManagedConditionType, Status: metav1.ConditionFalse}}, - wantIssue: "is unmanaged", - }, - { - name: "fallback to generic unhealthy", - conditions: []metav1.Condition{{Type: pacmkrv1.ResourceHealthyConditionType, Status: metav1.ConditionFalse}}, - wantIssue: "is unhealthy", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := controller.getResourceIssue(tt.conditions) - require.Equal(t, tt.wantIssue, got) - }) - } -} - -func TestHealthCheck_recordErrorEvent(t *testing.T) { - controller := createTestHealthCheck() - recorder := controller.eventRecorder.(events.InMemoryRecorder) - - // Test different error types get the correct event reason - tests := []struct { - name string - errorMsg string - expectedReason string - }{ - { - name: "resource_unhealthy", - errorMsg: "Kubelet resource is unhealthy on node master-0: resource has failed", - expectedReason: EventReasonResourceUnhealthy, - }, - { - name: "node_unhealthy", - errorMsg: "master-0 node is unhealthy: node has issues", - expectedReason: EventReasonNodeUnhealthy, - }, - { - name: "cluster_unhealthy", - errorMsg: "Cluster is unhealthy: Pacemaker cluster has issues that need investigation", - expectedReason: EventReasonClusterUnhealthy, - }, - { - name: "insufficient_nodes", - errorMsg: "Insufficient nodes in cluster (expected 2, found 1)", - expectedReason: EventReasonInsufficientNodes, - }, - { - name: "status_stale", - errorMsg: "Pacemaker status is stale", - expectedReason: EventReasonStatusStale, - }, - { - name: "cr_not_found", - errorMsg: "Failed to get PacemakerCluster CR: not found", - expectedReason: EventReasonCRNotFound, - }, - { - name: "cr_no_status", - errorMsg: "PacemakerCluster CR has no status populated", - expectedReason: EventReasonCRNotFound, - }, - { - name: "node_offline", - errorMsg: "Node master-0 is offline", - expectedReason: EventReasonNodeOffline, - }, - { - name: "excessive_nodes", - errorMsg: "Excessive nodes in cluster (expected 2, found 3)", - expectedReason: EventReasonInsufficientNodes, - }, - { - name: "cluster_maintenance", - errorMsg: "Cluster is in maintenance mode", - expectedReason: EventReasonClusterInMaintenance, - }, - { - name: "generic_error", - errorMsg: "Some unknown error", - expectedReason: EventReasonGenericError, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - initialCount := len(recorder.Events()) - controller.recordErrorEvent(tt.errorMsg) - - events := recorder.Events() - require.Len(t, events, initialCount+1, "Should have recorded one more event") - - // Get the last event - lastEvent := events[len(events)-1] - require.Equal(t, tt.expectedReason, lastEvent.Reason, "Event reason should match expected") - require.Contains(t, lastEvent.Message, tt.errorMsg, "Event message should contain error message") - }) - } -} - -func TestHealthCheck_eventDeduplication_StatusTransitions(t *testing.T) { - controller := createTestHealthCheck() - recorder := controller.eventRecorder.(events.InMemoryRecorder) - - // Track previous status through the test - var previousStatus *HealthStatus - - // Start with healthy status from Unknown state (first sync or recovering from stale) - // previousStatus=nil: PacemakerHealthy event fires on initial healthy status - healthyStatus := &HealthStatus{ - OverallStatus: statusHealthy, - Warnings: []string{}, - Errors: []string{}, - } - controller.recordHealthCheckEvents(healthyStatus, previousStatus) // previous=nil (first sync) - previousStatus = healthyStatus - afterFirstHealthy := len(recorder.Events()) - require.Equal(t, 1, afterFirstHealthy, "Healthy event should fire on transition from Unknown") - - // Stay healthy - should not record - controller.recordHealthCheckEvents(healthyStatus, previousStatus) - require.Equal(t, afterFirstHealthy, len(recorder.Events()), "No event when staying healthy") - - // Transition to Warning - should record warning but not healthy event (already in healthy state) - warningStatus := &HealthStatus{ - OverallStatus: statusWarning, - Warnings: []string{"Some warning"}, - Errors: []string{}, - } - controller.recordHealthCheckEvents(warningStatus, previousStatus) - previousStatus = warningStatus - afterWarning := len(recorder.Events()) - require.Equal(t, afterFirstHealthy+1, afterWarning, "Warning event should be recorded") - - // Transition from Warning to Healthy - should record PacemakerWarningsCleared event - // previousStatus has warnings, so WarningsCleared should fire - controller.recordHealthCheckEvents(healthyStatus, previousStatus) - previousStatus = healthyStatus - afterSecondHealthy := len(recorder.Events()) - require.Equal(t, afterWarning+1, afterSecondHealthy, "PacemakerWarningsCleared event should fire on Warning to Healthy transition") - - // Transition to Error - errorStatus := &HealthStatus{ - OverallStatus: statusError, - Warnings: []string{}, - Errors: []string{"Critical error"}, - } - controller.recordHealthCheckEvents(errorStatus, previousStatus) - previousStatus = errorStatus - afterError := len(recorder.Events()) - require.Greater(t, afterError, afterSecondHealthy, "Error should be recorded") - - // Transition from Error to Healthy - should record PacemakerHealthy event - // previousStatus is errorStatus (degraded), so healthy event should fire - controller.recordHealthCheckEvents(healthyStatus, previousStatus) - afterThirdHealthy := len(recorder.Events()) - require.Equal(t, afterError+1, afterThirdHealthy, "Healthy event should be recorded on transition from Error") -} - -func TestHealthCheck_eventDeduplication_CleanupOldEntries(t *testing.T) { - controller := createTestHealthCheck() - - // Add a very old entry to the recorded events map - oldEventKey := "warning:Very old warning" - controller.recordedEventsMu.Lock() - controller.recordedEvents[oldEventKey] = time.Now().Add(-25 * time.Hour) // 25 hours ago (older than longest window of 24 hours) - controller.recordedEventsMu.Unlock() - - // Trigger cleanup by recording a new event - previousStatus := &HealthStatus{OverallStatus: statusHealthy, Warnings: []string{}, Errors: []string{}} - currentStatus := &HealthStatus{ - OverallStatus: statusWarning, - Warnings: []string{"New warning"}, - Errors: []string{}, - } - controller.recordHealthCheckEvents(currentStatus, previousStatus) - - // Verify old entry was cleaned up - controller.recordedEventsMu.Lock() - _, exists := controller.recordedEvents[oldEventKey] - require.False(t, exists, "Old entry should have been cleaned up") - controller.recordedEventsMu.Unlock() -} - -func TestHealthCheck_getNodeConditionErrorsAndWarnings(t *testing.T) { - controller := &HealthCheck{} - now := metav1.Now() - - tests := []struct { - name string - conditions []metav1.Condition - expectedErrors []string - expectedWarnings []string - }{ - { - name: "in_maintenance_mode", - conditions: []metav1.Condition{ - {Type: pacmkrv1.NodeInServiceConditionType, Status: metav1.ConditionFalse, Reason: pacmkrv1.NodeInServiceReasonInMaintenance, LastTransitionTime: now}, - }, - expectedErrors: []string{"in maintenance mode"}, - expectedWarnings: []string{}, - }, - { - name: "in_standby_mode", - conditions: []metav1.Condition{ - {Type: pacmkrv1.NodeActiveConditionType, Status: metav1.ConditionFalse, Reason: pacmkrv1.NodeActiveReasonStandby, LastTransitionTime: now}, - }, - expectedErrors: []string{"in standby mode"}, - expectedWarnings: []string{}, - }, - { - name: "unclean_state", - conditions: []metav1.Condition{ - {Type: pacmkrv1.NodeCleanConditionType, Status: metav1.ConditionFalse, Reason: pacmkrv1.NodeCleanReasonUnclean, LastTransitionTime: now}, - }, - expectedErrors: []string{"unclean state (fencing/communication issue)"}, - expectedWarnings: []string{}, - }, - { - name: "not_a_member", - conditions: []metav1.Condition{ - {Type: pacmkrv1.NodeMemberConditionType, Status: metav1.ConditionFalse, Reason: pacmkrv1.NodeMemberReasonNotMember, LastTransitionTime: now}, - }, - expectedErrors: []string{"not a cluster member"}, - expectedWarnings: []string{}, - }, - { - name: "fencing_unavailable", - conditions: []metav1.Condition{ - {Type: pacmkrv1.NodeFencingAvailableConditionType, Status: metav1.ConditionFalse, Reason: pacmkrv1.NodeFencingAvailableReasonUnavailable, LastTransitionTime: now}, - }, - expectedErrors: []string{"fencing unavailable (no agents running)"}, - expectedWarnings: []string{}, - }, - { - name: "fencing_unhealthy_but_available_is_warning", - conditions: []metav1.Condition{ - {Type: pacmkrv1.NodeFencingAvailableConditionType, Status: metav1.ConditionTrue, Reason: pacmkrv1.NodeFencingAvailableReasonAvailable, LastTransitionTime: now}, - {Type: pacmkrv1.NodeFencingHealthyConditionType, Status: metav1.ConditionFalse, Reason: pacmkrv1.NodeFencingHealthyReasonUnhealthy, LastTransitionTime: now}, - }, - expectedErrors: []string{}, - expectedWarnings: []string{"fencing at risk"}, - }, - { - name: "pending_state_no_error_or_warning", - conditions: []metav1.Condition{ - {Type: pacmkrv1.NodeReadyConditionType, Status: metav1.ConditionFalse, Reason: pacmkrv1.NodeReadyReasonPending, LastTransitionTime: now}, - }, - expectedErrors: []string{}, - expectedWarnings: []string{}, - }, - { - name: "multiple_errors", - conditions: []metav1.Condition{ - {Type: pacmkrv1.NodeInServiceConditionType, Status: metav1.ConditionFalse, LastTransitionTime: now}, - {Type: pacmkrv1.NodeActiveConditionType, Status: metav1.ConditionFalse, LastTransitionTime: now}, - {Type: pacmkrv1.NodeCleanConditionType, Status: metav1.ConditionFalse, LastTransitionTime: now}, - }, - expectedErrors: []string{"in maintenance mode", "in standby mode", "unclean state (fencing/communication issue)"}, - expectedWarnings: []string{}, - }, - { - name: "error_and_warning_together", - conditions: []metav1.Condition{ - {Type: pacmkrv1.NodeInServiceConditionType, Status: metav1.ConditionFalse, LastTransitionTime: now}, - {Type: pacmkrv1.NodeFencingAvailableConditionType, Status: metav1.ConditionTrue, Reason: pacmkrv1.NodeFencingAvailableReasonAvailable, LastTransitionTime: now}, - {Type: pacmkrv1.NodeFencingHealthyConditionType, Status: metav1.ConditionFalse, Reason: pacmkrv1.NodeFencingHealthyReasonUnhealthy, LastTransitionTime: now}, - }, - expectedErrors: []string{"in maintenance mode"}, - expectedWarnings: []string{"fencing at risk"}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Check errors and warnings using the separate functions - errors := controller.getNodeConditionErrors(tt.conditions) - warnings := controller.getFencingWarnings(tt.conditions) - - // Check errors - require.Equal(t, len(tt.expectedErrors), len(errors), "Number of errors should match") - for _, expected := range tt.expectedErrors { - found := false - for _, err := range errors { - if strings.Contains(err, expected) { - found = true - break - } - } - require.True(t, found, "Expected error '%s' not found in %v", expected, errors) - } - - // Check warnings - require.Equal(t, len(tt.expectedWarnings), len(warnings), "Number of warnings should match") - for _, expected := range tt.expectedWarnings { - found := false - for _, warn := range warnings { - if strings.Contains(warn, expected) { - found = true - break - } - } - require.True(t, found, "Expected warning '%s' not found in %v", expected, warnings) - } - }) - } -} - -// TestHealthCheck_FencingRedundancyWarning tests fencing redundancy warnings in various scenarios. -// Fencing redundancy degraded (FencingHealthy=False but FencingAvailable=True) should be -// reported as a warning, not an error, and should be captured regardless of overall node health. -func TestHealthCheck_FencingRedundancyWarning(t *testing.T) { - tests := []struct { - name string - nodeHealthy bool // Overall node healthy condition - inMaintenance bool // Node in maintenance mode (causes error) - expectWarnings bool // Expect fencing warning - expectErrors bool // Expect errors - expectedWarningContain string // String warning should contain - expectedErrorContain string // String error should contain - }{ - { - name: "fencing_degraded_node_unhealthy", - nodeHealthy: false, - inMaintenance: false, - expectWarnings: true, - expectErrors: false, // Only fencing issue, which is a warning - expectedWarningContain: "fencing at risk", - }, - { - name: "fencing_degraded_node_healthy", - nodeHealthy: true, // Node is healthy overall, but fencing redundancy is degraded - inMaintenance: false, - expectWarnings: true, - expectErrors: false, - expectedWarningContain: "fencing at risk", - }, - { - name: "fencing_degraded_with_other_errors", - nodeHealthy: false, - inMaintenance: true, // Causes error - expectWarnings: true, - expectErrors: true, - expectedWarningContain: "fencing at risk", - expectedErrorContain: "maintenance mode", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - controller := &HealthCheck{} - now := metav1.Now() - - // Build conditions based on test case - nodeHealthyStatus := metav1.ConditionFalse - nodeHealthyReason := pacmkrv1.NodeHealthyReasonUnhealthy - if tt.nodeHealthy { - nodeHealthyStatus = metav1.ConditionTrue - nodeHealthyReason = pacmkrv1.NodeHealthyReasonHealthy - } - - inServiceStatus := metav1.ConditionTrue - inServiceReason := pacmkrv1.NodeInServiceReasonInService - if tt.inMaintenance { - inServiceStatus = metav1.ConditionFalse - inServiceReason = pacmkrv1.NodeInServiceReasonInMaintenance - } - - conditions := []metav1.Condition{ - {Type: pacmkrv1.NodeHealthyConditionType, Status: nodeHealthyStatus, Reason: nodeHealthyReason, LastTransitionTime: now}, - {Type: pacmkrv1.NodeOnlineConditionType, Status: metav1.ConditionTrue, LastTransitionTime: now}, - {Type: pacmkrv1.NodeInServiceConditionType, Status: inServiceStatus, Reason: inServiceReason, LastTransitionTime: now}, - {Type: pacmkrv1.NodeActiveConditionType, Status: metav1.ConditionTrue, LastTransitionTime: now}, - {Type: pacmkrv1.NodeReadyConditionType, Status: metav1.ConditionTrue, LastTransitionTime: now}, - {Type: pacmkrv1.NodeCleanConditionType, Status: metav1.ConditionTrue, LastTransitionTime: now}, - {Type: pacmkrv1.NodeMemberConditionType, Status: metav1.ConditionTrue, LastTransitionTime: now}, - // Fencing available but not healthy - warning - {Type: pacmkrv1.NodeFencingAvailableConditionType, Status: metav1.ConditionTrue, Reason: pacmkrv1.NodeFencingAvailableReasonAvailable, LastTransitionTime: now}, - {Type: pacmkrv1.NodeFencingHealthyConditionType, Status: metav1.ConditionFalse, Reason: pacmkrv1.NodeFencingHealthyReasonUnhealthy, LastTransitionTime: now}, - } - - healthyResources := []pacmkrv1.PacemakerClusterResourceStatus{ - {Name: pacmkrv1.PacemakerClusterResourceNameKubelet, Conditions: createHealthyResourceConditions()}, - {Name: pacmkrv1.PacemakerClusterResourceNameEtcd, Conditions: createHealthyResourceConditions()}, - } - - node := pacmkrv1.PacemakerClusterNodeStatus{ - NodeName: "master-0", - Conditions: conditions, - Resources: healthyResources, - } - - status := &HealthStatus{Warnings: []string{}, Errors: []string{}} - controller.checkNodeConditions(node, status) - - // Check warnings - if tt.expectWarnings { - require.NotEmpty(t, status.Warnings, "Should have warnings") - require.Contains(t, status.Warnings[0], "master-0", "Warning should include node name") - require.Contains(t, status.Warnings[0], tt.expectedWarningContain, "Warning should contain expected text") - } else { - require.Empty(t, status.Warnings, "Should not have warnings") - } - - // Check errors - if tt.expectErrors { - require.NotEmpty(t, status.Errors, "Should have errors") - require.Contains(t, status.Errors[0], tt.expectedErrorContain, "Error should contain expected text") - } else { - require.Empty(t, status.Errors, "Should not have errors") - } - }) - } -} - -// TestHealthCheck_WarningsClearedEvent tests PacemakerWarningsCleared and PacemakerHealthy -// event firing across various state transitions. -func TestHealthCheck_WarningsClearedEvent(t *testing.T) { - tests := []struct { - name string - previousStatus *HealthStatus - currentStatus *HealthStatus - expectHealthyEvent bool - expectWarningsCleared bool - }{ - { - name: "warning_to_healthy", - previousStatus: &HealthStatus{ - OverallStatus: statusWarning, - Warnings: []string{"master-0: " + msgFencingRedundancyLost}, - Errors: []string{}, - }, - currentStatus: &HealthStatus{ - OverallStatus: statusHealthy, - Warnings: []string{}, - Errors: []string{}, - }, - expectHealthyEvent: false, // Not degraded before - expectWarningsCleared: true, // Had warnings before - }, - { - name: "error_without_warnings_to_healthy", - previousStatus: &HealthStatus{ - OverallStatus: statusError, - Warnings: []string{}, - Errors: []string{"Critical error"}, - }, - currentStatus: &HealthStatus{ - OverallStatus: statusHealthy, - Warnings: []string{}, - Errors: []string{}, - }, - expectHealthyEvent: true, // Was degraded - expectWarningsCleared: false, // No warnings before - }, - { - name: "error_with_warnings_to_error_without_warnings", - previousStatus: &HealthStatus{ - OverallStatus: statusError, - Warnings: []string{"master-0: " + msgFencingRedundancyLost}, - Errors: []string{"Critical error"}, - }, - currentStatus: &HealthStatus{ - OverallStatus: statusError, - Warnings: []string{}, - Errors: []string{"Critical error"}, - }, - expectHealthyEvent: false, // Still degraded - expectWarningsCleared: true, // Warnings cleared while still degraded - }, - { - name: "error_with_warnings_to_healthy", - previousStatus: &HealthStatus{ - OverallStatus: statusError, - Warnings: []string{"master-0: " + msgFencingRedundancyLost}, - Errors: []string{"Critical error"}, - }, - currentStatus: &HealthStatus{ - OverallStatus: statusHealthy, - Warnings: []string{}, - Errors: []string{}, - }, - expectHealthyEvent: true, // Was degraded - expectWarningsCleared: true, // Had warnings - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - controller := createTestHealthCheck() - recorder := controller.eventRecorder.(events.InMemoryRecorder) - - controller.recordHealthCheckEvents(tt.currentStatus, tt.previousStatus) - - var healthyEventFound, warningsClearedFound bool - for _, e := range recorder.Events() { - if e.Reason == EventReasonHealthy { - healthyEventFound = true - } - if e.Reason == EventReasonWarningsCleared { - warningsClearedFound = true - } - } - - require.Equal(t, tt.expectHealthyEvent, healthyEventFound, - "PacemakerHealthy event expectation mismatch") - require.Equal(t, tt.expectWarningsCleared, warningsClearedFound, - "PacemakerWarningsCleared event expectation mismatch") - }) - } -} diff --git a/pkg/tnf/pkg/pacemaker/statuscollector.go b/pkg/tnf/pkg/pacemaker/statuscollector.go index bfd6649f75..c7f580ced4 100644 --- a/pkg/tnf/pkg/pacemaker/statuscollector.go +++ b/pkg/tnf/pkg/pacemaker/statuscollector.go @@ -86,7 +86,7 @@ func runCollector() error { } // Cluster config provides authoritative node list with IPs - clusterConfig, err := fetchClusterConfig(ctx) + clusterConfig, err := FetchClusterConfig(ctx) if err != nil { klog.Warningf("Failed to fetch cluster config: %v (continuing with limited node data)", err) } @@ -124,7 +124,9 @@ func createKubeClient() (kubernetes.Interface, error) { } // fetchClusterConfig returns the authoritative node list with IP addresses. -func fetchClusterConfig(ctx context.Context) (*ClusterConfig, error) { +// FetchClusterConfig queries pacemaker for the cluster configuration and returns the parsed result. +// This is exported so it can be shared by both the status collector and update-setup runner. +func FetchClusterConfig(ctx context.Context) (*ClusterConfig, error) { ctxExec, cancel := context.WithTimeout(ctx, execTimeout) defer cancel() @@ -387,7 +389,8 @@ func buildResourceStatuses(state *ResourceStatePerNode, now metav1.Time) []pacmk } func buildFencingAgentStatuses(agents []FencingAgentInfo, now metav1.Time) []pacmkrv1.PacemakerClusterFencingAgentStatus { - statuses := make([]pacmkrv1.PacemakerClusterFencingAgentStatus, 0, len(agents)) + // Always return a non-nil slice, even if empty, to satisfy CRD validation (fencingAgents is +required) + statuses := []pacmkrv1.PacemakerClusterFencingAgentStatus{} for _, agent := range agents { // Fencing agents use the same condition structure as resources @@ -788,7 +791,7 @@ func updatePacemakerStatusCR(ctx context.Context, status *pacmkrv1.PacemakerClus return err } - restClient, err := createPacemakerRESTClient(config) + restClient, err := CreatePacemakerRESTClient(config) if err != nil { return err } diff --git a/pkg/tnf/pkg/pacemaker/statuscollector_test.go b/pkg/tnf/pkg/pacemaker/statuscollector_test.go index 76dfe194d6..5318fdc38c 100644 --- a/pkg/tnf/pkg/pacemaker/statuscollector_test.go +++ b/pkg/tnf/pkg/pacemaker/statuscollector_test.go @@ -701,7 +701,7 @@ func TestBuildNodeConditions(t *testing.T) { // Create healthy fencing agents for testing healthyFencingAgents := []pacmkrv1.PacemakerClusterFencingAgentStatus{ { - Conditions: createHealthyResourceConditions(), + Conditions: CreateHealthyResourceConditions(), Name: "master-0_redfish", Method: pacmkrv1.FencingMethodRedfish, }, diff --git a/pkg/tnf/pkg/pacemaker/test_helpers.go b/pkg/tnf/pkg/pacemaker/test_helpers.go new file mode 100644 index 0000000000..64a2548d45 --- /dev/null +++ b/pkg/tnf/pkg/pacemaker/test_helpers.go @@ -0,0 +1,442 @@ +package pacemaker + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + pacmkrv1 "github.com/openshift/api/etcd/v1" +) + +// ============================================================================= +// Shared Test Fixtures +// +// Reusable test data constructors for healthcheck_test.go and statuscollector_test.go. +// XML test data files are in the testdata/ directory. +// ============================================================================= + +// ============================================================================= +// Cluster Condition Fixtures +// ============================================================================= + +// createHealthyClusterConditions creates healthy cluster-level conditions +func CreateHealthyClusterConditions() []metav1.Condition { + now := metav1.Now() + return []metav1.Condition{ + { + Type: pacmkrv1.ClusterHealthyConditionType, + Status: metav1.ConditionTrue, + Reason: pacmkrv1.ClusterHealthyReasonHealthy, + Message: "Cluster is healthy", + LastTransitionTime: now, + }, + { + Type: pacmkrv1.ClusterInServiceConditionType, + Status: metav1.ConditionTrue, + Reason: pacmkrv1.ClusterInServiceReasonInService, + Message: "Cluster is in service", + LastTransitionTime: now, + }, + { + Type: pacmkrv1.ClusterNodeCountAsExpectedConditionType, + Status: metav1.ConditionTrue, + Reason: pacmkrv1.ClusterNodeCountAsExpectedReasonAsExpected, + Message: "Expected 2 nodes, found 2", + LastTransitionTime: now, + }, + } +} + +// createInsufficientNodesClusterConditions creates cluster conditions with insufficient nodes +func CreateInsufficientNodesClusterConditions() []metav1.Condition { + now := metav1.Now() + return []metav1.Condition{ + { + Type: pacmkrv1.ClusterHealthyConditionType, + Status: metav1.ConditionFalse, + Reason: pacmkrv1.ClusterHealthyReasonUnhealthy, + Message: "Cluster has issues", + LastTransitionTime: now, + }, + { + Type: pacmkrv1.ClusterInServiceConditionType, + Status: metav1.ConditionTrue, + Reason: pacmkrv1.ClusterInServiceReasonInService, + Message: "Cluster is in service", + LastTransitionTime: now, + }, + { + Type: pacmkrv1.ClusterNodeCountAsExpectedConditionType, + Status: metav1.ConditionFalse, + Reason: pacmkrv1.ClusterNodeCountAsExpectedReasonInsufficientNodes, + Message: "Expected 2 nodes, found 1", + LastTransitionTime: now, + }, + } +} + +// createExcessiveNodesClusterConditions creates cluster conditions with excessive nodes +func CreateExcessiveNodesClusterConditions() []metav1.Condition { + now := metav1.Now() + return []metav1.Condition{ + { + Type: pacmkrv1.ClusterHealthyConditionType, + Status: metav1.ConditionFalse, + Reason: pacmkrv1.ClusterHealthyReasonUnhealthy, + Message: "Cluster has issues", + LastTransitionTime: now, + }, + { + Type: pacmkrv1.ClusterInServiceConditionType, + Status: metav1.ConditionTrue, + Reason: pacmkrv1.ClusterInServiceReasonInService, + Message: "In service", + LastTransitionTime: now, + }, + { + Type: pacmkrv1.ClusterNodeCountAsExpectedConditionType, + Status: metav1.ConditionFalse, + Reason: pacmkrv1.ClusterNodeCountAsExpectedReasonExcessiveNodes, + Message: "Expected 2 nodes, found 3", + LastTransitionTime: now, + }, + } +} + +// createMaintenanceModeClusterConditions creates cluster conditions for maintenance mode +func CreateMaintenanceModeClusterConditions() []metav1.Condition { + now := metav1.Now() + return []metav1.Condition{ + { + Type: pacmkrv1.ClusterHealthyConditionType, + Status: metav1.ConditionFalse, + Reason: pacmkrv1.ClusterHealthyReasonUnhealthy, + Message: "Cluster in maintenance", + LastTransitionTime: now, + }, + { + Type: pacmkrv1.ClusterInServiceConditionType, + Status: metav1.ConditionFalse, + Reason: pacmkrv1.ClusterInServiceReasonInMaintenance, + Message: "Cluster is in maintenance mode", + LastTransitionTime: now, + }, + { + Type: pacmkrv1.ClusterNodeCountAsExpectedConditionType, + Status: metav1.ConditionTrue, + Reason: pacmkrv1.ClusterNodeCountAsExpectedReasonAsExpected, + Message: "Expected nodes present", + LastTransitionTime: now, + }, + } +} + +// ============================================================================= +// Node Condition Fixtures +// ============================================================================= + +// createHealthyNodeConditions creates healthy node-level conditions +func createHealthyNodeConditions() []metav1.Condition { + now := metav1.Now() + return []metav1.Condition{ + { + Type: pacmkrv1.NodeHealthyConditionType, + Status: metav1.ConditionTrue, + Reason: pacmkrv1.NodeHealthyReasonHealthy, + Message: "Node is healthy", + LastTransitionTime: now, + }, + { + Type: pacmkrv1.NodeOnlineConditionType, + Status: metav1.ConditionTrue, + Reason: pacmkrv1.NodeOnlineReasonOnline, + Message: "Node is online", + LastTransitionTime: now, + }, + { + Type: pacmkrv1.NodeInServiceConditionType, + Status: metav1.ConditionTrue, + Reason: pacmkrv1.NodeInServiceReasonInService, + Message: "Node is in service", + LastTransitionTime: now, + }, + { + Type: pacmkrv1.NodeActiveConditionType, + Status: metav1.ConditionTrue, + Reason: pacmkrv1.NodeActiveReasonActive, + Message: "Node is active", + LastTransitionTime: now, + }, + { + Type: pacmkrv1.NodeReadyConditionType, + Status: metav1.ConditionTrue, + Reason: pacmkrv1.NodeReadyReasonReady, + Message: "Node is ready", + LastTransitionTime: now, + }, + { + Type: pacmkrv1.NodeCleanConditionType, + Status: metav1.ConditionTrue, + Reason: pacmkrv1.NodeCleanReasonClean, + Message: "Node is clean", + LastTransitionTime: now, + }, + { + Type: pacmkrv1.NodeMemberConditionType, + Status: metav1.ConditionTrue, + Reason: pacmkrv1.NodeMemberReasonMember, + Message: "Node is a member", + LastTransitionTime: now, + }, + { + Type: pacmkrv1.NodeFencingAvailableConditionType, + Status: metav1.ConditionTrue, + Reason: pacmkrv1.NodeFencingAvailableReasonAvailable, + Message: "At least one fencing agent is healthy", + LastTransitionTime: now, + }, + { + Type: pacmkrv1.NodeFencingHealthyConditionType, + Status: metav1.ConditionTrue, + Reason: pacmkrv1.NodeFencingHealthyReasonHealthy, + Message: "All fencing agents are healthy", + LastTransitionTime: now, + }, + } +} + +// ============================================================================= +// Resource Condition Fixtures +// ============================================================================= + +// createHealthyResourceConditions creates healthy resource-level conditions +func CreateHealthyResourceConditions() []metav1.Condition { + now := metav1.Now() + return []metav1.Condition{ + { + Type: pacmkrv1.ResourceHealthyConditionType, + Status: metav1.ConditionTrue, + Reason: pacmkrv1.ResourceHealthyReasonHealthy, + Message: "Resource is healthy", + LastTransitionTime: now, + }, + { + Type: pacmkrv1.ResourceInServiceConditionType, + Status: metav1.ConditionTrue, + Reason: pacmkrv1.ResourceInServiceReasonInService, + Message: "Resource is in service", + LastTransitionTime: now, + }, + { + Type: pacmkrv1.ResourceManagedConditionType, + Status: metav1.ConditionTrue, + Reason: pacmkrv1.ResourceManagedReasonManaged, + Message: "Resource is managed", + LastTransitionTime: now, + }, + { + Type: pacmkrv1.ResourceEnabledConditionType, + Status: metav1.ConditionTrue, + Reason: pacmkrv1.ResourceEnabledReasonEnabled, + Message: "Resource is enabled", + LastTransitionTime: now, + }, + { + Type: pacmkrv1.ResourceOperationalConditionType, + Status: metav1.ConditionTrue, + Reason: pacmkrv1.ResourceOperationalReasonOperational, + Message: "Resource is operational", + LastTransitionTime: now, + }, + { + Type: pacmkrv1.ResourceActiveConditionType, + Status: metav1.ConditionTrue, + Reason: pacmkrv1.ResourceActiveReasonActive, + Message: "Resource is active", + LastTransitionTime: now, + }, + { + Type: pacmkrv1.ResourceStartedConditionType, + Status: metav1.ConditionTrue, + Reason: pacmkrv1.ResourceStartedReasonStarted, + Message: "Resource is started", + LastTransitionTime: now, + }, + { + Type: pacmkrv1.ResourceSchedulableConditionType, + Status: metav1.ConditionTrue, + Reason: pacmkrv1.ResourceSchedulableReasonSchedulable, + Message: "Resource is schedulable", + LastTransitionTime: now, + }, + } +} + +// createUnhealthyResourceConditions creates unhealthy resource-level conditions +func CreateUnhealthyResourceConditions() []metav1.Condition { + now := metav1.Now() + return []metav1.Condition{ + { + Type: pacmkrv1.ResourceHealthyConditionType, + Status: metav1.ConditionFalse, + Reason: pacmkrv1.ResourceHealthyReasonUnhealthy, + Message: "Resource is unhealthy", + LastTransitionTime: now, + }, + { + Type: pacmkrv1.ResourceInServiceConditionType, + Status: metav1.ConditionTrue, + Reason: pacmkrv1.ResourceInServiceReasonInService, + Message: "Resource is in service", + LastTransitionTime: now, + }, + { + Type: pacmkrv1.ResourceManagedConditionType, + Status: metav1.ConditionTrue, + Reason: pacmkrv1.ResourceManagedReasonManaged, + Message: "Resource is managed", + LastTransitionTime: now, + }, + { + Type: pacmkrv1.ResourceEnabledConditionType, + Status: metav1.ConditionTrue, + Reason: pacmkrv1.ResourceEnabledReasonEnabled, + Message: "Resource is enabled", + LastTransitionTime: now, + }, + { + Type: pacmkrv1.ResourceOperationalConditionType, + Status: metav1.ConditionFalse, + Reason: pacmkrv1.ResourceOperationalReasonFailed, + Message: "Resource has failed", + LastTransitionTime: now, + }, + { + Type: pacmkrv1.ResourceActiveConditionType, + Status: metav1.ConditionFalse, + Reason: pacmkrv1.ResourceActiveReasonInactive, + Message: "Resource is not active", + LastTransitionTime: now, + }, + { + Type: pacmkrv1.ResourceStartedConditionType, + Status: metav1.ConditionFalse, + Reason: pacmkrv1.ResourceStartedReasonStopped, + Message: "Resource is stopped", + LastTransitionTime: now, + }, + { + Type: pacmkrv1.ResourceSchedulableConditionType, + Status: metav1.ConditionTrue, + Reason: pacmkrv1.ResourceSchedulableReasonSchedulable, + Message: "Resource is schedulable", + LastTransitionTime: now, + }, + } +} + +// ============================================================================= +// Complete Node Status Fixtures +// ============================================================================= + +// createHealthyNodeStatus creates a healthy node status for testing +func CreateHealthyNodeStatus(name string, ipAddresses []string) pacmkrv1.PacemakerClusterNodeStatus { + // Convert IP strings to PacemakerNodeAddress format + addresses := make([]pacmkrv1.PacemakerNodeAddress, len(ipAddresses)) + for i, ip := range ipAddresses { + addresses[i] = pacmkrv1.PacemakerNodeAddress{Type: pacmkrv1.PacemakerNodeInternalIP, Address: ip} + } + + return pacmkrv1.PacemakerClusterNodeStatus{ + Conditions: createHealthyNodeConditions(), + NodeName: name, + Addresses: addresses, + Resources: []pacmkrv1.PacemakerClusterResourceStatus{ + { + Conditions: CreateHealthyResourceConditions(), + Name: pacmkrv1.PacemakerClusterResourceNameKubelet, + }, + { + Conditions: CreateHealthyResourceConditions(), + Name: pacmkrv1.PacemakerClusterResourceNameEtcd, + }, + }, + FencingAgents: []pacmkrv1.PacemakerClusterFencingAgentStatus{ + { + Conditions: CreateHealthyResourceConditions(), + Name: name + "_redfish", + Method: pacmkrv1.FencingMethodRedfish, + }, + }, + } +} + +// createUnhealthyNodeStatus creates an unhealthy node status with specified unhealthy resource +func CreateUnhealthyNodeStatus(name string, ipAddresses []string, unhealthyResourceName pacmkrv1.PacemakerClusterResourceName) pacmkrv1.PacemakerClusterNodeStatus { + now := metav1.Now() + + // Convert IP strings to PacemakerNodeAddress format + addresses := make([]pacmkrv1.PacemakerNodeAddress, len(ipAddresses)) + for i, ip := range ipAddresses { + addresses[i] = pacmkrv1.PacemakerNodeAddress{Type: pacmkrv1.PacemakerNodeInternalIP, Address: ip} + } + + // Create node with unhealthy condition + nodeConditions := createHealthyNodeConditions() + // Mark node as unhealthy + for i := range nodeConditions { + if nodeConditions[i].Type == pacmkrv1.NodeHealthyConditionType { + nodeConditions[i].Status = metav1.ConditionFalse + nodeConditions[i].Reason = pacmkrv1.NodeHealthyReasonUnhealthy + nodeConditions[i].Message = "Node has unhealthy resources" + nodeConditions[i].LastTransitionTime = now + } + } + + resources := []pacmkrv1.PacemakerClusterResourceStatus{ + { + Conditions: CreateHealthyResourceConditions(), + Name: pacmkrv1.PacemakerClusterResourceNameKubelet, + }, + { + Conditions: CreateHealthyResourceConditions(), + Name: pacmkrv1.PacemakerClusterResourceNameEtcd, + }, + } + + // Mark the specified resource as unhealthy + for i := range resources { + if resources[i].Name == unhealthyResourceName { + resources[i].Conditions = CreateUnhealthyResourceConditions() + } + } + + // Create fencing agent - healthy by default + fencingAgents := []pacmkrv1.PacemakerClusterFencingAgentStatus{ + { + Conditions: CreateHealthyResourceConditions(), + Name: name + "_redfish", + Method: pacmkrv1.FencingMethodRedfish, + }, + } + + return pacmkrv1.PacemakerClusterNodeStatus{ + Conditions: nodeConditions, + NodeName: name, + Addresses: addresses, + Resources: resources, + FencingAgents: fencingAgents, + } +} + +// ============================================================================= +// Test Lookup Helpers +// ============================================================================= + +// findResourceInList finds a resource by name in a list of resources +func findResourceInList(resources []pacmkrv1.PacemakerClusterResourceStatus, name pacmkrv1.PacemakerClusterResourceName) *pacmkrv1.PacemakerClusterResourceStatus { + for i := range resources { + if resources[i].Name == name { + return &resources[i] + } + } + return nil +} From 1e5da0a4a9e782e633ddb8e304499f9c062a18ce Mon Sep 17 00:00:00 2001 From: Jeremy Poulin Date: Thu, 25 Jun 2026 15:57:20 -0400 Subject: [PATCH 5/6] OCPBUGS-84695: docs(tnf): architecture overview and alert fix Adds comprehensive architecture documentation and makes alert configuration non-fatal: Documentation: - docs/tnf/README.md: Quick start guide, component overview, workflow summary - docs/tnf/architecture-overview.md: Detailed architecture, dual-mode operation, job types table Alert configuration fix: - setup/update-setup jobs now log warnings instead of failing when alert configuration fails - Allows Pacemaker setup to succeed even if alerts can't be configured - Provides clear troubleshooting guidance in warning messages See docs/tnf/architecture-overview.md for canonical Job Types table and system architecture. Co-Authored-By: Claude Sonnet 4.5 --- docs/tnf/README.md | 104 +++++++++ docs/tnf/architecture-overview.md | 355 ++++++++++++++++++++++++++++++ pkg/tnf/setup/runner.go | 5 +- pkg/tnf/update-setup/runner.go | 5 +- 4 files changed, 467 insertions(+), 2 deletions(-) create mode 100644 docs/tnf/README.md create mode 100644 docs/tnf/architecture-overview.md diff --git a/docs/tnf/README.md b/docs/tnf/README.md new file mode 100644 index 0000000000..1e455c5e59 --- /dev/null +++ b/docs/tnf/README.md @@ -0,0 +1,104 @@ +# TNF (Two-Node with Fencing) Lifecycle Documentation + +This directory contains comprehensive documentation for the TNF lifecycle system in the cluster-etcd-operator. + +## Quick Reference + +| Topic | Document | Description | +|-------|----------|-------------| +| **System Design** | [architecture-overview.md](architecture-overview.md) | High-level architecture, dual-mode operation, component overview | +| **Controller Behavior** | [controller-lifecycle.md](controller-lifecycle.md) | Startup sequence, periodic sync loop, event handling, health monitoring | +| **Job Management** | [job-controllers.md](job-controllers.md) | Bootstrap vs runtime job behavior, retry logic, node selection | +| **Drift Detection** | [reconciliation.md](reconciliation.md) | When reconciliation runs, drift scenarios, safety checks | + +## System Overview + +The TNF lifecycle system manages the integration between OpenShift control plane nodes and a Pacemaker high-availability cluster. It operates in two modes: + +**Bootstrap Mode** (Pre-Transition) +- Drives initial etcd transition from static pods to Pacemaker control +- Requires exactly 2 ready control plane nodes +- Uses exponential backoff retry logic +- Sets degraded conditions on failure + +**Runtime Mode** (Post-Transition) +- Monitors Pacemaker cluster health every 30 seconds +- Detects drift between Kubernetes nodes and Pacemaker membership +- Automatically reconciles differences by adding/removing nodes +- Supports single-node scenarios during node replacement + +## Key Components + +**PacemakerLifecycleManager** (`pkg/tnf/operator/lifecycle_manager.go`) +- Main controller coordinating all TNF operations +- Runs periodic sync loop (every 30 seconds) +- Registers event handlers for node Add/Update/Delete + +**StartJobControllers** (`pkg/tnf/operator/lifecycle_job_controllers.go`) +- Manages TNF job lifecycle +- Dual-mode behavior (bootstrap vs runtime) +- Handles operator restarts gracefully + +**ReconcilePacemakerConfig** (`pkg/tnf/operator/lifecycle_reconciliation.go`) +- Detects and fixes drift between K8s and Pacemaker +- Creates update-setup ConfigMaps with reconciliation decisions +- Protected by mutex to serialize reconciliation attempts + +**MonitorHealth** (`pkg/tnf/operator/lifecycle_health.go`) +- Monitors Pacemaker cluster health +- Updates operator conditions (Degraded, Progressing, Available) +- Records events with deduplication + +## Job Types + +| Job Type | Scope | When | Purpose | +|----------|-------|------|---------| +| **auth** | Per-node | Bootstrap + reconciliation | Set hacluster password | +| **setup** | Cluster | Bootstrap only | Initial Pacemaker cluster creation | +| **after-setup** | Per-node | Bootstrap only | Add node to newly created cluster | +| **fencing** | Cluster | Bootstrap + secret changes | Configure STONITH fencing agents | +| **update-setup** | Cluster | Reconciliation | Add/remove nodes from existing cluster | + +## Common Workflows + +**Initial Bootstrap** +1. Wait for etcd bootstrap completion +2. Wait for all nodes at latest revision +3. Run auth jobs on all nodes +4. Run setup job (creates Pacemaker cluster) +5. Run fencing job (configures STONITH) +6. Run after-setup jobs on all nodes +7. Mark transition complete + +**Node Replacement (Reconciliation)** +1. Detect drift (K8s has different nodes than Pacemaker) +2. Create update-setup ConfigMap with desired state +3. Run auth jobs on all current K8s nodes +4. Run update-setup job (add/remove nodes from Pacemaker) +5. Run after-setup jobs on all current K8s nodes +6. Clean up orphaned jobs for deleted nodes + +**Health Monitoring** +1. Read PacemakerCluster CR status (updated every 1 minute) +2. Check staleness (CR updated recently?) +3. Parse cluster state (nodes online, fencing status, errors) +4. Update operator conditions +5. Record events with deduplication + +## Design Principles + +**Self-Healing** +- Periodic reconciliation detects and fixes drift automatically +- Recovers from missed events or operator downtime + +**Graceful Degradation** +- Operations continue independently +- Health check failure doesn't prevent reconciliation + +**Idempotency** +- Safe to call repeatedly +- Operations detect current state and only act when needed + +**Event + Periodic Hybrid** +- Node events trigger immediate action +- Periodic sync provides backup detection and recovery diff --git a/docs/tnf/architecture-overview.md b/docs/tnf/architecture-overview.md new file mode 100644 index 0000000000..3ce6acca62 --- /dev/null +++ b/docs/tnf/architecture-overview.md @@ -0,0 +1,355 @@ +# TNF Lifecycle Architecture Overview + +## High-Level Design + +The TNF (Two-Node Fencing) lifecycle system ensures that a Pacemaker high-availability cluster stays synchronized with OpenShift control plane nodes. It operates as a Kubernetes controller that runs continuously, executing a periodic sync loop every 30 seconds and responding to node lifecycle events. + +## Dual-Mode Operation + +The lifecycle manager adapts its behavior based on whether the external etcd transition has completed: + +### Bootstrap Mode (Pre-Transition) + +**State:** External etcd is transitioning from static pods to Pacemaker control + +**Operations:** +- Drive the initial transition by starting TNF jobs +- Requires exactly 2 ready control plane nodes +- Uses exponential backoff retry (5s to 2min, ~10 min total) +- Sets `TNFJobControllersDegraded` condition on failures +- Cleans up orphaned jobs from previous attempts + +**Goal:** Successfully transition etcd from static pod management to Pacemaker cluster control + +### Post-Transition Mode (Runtime) + +**State:** External etcd transition is complete, Pacemaker is managing etcd + +**Operations:** +- Monitor Pacemaker cluster health every 30 seconds +- Detect drift between Kubernetes nodes and Pacemaker membership +- Reconcile differences by adding/removing nodes from Pacemaker +- Support single-node scenarios during node replacement +- Clean up orphaned jobs when nodes are deleted + +**Goal:** Maintain a healthy, synchronized Pacemaker cluster that matches the Kubernetes control plane + +## System Architecture + +```text +┌─────────────────────────────────────────────────────────────────┐ +│ PacemakerLifecycleManager │ +│ (Main Controller) │ +└─────────────────────────────────────────────────────────────────┘ + │ + │ Periodic Sync (30s) + │ + Node Events (Add/Update/Delete) + ▼ + ┌────────────────────────────────────────────┐ + │ Check External Etcd │ + │ Transition Complete? │ + └────────────────────────────────────────────┘ + │ + ┌────────────────────┴────────────────────┐ + │ false (Bootstrap) │ true (Runtime) │ + ▼ ▼ ▼ +┌──────────────────┐ ┌──────────────────┐ +│ Bootstrap Mode │ │ Post-Transition │ +│ │ │ Mode │ +│ 1. StartJob │ │ 1. StartJob │ +│ Controllers │ │ Controllers │ +│ (drive │ │ (ensure │ +│ transition) │ │ running) │ +│ │ │ │ +│ 2. CleanupOrphan │ │ 2. CleanupOrphan │ +│ edJobs │ │ edJobs │ +│ (bootstrap │ │ (deleted │ +│ cleanup) │ │ nodes) │ +└──────────────────┘ │ │ + │ 3. MonitorHealth │ + │ (health │ + │ checks) │ + │ │ + │ 4. Reconcile │ + │ Pacemaker │ + │ Config │ + │ (drift fix) │ + └──────────────────┘ +``` + +## Key Components + +### 1. PacemakerLifecycleManager + +**Location:** `pkg/tnf/operator/lifecycle_manager.go` + +The main controller that orchestrates all TNF operations: +- Runs a periodic sync loop (every 30 seconds) +- Registers event handlers for node Add/Update/Delete events +- Coordinates startup, health monitoring, reconciliation, and cleanup +- Maintains state for health transitions and event deduplication + +### 2. StartJobControllers + +**Location:** `pkg/tnf/operator/lifecycle_job_controllers.go` + +Manages the lifecycle of TNF Kubernetes jobs: +- **Bootstrap Mode:** Drives initial transition with retry logic and degraded conditions +- **Runtime Mode:** Ensures jobs stay running, handles operator restarts +- Adapts behavior based on transition state (dual-mode function) +- Protected by mutex to prevent concurrent startup attempts + +### 3. ReconcilePacemakerConfig + +**Location:** `pkg/tnf/operator/lifecycle_reconciliation.go` + +Detects and fixes drift between Kubernetes and Pacemaker: +- Compares K8s nodes (from informer) vs Pacemaker membership (from CR) +- Creates update-setup ConfigMaps with reconciliation decisions +- Triggers update-setup job to add/remove nodes +- Uses mutex to serialize reconciliation attempts +- Only runs after external etcd transition completes + +### 4. MonitorHealth + +**Location:** `pkg/tnf/operator/lifecycle_health.go` + +Monitors Pacemaker cluster health and reports status: +- Reads PacemakerCluster CR for cluster state +- Detects staleness (CR not updated recently) +- Updates operator conditions (Degraded, Progressing, Available) +- Records events with deduplication (5 min default, 24 hr for fencing) +- Only runs after external etcd transition completes + +### 5. CleanupOrphanedJobs + +**Location:** `pkg/tnf/operator/lifecycle_cleanup.go` + +Removes jobs and pods for deleted nodes: +- Lists TNF jobs (auth, after-setup, update-setup) +- Identifies jobs tied to non-existent nodes (via node label) +- Deletes orphaned jobs to prevent resource leaks +- **Also deletes orphaned pods** whose owner Jobs no longer exist +- Runs in both bootstrap and runtime modes + +## Periodic Sync Loop + +The controller runs `sync()` every 30 seconds, executing this sequence: + +```text +┌─────────────────────────────────────────────────────────────┐ +│ sync() - called every 30 seconds │ +└─────────────────────────────────────────────────────────────┘ + │ + ▼ + Check HasExternalEtcdCompletedTransition() + │ + ▼ + ┌────────────────────────────────┐ + │ Run in both modes: │ + │ 1. StartJobControllers() │ + │ 2. CleanupOrphanedJobs() │ + └────────────────────────────────┘ + │ + ▼ + Transition complete? + │ + ┌────┴────┐ + No Yes + │ │ + ▼ ▼ + Return ┌────────────────────┐ + │ 3. MonitorHealth() │ + │ 4. Reconcile │ + │ PacemakerConfig│ + └────────────────────┘ + │ + ▼ + Return errors +``` + +**Error Handling:** +- Each operation continues even if others fail +- Errors are aggregated and returned +- Controller framework marks operator as Degraded if errors persist +- Periodic sync ensures eventual consistency + +## Node Event Handlers + +The controller responds to Kubernetes node lifecycle events: + +### Add Event + +```text +Node added to cluster + │ + ▼ +Trigger ReconcilePacemakerConfig() + │ + ▼ +Drift detection + │ + ▼ +If new node detected: + Create update-setup ConfigMap + Run update-setup job + Add node to Pacemaker +``` + +### Update Event (Ready Transition) + +```text +Node transitions NotReady → Ready + │ + ▼ +Trigger StartJobControllers() + │ + ▼ +Bootstrap Mode: Retry transition +Runtime Mode: Ensure jobs running +``` + +### Update Event (IP Change) + +```text +Node IP changes while Ready + │ + ▼ +Trigger ReconcilePacemakerConfig() + │ + ▼ +Drift detection + │ + ▼ +Update Pacemaker with new IP +``` + +### Delete Event + +```text +Node deleted from cluster + │ + ▼ +Trigger ReconcilePacemakerConfig() + │ + ▼ +Drift detection + │ + ▼ +If node still in Pacemaker: + Create update-setup ConfigMap + Run update-setup job + Remove node from Pacemaker + │ + ▼ +CleanupOrphanedJobs() + │ + ▼ +Delete auth/after-setup jobs +``` + +## Data Flow + +```text +┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ +│ Kubernetes │ │ TNF Lifecycle │ │ Pacemaker │ +│ API Server │────▶│ Manager │────▶│ Cluster │ +└─────────────────┘ └─────────────────┘ └─────────────────┘ + │ │ │ + │ Node informer │ PacemakerCluster │ + │ (watch nodes) │ informer (watch CR) │ + │ │ │ + ▼ ▼ ▼ +┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ +│ Node Events: │ │ Periodic Sync: │ │ Status Updates: │ +│ - Add │ │ - Every 30s │ │ - Cluster state │ +│ - Update │ │ - Drift check │ │ - Node list │ +│ - Delete │ │ - Health check │ │ - Fencing status│ +└─────────────────┘ └─────────────────┘ └─────────────────┘ +``` + +## Concurrency Model + +The lifecycle manager handles concurrent operations safely: + +### Instance-Level Mutexes + +**recordedEventsMu** - Protects event deduplication map +- Prevents concurrent map access +- Short critical sections (map reads/writes only) + +**previousMu** - Protects previous health status +- Serializes health state transitions +- Prevents race conditions in event recording + +**startJobControllersMu** - Serializes job controller startup +- Prevents duplicate job creation +- Critical during bootstrap retry attempts + +### Package-Level Mutexes + +**reconcilePacemakerConfigMutex** (in `lifecycle_reconciliation.go`) +- Serializes all reconciliation attempts +- Shared across all controller instances +- Protects update-setup ConfigMap generation counter + +### Context Management + +**lifecycleCtx / lifecycleCtxCancel** +- Manages goroutine lifecycle for event handlers +- Cancelled during controller shutdown +- Prevents goroutine leaks + +## Job Types + +TNF jobs are Kubernetes Jobs that configure and manage Pacemaker: + +| Job Type | Scope | When | Purpose | +|----------|-------|------|---------| +| **auth** | Per-node | Bootstrap + reconciliation | Set hacluster password for Pacemaker auth | +| **setup** | Cluster | Bootstrap only | Initial Pacemaker cluster creation (Day 1) | +| **after-setup** | Per-node | Bootstrap only | Add node to newly created cluster (Day 1) | +| **fencing** | Cluster | Bootstrap + secret changes | Configure STONITH fencing agents (BMC credentials) - see [Fencing Secret Changes](job-controllers.md#fencing-secret-changes) | +| **update-setup** | Cluster | Reconciliation | Add/remove nodes from existing cluster (Day 2+) | + +**Node-specific jobs** (auth, after-setup): +- Labeled with `node: ` +- Tied to node lifecycle via UID +- Cleaned up when node is deleted + +**Cluster-wide jobs** (setup, fencing, update-setup): +- Not tied to specific nodes +- Persist across node changes +- Update-setup regenerated during reconciliation + +## External Dependencies + +The lifecycle manager integrates with several systems: + +**Kubernetes API:** +- Node informer (control plane nodes) +- Job API (TNF jobs) +- ConfigMap API (update-setup decisions) +- Secret API (fencing credentials) + +**Operator Framework:** +- StaticPodOperatorClient (operator status) +- Degraded/Progressing/Available conditions +- Event recorder + +**Pacemaker:** +- PacemakerCluster CR (cluster status) +- Status collector CronJob (periodic `pcs status xml`) +- Node membership via `pcmk_node_list` + +## Design Principles + +**Self-Healing:** Periodic reconciliation detects and fixes drift automatically, recovering from missed events or operator downtime + +**Dual-Mode Behavior:** Single codebase handles both bootstrap (driving transition) and runtime (maintaining health) without separate code paths + +**Graceful Degradation:** Operations continue independently - a health check failure doesn't prevent reconciliation + +**Idempotency:** Safe to call repeatedly - operations detect current state and only take action when needed + +**Event + Periodic Hybrid:** Node events trigger immediate action, but periodic sync provides backup detection and recovery diff --git a/pkg/tnf/setup/runner.go b/pkg/tnf/setup/runner.go index fac2f5babc..7928626468 100644 --- a/pkg/tnf/setup/runner.go +++ b/pkg/tnf/setup/runner.go @@ -130,9 +130,12 @@ func RunTnfSetup() error { // register pacemaker alert agents for fencing taint/untaint // this is done last so core functionality (etcd resource, stonith) works even if alerts fail + // alerts are optional - don't fail setup if they can't be configured err = pcs.ConfigureAlerts(ctx) if err != nil { - return err + klog.Warningf("Failed to configure pacemaker alerts (non-fatal, continuing): %v", err) + } else { + klog.Infof("Successfully configured pacemaker alerts") } // get pcs cib diff --git a/pkg/tnf/update-setup/runner.go b/pkg/tnf/update-setup/runner.go index 61f26ddc52..4898a502a8 100644 --- a/pkg/tnf/update-setup/runner.go +++ b/pkg/tnf/update-setup/runner.go @@ -306,9 +306,12 @@ func RunTnfUpdateSetup() error { // Register pacemaker alert agents for fencing taint/untaint // This is done last so core functionality (node membership, fencing, etcd resource) works even if alerts fail + // Alerts are optional - don't fail update-setup if they can't be configured err = pcs.ConfigureAlerts(ctx) if err != nil { - return err + klog.Warningf("Failed to configure pacemaker alerts (non-fatal, continuing): %v", err) + } else { + klog.Infof("Successfully configured pacemaker alerts") } return nil From 04fcb8df75e31e85cb6de6039fe339f8a5d662c0 Mon Sep 17 00:00:00 2001 From: Jeremy Poulin Date: Fri, 26 Jun 2026 16:52:55 -0400 Subject: [PATCH 6/6] Capturing WIP review updates --- pkg/operator/ceohelpers/node_helpers.go | 12 +- pkg/operator/ceohelpers/node_helpers_test.go | 120 -------------- pkg/tnf/after-setup/runner.go | 38 ++--- pkg/tnf/fencing/runner.go | 151 ++++++++++++++---- pkg/tnf/operator/lifecycle_helpers.go | 9 -- pkg/tnf/operator/lifecycle_job_controllers.go | 44 ++--- pkg/tnf/operator/lifecycle_manager.go | 2 - pkg/tnf/operator/lifecycle_reconciliation.go | 14 +- pkg/tnf/operator/starter.go | 32 +++- pkg/tnf/pkg/jobs/tnf.go | 4 +- pkg/tnf/pkg/tools/jobs.go | 4 + pkg/tnf/pkg/tools/tnf_labels.go | 5 - pkg/tnf/setup/runner.go | 14 +- pkg/tnf/update-setup/runner.go | 24 +-- 14 files changed, 206 insertions(+), 267 deletions(-) delete mode 100644 pkg/tnf/pkg/tools/tnf_labels.go diff --git a/pkg/operator/ceohelpers/node_helpers.go b/pkg/operator/ceohelpers/node_helpers.go index 906ba8b8b8..bd63e1f5b1 100644 --- a/pkg/operator/ceohelpers/node_helpers.go +++ b/pkg/operator/ceohelpers/node_helpers.go @@ -20,6 +20,8 @@ const ( // ListNodesFromInformer returns all nodes from the given informer. // This is a convenience wrapper around creating a lister and listing all nodes. +// Note: The informer is typically pre-filtered (e.g., controlPlaneNodeInformer), +// so this returns only the nodes that match the informer's filter. func ListNodesFromInformer(informer cache.SharedIndexInformer) ([]*corev1.Node, error) { if informer == nil { return nil, fmt.Errorf("informer is nil") @@ -28,13 +30,3 @@ func ListNodesFromInformer(informer cache.SharedIndexInformer) ([]*corev1.Node, lister := corev1listers.NewNodeLister(informer.GetIndexer()) return lister.List(labels.Everything()) } - -// ListNodesBySelector returns nodes matching the given label selector from the informer. -func ListNodesBySelector(informer cache.SharedIndexInformer, selector labels.Selector) ([]*corev1.Node, error) { - if informer == nil { - return nil, fmt.Errorf("informer is nil") - } - - lister := corev1listers.NewNodeLister(informer.GetIndexer()) - return lister.List(selector) -} diff --git a/pkg/operator/ceohelpers/node_helpers_test.go b/pkg/operator/ceohelpers/node_helpers_test.go index a7bf003add..f0b8670a1b 100644 --- a/pkg/operator/ceohelpers/node_helpers_test.go +++ b/pkg/operator/ceohelpers/node_helpers_test.go @@ -72,126 +72,6 @@ func TestListNodesFromInformer_NilInformer(t *testing.T) { require.Contains(t, err.Error(), "informer is nil") } -func TestListNodesBySelector(t *testing.T) { - tests := []struct { - name string - nodes []*corev1.Node - selector labels.Selector - expectError bool - expectCount int - expectNames []string - }{ - { - name: "select master nodes only", - nodes: []*corev1.Node{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "master-0", - Labels: map[string]string{ControlPlaneNodeLabelSelector: ""}, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "master-1", - Labels: map[string]string{ControlPlaneNodeLabelSelector: ""}, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "worker-0", - Labels: map[string]string{"node-role.kubernetes.io/worker": ""}, - }, - }, - }, - selector: labels.SelectorFromSet(map[string]string{ControlPlaneNodeLabelSelector: ""}), - expectError: false, - expectCount: 2, - expectNames: []string{"master-0", "master-1"}, - }, - { - name: "select arbiter nodes only", - nodes: []*corev1.Node{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "master-0", - Labels: map[string]string{ControlPlaneNodeLabelSelector: ""}, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "arbiter-0", - Labels: map[string]string{ArbiterNodeLabelSelector: ""}, - }, - }, - }, - selector: labels.SelectorFromSet(map[string]string{ArbiterNodeLabelSelector: ""}), - expectError: false, - expectCount: 1, - expectNames: []string{"arbiter-0"}, - }, - { - name: "no matching nodes", - nodes: []*corev1.Node{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "worker-0", - Labels: map[string]string{"node-role.kubernetes.io/worker": ""}, - }, - }, - }, - selector: labels.SelectorFromSet(map[string]string{ControlPlaneNodeLabelSelector: ""}), - expectError: false, - expectCount: 0, - expectNames: []string{}, - }, - { - name: "select all nodes with Everything() selector", - nodes: []*corev1.Node{ - {ObjectMeta: metav1.ObjectMeta{Name: "master-0"}}, - {ObjectMeta: metav1.ObjectMeta{Name: "worker-0"}}, - }, - selector: labels.Everything(), - expectError: false, - expectCount: 2, - expectNames: []string{"master-0", "worker-0"}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Create fake informer with nodes - informer := createFakeNodeInformer(t, tt.nodes) - - // Execute - nodes, err := ListNodesBySelector(informer, tt.selector) - - // Verify - if tt.expectError { - require.Error(t, err) - } else { - require.NoError(t, err) - require.Len(t, nodes, tt.expectCount) - - // Verify node names match - actualNames := make([]string, len(nodes)) - for i, node := range nodes { - actualNames[i] = node.Name - } - require.ElementsMatch(t, tt.expectNames, actualNames) - } - }) - } -} - -func TestListNodesBySelector_NilInformer(t *testing.T) { - selector := labels.SelectorFromSet(map[string]string{ControlPlaneNodeLabelSelector: ""}) - nodes, err := ListNodesBySelector(nil, selector) - - require.Error(t, err) - require.Nil(t, nodes) - require.Contains(t, err.Error(), "informer is nil") -} - // createFakeNodeInformer creates a fake SharedIndexInformer populated with the given nodes func createFakeNodeInformer(t *testing.T, nodes []*corev1.Node) cache.SharedIndexInformer { indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{}) diff --git a/pkg/tnf/after-setup/runner.go b/pkg/tnf/after-setup/runner.go index b85d69a6be..df721fc671 100644 --- a/pkg/tnf/after-setup/runner.go +++ b/pkg/tnf/after-setup/runner.go @@ -24,6 +24,7 @@ import ( "github.com/openshift/cluster-etcd-operator/pkg/tnf/pkg/kubelet" "github.com/openshift/cluster-etcd-operator/pkg/tnf/pkg/tools" "github.com/openshift/library-go/pkg/operator/genericoperatorclient" + "github.com/openshift/library-go/pkg/operator/v1helpers" ) func RunTnfAfterSetup() error { @@ -76,9 +77,10 @@ func RunTnfAfterSetup() error { } if !transitionComplete { - // Pre-transition (bootstrap): wait for setup job - klog.Info("Pre-transition: waiting for setup job to complete") - err = waitForSetupJobCompletion(ctx, kubeClient) + // Pre-transition (bootstrap): wait for transition flag to be set + // The setup job sets this flag when Pacemaker configuration is complete + klog.Info("Pre-transition: waiting for transition to complete") + err = waitForTransitionComplete(ctx, operatorClient) if err != nil { return err } @@ -103,32 +105,26 @@ func RunTnfAfterSetup() error { return nil } -// waitForSetupJobCompletion waits for the setup job to complete (bootstrap phase) -func waitForSetupJobCompletion(ctx context.Context, kubeClient kubernetes.Interface) error { - klog.Info("Waiting for setup job to complete") - setupDone := func(context.Context) (done bool, err error) { - setupJobs, err := kubeClient.BatchV1().Jobs("openshift-etcd").List(ctx, metav1.ListOptions{ - LabelSelector: fmt.Sprintf("app.kubernetes.io/name=%s", tools.JobTypeSetup.GetNameLabelValue()), - }) +// waitForTransitionComplete waits for the ExternalEtcdTransitionCompleted flag to be set (bootstrap phase) +// The setup job sets this flag when Pacemaker configuration is complete +func waitForTransitionComplete(ctx context.Context, operatorClient v1helpers.StaticPodOperatorClient) error { + klog.Info("Waiting for external etcd transition to complete") + transitionDone := func(context.Context) (done bool, err error) { + complete, err := ceohelpers.HasExternalEtcdCompletedTransition(ctx, operatorClient) if err != nil { - klog.Warningf("Failed to list setup jobs: %v", err) - return false, nil - } - if setupJobs.Items == nil || len(setupJobs.Items) != 1 { - klog.V(4).Infof("Expected 1 setup job, got %d", len(setupJobs.Items)) + klog.Warningf("Failed to check transition status: %v", err) return false, nil } - job := setupJobs.Items[0] - if !jobs.IsConditionTrue(job.Status.Conditions, batchv1.JobComplete) { - klog.V(4).Infof("Setup job %s not yet complete", job.Name) + if !complete { + klog.V(4).Info("Transition not yet complete") return false, nil } - klog.Info("Setup job completed successfully") + klog.Info("External etcd transition completed successfully") return true, nil } - err := wait.PollUntilContextTimeout(ctx, tools.JobPollInterval, tools.SetupJobCompletedTimeout, true, setupDone) + err := wait.PollUntilContextTimeout(ctx, tools.JobPollInterval, tools.SetupJobCompletedTimeout, true, transitionDone) if err != nil { - return fmt.Errorf("timed out waiting for setup job to complete: %w", err) + return fmt.Errorf("timed out waiting for transition to complete: %w", err) } return nil } diff --git a/pkg/tnf/fencing/runner.go b/pkg/tnf/fencing/runner.go index 6487eefbe0..715f05f495 100644 --- a/pkg/tnf/fencing/runner.go +++ b/pkg/tnf/fencing/runner.go @@ -4,18 +4,24 @@ import ( "context" "fmt" - batchv1 "k8s.io/api/batch/v1" + operatorv1 "github.com/openshift/api/operator/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/server" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/klog/v2" + "k8s.io/utils/clock" - "github.com/openshift/cluster-etcd-operator/pkg/tnf/pkg/config" + "github.com/openshift/cluster-etcd-operator/pkg/operator" + "github.com/openshift/cluster-etcd-operator/pkg/operator/ceohelpers" + "github.com/openshift/cluster-etcd-operator/pkg/operator/operatorclient" "github.com/openshift/cluster-etcd-operator/pkg/tnf/pkg/jobs" "github.com/openshift/cluster-etcd-operator/pkg/tnf/pkg/pcs" "github.com/openshift/cluster-etcd-operator/pkg/tnf/pkg/tools" + "github.com/openshift/library-go/pkg/operator/genericoperatorclient" + "github.com/openshift/library-go/pkg/operator/v1helpers" ) func RunFencingSetup() error { @@ -47,46 +53,65 @@ func RunFencingSetup() error { klog.Info("Running TNF pacemaker fencing configuration") - klog.Info("Waiting for completed setup job") - setupDone := func(context.Context) (done bool, err error) { - setupJobs, err := kubeClient.BatchV1().Jobs("openshift-etcd").List(ctx, metav1.ListOptions{ - LabelSelector: fmt.Sprintf("app.kubernetes.io/name=%s", tools.JobTypeSetup.GetNameLabelValue()), - }) + // Set up operator client to check transition status + operatorClient, dynamicInformers, err := genericoperatorclient.NewStaticPodOperatorClient( + clock.RealClock{}, + clientConfig, + operatorv1.GroupVersion.WithResource("etcds"), + operatorv1.GroupVersion.WithKind("Etcd"), + operator.ExtractStaticPodOperatorSpec, + operator.ExtractStaticPodOperatorStatus, + ) + if err != nil { + return err + } + dynamicInformers.Start(ctx.Done()) + dynamicInformers.WaitForCacheSync(ctx.Done()) + + // Check if update-setup ConfigMap exists to determine what to wait for + configMaps, err := kubeClient.CoreV1().ConfigMaps(operatorclient.TargetNamespace).List(ctx, metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/component=" + tools.TnfUpdateSetupComponentValue, + }) + if err != nil { + klog.Errorf("Failed to list update-setup ConfigMaps: %v", err) + return fmt.Errorf("failed to list update-setup ConfigMaps: %w", err) + } + + if len(configMaps.Items) > 0 { + // Post-transition: update-setup ConfigMap exists, wait for job to complete + klog.Info("Update-setup ConfigMap found, waiting for update-setup job to complete") + err = waitForUpdateSetupCompletion(ctx, kubeClient) if err != nil { - klog.Warningf("Failed to list setupJobs: %v", err) - return false, nil - } - if setupJobs.Items == nil { - klog.Warningf("Expected 1 job, found none") - return false, nil - } - if len(setupJobs.Items) != 1 { - klog.Warningf("Expected 1 job, found %d", len(setupJobs.Items)) - return false, nil + return err } - job := setupJobs.Items[0] - if !jobs.IsConditionTrue(job.Status.Conditions, batchv1.JobComplete) { - klog.Warningf("Job %s not complete", job.Name) - return false, nil + } else { + // Pre-transition: no ConfigMap, wait for transition + klog.Info("No update-setup ConfigMap found, waiting for transition to complete") + err = waitForTransitionComplete(ctx, operatorClient) + if err != nil { + return err } - klog.Info("Setup job completed successfully") - return true, nil } - err = wait.PollUntilContextTimeout(ctx, tools.JobPollInterval, tools.SetupJobCompletedTimeout, true, setupDone) + + // Get current ready control plane nodes from K8s + nodes, err := ceohelpers.ListReadyNodes(ctx, kubeClient) if err != nil { - klog.Errorf("Timed out waiting for setup job to complete: %v", err) - return err + klog.Errorf("Failed to list ready control plane nodes: %v", err) + return fmt.Errorf("failed to list ready control plane nodes: %w", err) } - klog.Info("Running TNF pacemaker fencing configuration") + if len(nodes) == 0 { + klog.Errorf("No ready control plane nodes found") + return fmt.Errorf("no ready control plane nodes found") + } - // create tnf cluster config - cfg, err := config.GetClusterConfigIgnoreMissingNode(ctx, kubeClient) - if err != nil { - return err + nodeNames := make([]string, len(nodes)) + for i, node := range nodes { + nodeNames[i] = node.Name } - err = pcs.ConfigureFencing(ctx, kubeClient, []string{cfg.NodeName1, cfg.NodeName2}) + klog.Infof("Configuring fencing for nodes: %v", nodeNames) + err = pcs.ConfigureFencing(ctx, kubeClient, nodeNames) if err != nil { return err } @@ -101,3 +126,65 @@ func RunFencingSetup() error { return nil } + +// waitForTransitionComplete waits for the ExternalEtcdTransitionCompleted flag to be set (bootstrap phase). +func waitForTransitionComplete(ctx context.Context, operatorClient v1helpers.StaticPodOperatorClient) error { + klog.Info("Waiting for external etcd transition to complete") + transitionDone := func(context.Context) (done bool, err error) { + complete, err := ceohelpers.HasExternalEtcdCompletedTransition(ctx, operatorClient) + if err != nil { + klog.Warningf("Failed to check transition status: %v", err) + return false, nil + } + if !complete { + klog.V(4).Info("Transition not yet complete") + return false, nil + } + klog.Info("External etcd transition completed successfully") + return true, nil + } + err := wait.PollUntilContextTimeout(ctx, tools.JobPollInterval, tools.SetupJobCompletedTimeout, true, transitionDone) + if err != nil { + klog.Errorf("Timed out waiting for transition to complete: %v", err) + return fmt.Errorf("timed out waiting for transition to complete: %w", err) + } + return nil +} + +// waitForUpdateSetupCompletion waits for the update-setup job to complete (post-transition phase). +func waitForUpdateSetupCompletion(ctx context.Context, kubeClient kubernetes.Interface) error { + klog.Info("Waiting for update-setup job to complete") + updateSetupDone := func(context.Context) (done bool, err error) { + updateSetupJobName := tools.JobTypeUpdateSetup.GetJobName(nil) + job, err := kubeClient.BatchV1().Jobs(operatorclient.TargetNamespace).Get(ctx, updateSetupJobName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + // No job exists - nothing to wait for + klog.V(4).Info("Update-setup job not found, proceeding") + return true, nil + } + klog.Warningf("Failed to get update-setup job: %v", err) + return false, nil + } + + if jobs.IsComplete(*job) { + klog.Info("Update-setup job completed successfully") + return true, nil + } + + if jobs.IsFailed(*job) { + klog.Warningf("Update-setup job failed, but proceeding with fencing configuration") + return true, nil + } + + klog.V(4).Info("Update-setup job still running") + return false, nil + } + + err := wait.PollUntilContextTimeout(ctx, tools.JobPollInterval, tools.UpdateSetupJobCompletedTimeout, true, updateSetupDone) + if err != nil { + klog.Errorf("Timed out waiting for update-setup job to complete: %v", err) + return fmt.Errorf("timed out waiting for update-setup job to complete: %w", err) + } + return nil +} diff --git a/pkg/tnf/operator/lifecycle_helpers.go b/pkg/tnf/operator/lifecycle_helpers.go index 5769b831b8..63f53c0d23 100644 --- a/pkg/tnf/operator/lifecycle_helpers.go +++ b/pkg/tnf/operator/lifecycle_helpers.go @@ -56,12 +56,3 @@ func (c *PacemakerLifecycleManager) getPacemakerNodesWithCR() (map[string]string return pmNodes, pacemakerCR, nil } - -// getNodeNames extracts node names from a list of nodes. -func getNodeNames(nodes []*corev1.Node) []string { - names := make([]string, len(nodes)) - for i, node := range nodes { - names[i] = node.Name - } - return names -} diff --git a/pkg/tnf/operator/lifecycle_job_controllers.go b/pkg/tnf/operator/lifecycle_job_controllers.go index 9ae65930d4..94f23e0642 100644 --- a/pkg/tnf/operator/lifecycle_job_controllers.go +++ b/pkg/tnf/operator/lifecycle_job_controllers.go @@ -103,8 +103,9 @@ func (c *PacemakerLifecycleManager) StartJobControllers(ctx context.Context) err klog.V(2).Infof("Both control plane nodes ready - starting initial job controllers with retry") return c.retryInitialTransitionOrDegrade(ctx, k8sNodes) } else { - // Post-transition: idempotent call (handles operator restart, single-node case) - // Uses mutex to prevent concurrent startup attempts + // Post-transition: ensure controllers running for all ready nodes + // Called on every sync to handle missed node add/delete/update events + // RunTNFJobController has internal duplicate prevention // Defensive check: ensure we have at least one control plane node if len(k8sNodes) == 0 { @@ -120,26 +121,12 @@ func (c *PacemakerLifecycleManager) StartJobControllers(ctx context.Context) err } } - // Check if controllers have already been started - c.startJobControllersMu.Lock() - if c.jobControllersStarted { - c.startJobControllersMu.Unlock() - klog.V(4).Infof("Job controllers already started, skipping") - return nil - } - c.startJobControllersMu.Unlock() - - klog.V(2).Infof("Transition complete - ensuring job controllers running with %d ready control plane nodes", len(k8sNodes)) + klog.V(4).Infof("Transition complete - ensuring job controllers running for %d ready control plane nodes", len(k8sNodes)) err = c.startJobControllersWithLock(ctx, k8sNodes) if err != nil { return err } - // Mark as started on success - c.startJobControllersMu.Lock() - c.jobControllersStarted = true - c.startJobControllersMu.Unlock() - klog.Infof("Job controllers successfully started") return nil } } @@ -223,15 +210,18 @@ func startTnfJobcontrollers( etcdInformer operatorv1informers.EtcdInformer, lifecycleManager *PacemakerLifecycleManager) error { - // Check if setup job already exists and is complete (operator restart scenario) + // Check if transition already complete (operator restart scenario) // If so, skip bootstrap flow and just ensure controllers are running - setupJobName := tools.JobTypeSetup.GetJobName(nil) - setupJob, err := kubeClient.BatchV1().Jobs(operatorclient.TargetNamespace).Get(ctx, setupJobName, metav1.GetOptions{}) - if err == nil && jobs.IsComplete(*setupJob) { - klog.Infof("Setup job already complete - skipping bootstrap flow, ensuring controllers are running") + transitionComplete, err := ceohelpers.HasExternalEtcdCompletedTransition(ctx, operatorClient) + if err != nil { + klog.Warningf("Failed to check transition status: %v - proceeding with bootstrap flow", err) + } + + if transitionComplete { + klog.Infof("Transition already complete - skipping bootstrap flow, ensuring controllers are running") // Just start the controllers without going through bootstrap/setup again - // This prevents recreating jobs and racing with reconciliation + // This prevents recreating setup job and racing with reconciliation for _, node := range nodeList { nodeTarget := &jobs.NodeTarget{Name: node.Name, UID: string(node.UID)} jobs.RunTNFJobController(ctx, tools.JobTypeAuth, nodeTarget, nil, nil, 3, controllerContext, operatorClient, kubeClient, kubeInformersForNamespaces, jobs.DefaultConditions) @@ -242,7 +232,8 @@ func startTnfJobcontrollers( return lifecycleManager.getActivePacemakerNodes() } - jobs.RunTNFJobController(ctx, tools.JobTypeSetup, nil, clusterWideTargetNodesFunc, nil, 3, controllerContext, operatorClient, kubeClient, kubeInformersForNamespaces, jobs.AllConditions) + // DO NOT start setup job controller - transition already complete, setup job is one-time only + // Day 2 changes are handled by update-setup job via ReconcilePacemakerConfig fencingJobConfigFunc := createFencingJobConfigFunc(nodeList, kubeInformersForNamespaces) jobs.RunTNFJobController(ctx, tools.JobTypeFencing, nil, clusterWideTargetNodesFunc, fencingJobConfigFunc, 3, controllerContext, operatorClient, kubeClient, kubeInformersForNamespaces, jobs.DefaultConditions) @@ -256,12 +247,9 @@ func startTnfJobcontrollers( jobs.RunTNFJobController(ctx, tools.JobTypeUpdateSetup, nil, clusterWideTargetNodesFunc, nil, 3, controllerContext, operatorClient, kubeClient, kubeInformersForNamespaces, jobs.DefaultConditions) } - klog.Infof("Controllers running for existing completed jobs") + klog.V(4).Infof("Controllers running (post-transition)") return nil } - if err != nil && !apierrors.IsNotFound(err) { - klog.Warningf("Failed to check setup job status: %v - proceeding with bootstrap flow", err) - } klog.Infof("Running TNF setup procedure. Waiting for etcd bootstrap to complete") diff --git a/pkg/tnf/operator/lifecycle_manager.go b/pkg/tnf/operator/lifecycle_manager.go index b53bc2b44a..d209c93163 100644 --- a/pkg/tnf/operator/lifecycle_manager.go +++ b/pkg/tnf/operator/lifecycle_manager.go @@ -59,8 +59,6 @@ type PacemakerLifecycleManager struct { // Job controller startup protection: prevents concurrent StartJobControllers calls startJobControllersMu sync.Mutex - // Track if job controllers have been started (set once, never reset) - jobControllersStarted bool // Lifecycle context for goroutines spawned by event handlers lifecycleCtx context.Context diff --git a/pkg/tnf/operator/lifecycle_reconciliation.go b/pkg/tnf/operator/lifecycle_reconciliation.go index bebfc535ac..7be2056f04 100644 --- a/pkg/tnf/operator/lifecycle_reconciliation.go +++ b/pkg/tnf/operator/lifecycle_reconciliation.go @@ -148,7 +148,7 @@ func (c *PacemakerLifecycleManager) ReconcilePacemakerConfig(ctx context.Context needsReconciliation = true if reason == "" { // Format node names for better readability - k8sNodeNames := getNodeNames(k8sNodes) + k8sNodeNames := tools.GetNodeNames(k8sNodes) pmNodeNames := make([]string, 0, len(pacemakerNodes)) for nodeName := range pacemakerNodes { pmNodeNames = append(pmNodeNames, nodeName) @@ -221,7 +221,7 @@ func (c *PacemakerLifecycleManager) ReconcilePacemakerConfig(ctx context.Context return fmt.Errorf("no nodes in both K8s and pacemaker - manual intervention may be required") } validTargetNodes = intersection - klog.Infof("Found %d nodes in intersection (K8s ∩ pacemaker): %v", len(validTargetNodes), getNodeNames(validTargetNodes)) + klog.Infof("Found %d nodes in intersection (K8s ∩ pacemaker): %v", len(validTargetNodes), tools.GetNodeNames(validTargetNodes)) } else { // CR unavailable/stale - try nodes sequentially to discover which has pacemaker running // Use stopped job to determine which node we already tried (to avoid retrying same node) @@ -229,7 +229,7 @@ func (c *PacemakerLifecycleManager) ReconcilePacemakerConfig(ctx context.Context if len(validTargetNodes) == 0 { return fmt.Errorf("no valid nodes for discovery - all nodes have been tried and failed") } - klog.Infof("Discovery mode (no actionable CR): will try nodes %v", getNodeNames(validTargetNodes)) + klog.Infof("Discovery mode (no actionable CR): will try nodes %v", tools.GetNodeNames(validTargetNodes)) } // Create function that calculates valid target nodes @@ -332,7 +332,7 @@ func (c *PacemakerLifecycleManager) getActivePacemakerNodes() ([]*corev1.Node, e // CR exists and is fresh - use intersection logic (K8s ∩ pacemaker) intersection := c.getIntersection(readyNodes, pacemakerNodes) if len(intersection) > 0 { - klog.V(2).Infof("Valid targets (intersection, ready only): %v", getNodeNames(intersection)) + klog.V(2).Infof("Valid targets (intersection, ready only): %v", tools.GetNodeNames(intersection)) return intersection, nil } // No intersection - fall through to using all ready nodes @@ -346,7 +346,7 @@ func (c *PacemakerLifecycleManager) getActivePacemakerNodes() ([]*corev1.Node, e } // CR doesn't exist, is stale, or no intersection - use all ready nodes - klog.V(2).Infof("Valid targets (all ready nodes): %v", getNodeNames(readyNodes)) + klog.V(2).Infof("Valid targets (all ready nodes): %v", tools.GetNodeNames(readyNodes)) return readyNodes, nil } @@ -580,7 +580,7 @@ func updateSetup( } klog.Infof("Generation %d: Will try nodes %v (desired state: %v)", - generation, getNodeNames(validTargetNodes), getNodeNames(allK8sNodes)) + generation, tools.GetNodeNames(validTargetNodes), tools.GetNodeNames(allK8sNodes)) cmName := fmt.Sprintf("tnf-update-setup-%d", generation) @@ -650,7 +650,7 @@ func updateSetup( // Restart update-setup job with new information (deletes existing job immediately, starts controller) // Controller will manage job lifecycle via sync loop and retry across valid nodes // Note: Auth and after-setup jobs are managed by separate background controllers (lifecycle_job_controllers.go) - klog.Infof("Starting update-setup job controller for generation %d with %d initial valid target(s): %v", generation, len(validTargetNodes), getNodeNames(validTargetNodes)) + klog.Infof("Starting update-setup job controller for generation %d with %d initial valid target(s): %v", generation, len(validTargetNodes), tools.GetNodeNames(validTargetNodes)) // Create jobConfigFunc that captures config for drift detection jobConfigFunc := func() (string, error) { diff --git a/pkg/tnf/operator/starter.go b/pkg/tnf/operator/starter.go index 1ef13823b2..928c9178aa 100644 --- a/pkg/tnf/operator/starter.go +++ b/pkg/tnf/operator/starter.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "sort" + "strconv" "time" configv1informers "github.com/openshift/client-go/config/informers/externalversions/config/v1" @@ -248,17 +249,39 @@ func runPacemakerControllers(ctx context.Context, controllerContext *controllerc } // createFencingJobConfigFunc creates a JobConfigFunc for the fencing job. -// Returns a function that captures node UIDs and fencing secret UIDs for drift detection. +// Returns a function that captures update-setup generation, node UIDs, and fencing secret UIDs for drift detection. +// Drift triggers fencing job restart: +// - Generation change: membership changed (node add/remove) +// - Secret UID change: BMC credentials rotated +// - Node UID change: node replaced (same name, different UID) func createFencingJobConfigFunc(nodeList []*corev1.Node, kubeInformersForNamespaces v1helpers.KubeInformersForNamespaces) jobs.JobConfigFunc { return func() (string, error) { - // Collect node UIDs + // Get latest update-setup ConfigMap generation (detects membership changes) + configMapsLister := kubeInformersForNamespaces.InformersFor(operatorclient.TargetNamespace).Core().V1().ConfigMaps().Lister() + configMaps, err := configMapsLister.List(labels.SelectorFromSet(labels.Set{ + "app.kubernetes.io/component": tools.TnfUpdateSetupComponentValue, + })) + if err != nil { + return "", fmt.Errorf("failed to list update-setup ConfigMaps: %w", err) + } + + var latestGeneration int64 = 0 + for _, cm := range configMaps { + genStr := cm.Data["generation"] + gen, _ := strconv.ParseInt(genStr, 10, 64) + if gen > latestGeneration { + latestGeneration = gen + } + } + + // Collect node UIDs (detects node replacements) nodeUIDs := make([]string, len(nodeList)) for i, node := range nodeList { nodeUIDs[i] = string(node.UID) } sort.Strings(nodeUIDs) - // Collect fencing secret UIDs from informer + // Collect fencing secret UIDs from informer (detects credential rotation) secretsLister := kubeInformersForNamespaces.InformersFor(operatorclient.TargetNamespace).Core().V1().Secrets().Lister() allSecrets, err := secretsLister.List(labels.Everything()) if err != nil { @@ -273,8 +296,9 @@ func createFencingJobConfigFunc(nodeList []*corev1.Node, kubeInformersForNamespa } sort.Strings(secretUIDs) - // Create config map with both node and secret UIDs + // Create config with generation + UIDs config := map[string]interface{}{ + "generation": latestGeneration, "nodeUIDs": nodeUIDs, "secretUIDs": secretUIDs, } diff --git a/pkg/tnf/pkg/jobs/tnf.go b/pkg/tnf/pkg/jobs/tnf.go index e0f13c59b0..a71e33a9d9 100644 --- a/pkg/tnf/pkg/jobs/tnf.go +++ b/pkg/tnf/pkg/jobs/tnf.go @@ -363,14 +363,14 @@ func RunTNFJobController(ctx context.Context, jobType tools.JobType, nodeTarget runningControllersMutex.Lock() if runningControllers[controllerKey] { runningControllersMutex.Unlock() - klog.Infof("Two Node Fencing job controller for command %q on node %q is already running, skipping duplicate start", jobType.GetSubCommand(), nodeNameForLogs) + klog.V(4).Infof("Two Node Fencing job controller for command %q on node %q is already running, skipping duplicate start", jobType.GetSubCommand(), nodeNameForLogs) return } // Mark this controller as running runningControllers[controllerKey] = true runningControllersMutex.Unlock() - klog.Infof("starting Two Node Fencing job controller for command %q on node %q", jobType.GetSubCommand(), nodeNameForLogs) + klog.Infof("Starting Two Node Fencing job controller for command %q on node %q", jobType.GetSubCommand(), nodeNameForLogs) tnfJobController := NewJobController( jobType.GetJobName(jobNodeName), bindata.MustAsset("tnfdeployment/job.yaml"), diff --git a/pkg/tnf/pkg/tools/jobs.go b/pkg/tnf/pkg/tools/jobs.go index 9822cdff19..4c8ef9b227 100644 --- a/pkg/tnf/pkg/tools/jobs.go +++ b/pkg/tnf/pkg/tools/jobs.go @@ -21,6 +21,10 @@ const ( UpdateSetupJobCompletedTimeout = 20 * time.Minute AllCompletedTimeout = 30 * time.Minute FencingJobCompletedTimeout = 25 * time.Minute + + // TnfUpdateSetupComponentValue is app.kubernetes.io/component for update-setup snapshot ConfigMaps. + // CEO creates ConfigMaps with this value; the in-cluster update-setup job lists by the same selector. + TnfUpdateSetupComponentValue = "tnf-update-setup" ) // JobType represent the different jobs we run, with some methods needed diff --git a/pkg/tnf/pkg/tools/tnf_labels.go b/pkg/tnf/pkg/tools/tnf_labels.go deleted file mode 100644 index f6029b64fc..0000000000 --- a/pkg/tnf/pkg/tools/tnf_labels.go +++ /dev/null @@ -1,5 +0,0 @@ -package tools - -// TnfUpdateSetupComponentValue is app.kubernetes.io/component for update-setup snapshot ConfigMaps. -// CEO creates ConfigMaps with this value; the in-cluster update-setup job lists by the same selector. -const TnfUpdateSetupComponentValue = "tnf-update-setup" diff --git a/pkg/tnf/setup/runner.go b/pkg/tnf/setup/runner.go index 7928626468..8f0e240bae 100644 --- a/pkg/tnf/setup/runner.go +++ b/pkg/tnf/setup/runner.go @@ -122,14 +122,7 @@ func RunTnfSetup() error { time.Sleep(5 * time.Second) } - // Signal CEO that TNF setup is ready for etcd container removal - err = etcd.RemoveStaticContainer(ctx, operatorClient) - if err != nil { - return err - } - // register pacemaker alert agents for fencing taint/untaint - // this is done last so core functionality (etcd resource, stonith) works even if alerts fail // alerts are optional - don't fail setup if they can't be configured err = pcs.ConfigureAlerts(ctx) if err != nil { @@ -138,6 +131,13 @@ func RunTnfSetup() error { klog.Infof("Successfully configured pacemaker alerts") } + // Signal CEO that TNF setup is ready for etcd container removal + // After this point, the transition flag is set and other components can proceed + err = etcd.RemoveStaticContainer(ctx, operatorClient) + if err != nil { + return err + } + // get pcs cib cib, err := pcs.GetCIB(ctx) if err != nil { diff --git a/pkg/tnf/update-setup/runner.go b/pkg/tnf/update-setup/runner.go index 4898a502a8..ce1c037657 100644 --- a/pkg/tnf/update-setup/runner.go +++ b/pkg/tnf/update-setup/runner.go @@ -137,7 +137,7 @@ func RunTnfUpdateSetup() error { } klog.Infof("Running update-setup for %d-node cluster", len(capturedNodes)) - klog.V(2).Infof("Captured nodes: %v", getNodeNames(capturedNodes)) + klog.V(2).Infof("Captured nodes: %v", tools.GetNodeNames(capturedNodes)) cfg, err := buildClusterConfigFromNodeList(capturedNodes) if err != nil { @@ -235,17 +235,9 @@ func RunTnfUpdateSetup() error { } } - // Reconcile STONITH after membership changes (other node first, then current). - if len(nodesToRemove) > 0 || len(nodesToAdd) > 0 { - fencingNodes := make([]string, 0, 2) - if otherNodeName != "" { - fencingNodes = append(fencingNodes, otherNodeName) - } - fencingNodes = append(fencingNodes, currentNodeName) - if err = pcs.ConfigureFencing(ctx, kubeClient, fencingNodes); err != nil { - return fmt.Errorf("failed to configure fencing: %w", err) - } - } + // Fencing configuration is now handled by the fencing job (not inline). + // The fencing job will detect the generation change via jobConfigFunc and restart, + // waiting for this update-setup job to complete before running ConfigureFencing. // Update etcd resource with current node IPs after any membership changes // This runs once regardless of whether we removed, added, or both @@ -547,14 +539,6 @@ func decodeNodeList(data string) ([]*corev1.Node, error) { return nodes, nil } -func getNodeNames(nodes []*corev1.Node) []string { - names := make([]string, len(nodes)) - for i, node := range nodes { - names[i] = node.Name - } - return names -} - func buildClusterConfigFromNodeList(nodes []*corev1.Node) (config.ClusterConfig, error) { cfg := config.ClusterConfig{}