diff --git a/hack/test-transition-events.sh b/hack/test-transition-events.sh new file mode 100755 index 0000000000..0c31b1cf0c --- /dev/null +++ b/hack/test-transition-events.sh @@ -0,0 +1,64 @@ +#!/usr/bin/bash +# Test script to validate transition event format on a live cluster. +# Creates mock events identical to what our code produces, then verifies +# they appear correctly in `oc get events`. +# +# Usage: ./hack/test-transition-events.sh +# Requires: oc/kubectl with cluster-admin access + +set -euo pipefail + +NAMESPACE="openshift-etcd" +TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + +events=( + "EtcdTransitionAuthCompleted|PCS authentication completed on all nodes" + "EtcdTransitionClusterConfigured|Pacemaker cluster configured successfully" + "EtcdTransitionFencingConfigured|STONITH fencing configured successfully" + "EtcdTransitionEtcdResourceCreated|Pacemaker etcd resource agent (podman-etcd) configured" + "EtcdTransitionConstraintsConfigured|Pacemaker ordering and colocation constraints configured" + "EtcdTransitionStarted|Etcd transition from CEO-controlled to pacemaker-controlled has started" + "EtcdTransitionWaitingForRemoval|Waiting for CEO to remove static etcd container from all nodes" + "EtcdTransitionStaticContainerRemoved|Static etcd container removed from all nodes, revision is stable" + "EtcdTransitionCompleted|Etcd transition to pacemaker-controlled has completed" +) + +echo "Creating ${#events[@]} test transition events in ${NAMESPACE}..." + +for entry in "${events[@]}"; do + reason="${entry%%|*}" + message="${entry#*|}" + name="test-${reason,,}-$(date +%s%N)" + + oc apply -f - < StatusStalenessThreshold { - // Unknown status - don't update previous - // Use absolute timestamp (stable) for event deduplication. - return &HealthStatus{ - OverallStatus: statusUnknown, - Warnings: []string{}, - Errors: []string{fmt.Sprintf("Pacemaker status is stale (last updated: %s)", crLastUpdated.Format(time.RFC3339))}, - }, previous, nil - } - - // Check if lastUpdated timestamp has changed since last sync. - // If unchanged, skip processing to avoid redundant work on timer-triggered syncs. - // Use previous.CRLastUpdated for comparison (only set for non-Unknown status). - if previous != nil && !previous.CRLastUpdated.IsZero() && crLastUpdated.Equal(previous.CRLastUpdated) { - klog.V(4).Infof("Skipping sync: lastUpdated timestamp unchanged (%v)", crLastUpdated) + return unknownHealthStatus("Failed to convert cached item to PacemakerCluster"), previous, nil + } + + // Staleness grows over time and must bypass the timestamp-unchanged optimization. + isStale := pacemakerCR.Status.LastUpdated.IsZero() || + time.Since(pacemakerCR.Status.LastUpdated.Time) > StatusStalenessThreshold + + if !isStale && previous != nil && !previous.CRLastUpdated.IsZero() && + pacemakerCR.Status.LastUpdated.Time.Equal(previous.CRLastUpdated) { + klog.V(4).Infof("Skipping sync: lastUpdated timestamp unchanged (%v)", pacemakerCR.Status.LastUpdated.Time) return nil, nil, nil } - // Build health status from the CRD status fields - currentStatus := c.buildHealthStatusFromCR(pacemakerCR) - currentStatus.CRLastUpdated = crLastUpdated + currentStatus := BuildHealthStatusFromCR(pacemakerCR) - // Only update previous for non-Unknown status (preserves last valid for grace period) - if currentStatus.OverallStatus != statusUnknown { + if currentStatus.OverallStatus != StatusUnknown { c.previousMu.Lock() c.previous = currentStatus c.previousMu.Unlock() @@ -334,36 +291,44 @@ func (c *HealthCheck) getPacemakerStatus(ctx context.Context) (*HealthStatus, *H return currentStatus, previous, nil } -// buildHealthStatusFromCR builds a HealthStatus from the PacemakerCluster CR status fields -// Note: This function assumes Status is not nil (checked by caller in getPacemakerStatus) -func (c *HealthCheck) buildHealthStatusFromCR(pacemakerStatus *pacmkrv1.PacemakerCluster) *HealthStatus { +func unknownHealthStatus(msg string) *HealthStatus { + return &HealthStatus{ + OverallStatus: StatusUnknown, + Warnings: []string{}, + Errors: []string{msg}, + } +} + +// BuildHealthStatusFromCR evaluates a PacemakerCluster CR and returns its complete health status. +func BuildHealthStatusFromCR(cr *pacmkrv1.PacemakerCluster) *HealthStatus { status := &HealthStatus{ - OverallStatus: statusUnknown, + OverallStatus: StatusUnknown, Warnings: []string{}, Errors: []string{}, } - // Defensive check: this should not happen as getPacemakerStatus checks for unpopulated Status - if pacemakerStatus == nil || pacemakerStatus.Status.LastUpdated.IsZero() { - klog.Errorf("buildHealthStatusFromCR called with nil PacemakerCluster or unpopulated Status") - status.Errors = append(status.Errors, "Internal error: unpopulated Status in buildHealthStatusFromCR") + if cr == nil || cr.Status.LastUpdated.IsZero() { + status.Errors = append(status.Errors, "PacemakerCluster CR has no status populated") return status } - // Check cluster-level configuration issues FIRST (node count, maintenance mode) - // These are often root causes (e.g., "excessive nodes" causes resource failures on extra node) - c.checkClusterConditions(pacemakerStatus, status) + status.CRLastUpdated = cr.Status.LastUpdated.Time - // Check node statuses (node-level conditions and resource health) - c.checkNodeStatuses(pacemakerStatus, status) + if time.Since(cr.Status.LastUpdated.Time) > StatusStalenessThreshold { + status.Errors = append(status.Errors, fmt.Sprintf("Pacemaker status is stale (last updated: %s)", + cr.Status.LastUpdated.Time.Format(time.RFC3339))) + return status + } + + checkClusterConditions(cr, status) + checkNodeStatuses(cr, status) - // Determine overall status: errors > warnings > healthy if len(status.Errors) > 0 { - status.OverallStatus = statusError + status.OverallStatus = StatusError } else if len(status.Warnings) > 0 { - status.OverallStatus = statusWarning + status.OverallStatus = StatusWarning } else { - status.OverallStatus = statusHealthy + status.OverallStatus = StatusHealthy } return status @@ -372,34 +337,29 @@ func (c *HealthCheck) buildHealthStatusFromCR(pacemakerStatus *pacmkrv1.Pacemake // checkClusterConditions checks cluster-level configuration issues (node count, maintenance mode). // These are ALWAYS reported regardless of node-level errors, as they often represent root causes. // For example, "excessive nodes" causes resource failures on the extra node. -func (c *HealthCheck) checkClusterConditions(pacemakerStatus *pacmkrv1.PacemakerCluster, status *HealthStatus) { - conditions := pacemakerStatus.Status.Conditions +func checkClusterConditions(cr *pacmkrv1.PacemakerCluster, status *HealthStatus) { + conditions := cr.Status.Conditions if len(conditions) == 0 { - // Missing cluster conditions means we can't verify cluster health configuration klog.V(2).Infof("No cluster conditions present in status") status.Errors = append(status.Errors, "No cluster conditions available") return } - // Always check cluster-level configuration issues - these are root causes - clusterIssues := c.getClusterConditionIssues(conditions, pacemakerStatus) - - // Add cluster-level error if there are configuration issues - if len(clusterIssues) > 0 { - status.Errors = append(status.Errors, fmt.Sprintf(msgClusterUnhealthy, strings.Join(clusterIssues, ", "))) + if issues := getClusterConditionIssues(conditions, cr); len(issues) > 0 { + status.Errors = append(status.Errors, fmt.Sprintf(msgClusterUnhealthy, strings.Join(issues, ", "))) } } // getClusterConditionIssues returns specific issues from cluster-level conditions (non-summary conditions) -func (c *HealthCheck) getClusterConditionIssues(conditions []metav1.Condition, pacemakerStatus *pacmkrv1.PacemakerCluster) []string { +func getClusterConditionIssues(conditions []metav1.Condition, cr *pacmkrv1.PacemakerCluster) []string { var issues []string // Check NodeCountAsExpected condition nodeCountCondition := FindCondition(conditions, pacmkrv1.ClusterNodeCountAsExpectedConditionType) if nodeCountCondition != nil && nodeCountCondition.Status != metav1.ConditionTrue { nodeCount := 0 - if pacemakerStatus.Status.Nodes != nil { - nodeCount = len(*pacemakerStatus.Status.Nodes) + if cr.Status.Nodes != nil { + nodeCount = len(*cr.Status.Nodes) } switch nodeCountCondition.Reason { case pacmkrv1.ClusterNodeCountAsExpectedReasonInsufficientNodes: @@ -419,15 +379,15 @@ func (c *HealthCheck) getClusterConditionIssues(conditions []metav1.Condition, p } // checkNodeStatuses checks if all nodes have healthy conditions and resources -func (c *HealthCheck) checkNodeStatuses(pacemakerStatus *pacmkrv1.PacemakerCluster, status *HealthStatus) { +func checkNodeStatuses(cr *pacmkrv1.PacemakerCluster, status *HealthStatus) { // Nil-guard for Nodes field - missing node data is an error (cannot verify cluster health) - if pacemakerStatus.Status.Nodes == nil { + if cr.Status.Nodes == nil { klog.V(2).Infof("Pacemaker.Status.Nodes is nil, cannot determine node status") status.Errors = append(status.Errors, msgNoNodesFound) return } - nodes := *pacemakerStatus.Status.Nodes + nodes := *cr.Status.Nodes // Empty nodes list is also an error - cannot verify cluster health without node data if len(nodes) == 0 { @@ -438,58 +398,39 @@ func (c *HealthCheck) checkNodeStatuses(pacemakerStatus *pacmkrv1.PacemakerClust // Check each node's conditions and resource health (consolidated into single error per node) for _, node := range nodes { - c.checkNodeConditions(node, status) + checkNodeConditions(node, status) } } // checkNodeConditions checks the conditions of a single node and its resources, // routing issues to errors or warnings based on severity. // Errors degrade the operator; warnings are informational (e.g., fencing redundancy lost). -func (c *HealthCheck) checkNodeConditions(node pacmkrv1.PacemakerClusterNodeStatus, status *HealthStatus) { +func checkNodeConditions(node pacmkrv1.PacemakerClusterNodeStatus, status *HealthStatus) { conditions := node.Conditions if len(conditions) == 0 { klog.V(2).Infof("Node %s has no conditions", node.NodeName) return } - // Check Online condition - this is critical for degraded status - onlineCondition := FindCondition(conditions, pacmkrv1.NodeOnlineConditionType) - if onlineCondition != nil && onlineCondition.Status != metav1.ConditionTrue { + if cond := FindCondition(conditions, pacmkrv1.NodeOnlineConditionType); cond != nil && cond.Status != metav1.ConditionTrue { status.Errors = append(status.Errors, fmt.Sprintf(msgNodeOffline, node.NodeName)) - return // If node is offline, other conditions don't matter + return } - // Always check for fencing warnings - these are independent of overall node health. - // Fencing redundancy degraded (FencingHealthy=False but FencingAvailable=True) is a warning - // that should be reported even when the node is otherwise healthy. - fencingWarnings := c.getFencingWarnings(conditions) + fencingWarnings := getFencingWarnings(conditions) for _, warning := range fencingWarnings { status.Warnings = append(status.Warnings, fmt.Sprintf("%s: %s", node.NodeName, warning)) } - // Check overall node Healthy condition healthyCondition := FindCondition(conditions, pacmkrv1.NodeHealthyConditionType) if healthyCondition == nil || healthyCondition.Status == metav1.ConditionTrue { - return // Node is healthy (except for warnings already captured above) + return } - // Node is unhealthy - collect errors (warnings already captured above) - nodeErrors := c.getNodeConditionErrors(conditions) - - // Collect resource errors (all resource issues are currently errors) - resourceErrors := c.getNodeResourceSummaries(node) - - // Combine all errors - allErrors := append(nodeErrors, resourceErrors...) - - // If there are errors, build consolidated error message + allErrors := slices.Concat(getNodeConditionErrors(conditions), getNodeResourceSummaries(node)) if len(allErrors) > 0 { status.Errors = append(status.Errors, fmt.Sprintf(msgNodeUnhealthy, node.NodeName, strings.Join(allErrors, ", "))) - } - - // If no specific issues found but node is unhealthy, use generic message. - // Skip if we already captured fencing warnings - those explain the unhealthy state. - if len(allErrors) == 0 && len(fencingWarnings) == 0 { + } else if len(fencingWarnings) == 0 { status.Errors = append(status.Errors, fmt.Sprintf(msgNodeUnhealthy, node.NodeName, healthyCondition.Message)) } } @@ -497,11 +438,9 @@ func (c *HealthCheck) checkNodeConditions(node pacmkrv1.PacemakerClusterNodeStat // getFencingWarnings returns warnings about degraded fencing redundancy. // This is checked independently of overall node health because fencing redundancy // degradation should be reported even when the node is otherwise healthy. -func (c *HealthCheck) getFencingWarnings(conditions []metav1.Condition) []string { +func getFencingWarnings(conditions []metav1.Condition) []string { var warnings []string - // Check FencingHealthy - if false but FencingAvailable is true, fencing redundancy is degraded (warning) - // This is a warning because the node CAN still be fenced, just with reduced redundancy. fencingAvailableCondition := FindCondition(conditions, pacmkrv1.NodeFencingAvailableConditionType) fencingHealthyCondition := FindCondition(conditions, pacmkrv1.NodeFencingHealthyConditionType) @@ -538,7 +477,7 @@ var nodeConditionChecks = []nodeConditionCheck{ // getNodeConditionErrors returns errors from node-level conditions. // Errors require immediate action and cause the operator to degrade. -func (c *HealthCheck) getNodeConditionErrors(conditions []metav1.Condition) []string { +func getNodeConditionErrors(conditions []metav1.Condition) []string { var errors []string for _, check := range nodeConditionChecks { @@ -557,50 +496,34 @@ func (c *HealthCheck) getNodeConditionErrors(conditions []metav1.Condition) []st // getNodeResourceSummaries returns summaries of unhealthy resources on a node. // Each summary includes the resource name and its specific issue. -func (c *HealthCheck) getNodeResourceSummaries(node pacmkrv1.PacemakerClusterNodeStatus) []string { +func getNodeResourceSummaries(node pacmkrv1.PacemakerClusterNodeStatus) []string { var summaries []string for _, resource := range node.Resources { - healthyCondition := FindCondition(resource.Conditions, pacmkrv1.ResourceHealthyConditionType) - if healthyCondition != nil && healthyCondition.Status != metav1.ConditionTrue { - // Get specific reason for this resource - reason := c.getResourceIssue(resource.Conditions) - summaries = append(summaries, fmt.Sprintf("%s %s", resource.Name, reason)) + if cond := FindCondition(resource.Conditions, pacmkrv1.ResourceHealthyConditionType); cond != nil && cond.Status != metav1.ConditionTrue { + summaries = append(summaries, fmt.Sprintf("%s %s", resource.Name, getResourceIssue(resource.Conditions))) } } return summaries } -// getResourceIssue returns a specific issue description for an unhealthy resource. -// -// All resource-level issues are treated as errors. The Active=True with Started=False state -// (anomalous transitional state) could theoretically be a warning since it might self-resolve, -// but routing resource issues to warnings vs errors would require significant refactoring. -// Additionally: (1) this state is rare and brief and (2) etcd not running causes API server -// degradation anyway, so treating it as an error is appropriate. -func (c *HealthCheck) getResourceIssue(conditions []metav1.Condition) string { - // Check for specific failure conditions (prioritized by severity) - operationalCondition := FindCondition(conditions, pacmkrv1.ResourceOperationalConditionType) - if operationalCondition != nil && operationalCondition.Status != metav1.ConditionTrue { - return "has failed" - } - - startedCondition := FindCondition(conditions, pacmkrv1.ResourceStartedConditionType) - if startedCondition != nil && startedCondition.Status != metav1.ConditionTrue { - return "is stopped" - } - - activeCondition := FindCondition(conditions, pacmkrv1.ResourceActiveConditionType) - if activeCondition != nil && activeCondition.Status != metav1.ConditionTrue { - return "is not active" - } - - managedCondition := FindCondition(conditions, pacmkrv1.ResourceManagedConditionType) - if managedCondition != nil && managedCondition.Status != metav1.ConditionTrue { - return "is unmanaged" +// getResourceIssue returns a human-readable issue for an unhealthy resource, prioritized by severity. +func getResourceIssue(conditions []metav1.Condition) string { + checks := []struct { + condType string + msg string + }{ + {pacmkrv1.ResourceOperationalConditionType, "has failed"}, + {pacmkrv1.ResourceStartedConditionType, "is stopped"}, + {pacmkrv1.ResourceActiveConditionType, "is not active"}, + {pacmkrv1.ResourceManagedConditionType, "is unmanaged"}, + } + for _, check := range checks { + if cond := FindCondition(conditions, check.condType); cond != nil && cond.Status != metav1.ConditionTrue { + return check.msg + } } - return "is unhealthy" } @@ -619,16 +542,16 @@ func (c *HealthCheck) updateOperatorStatus(ctx context.Context, status *HealthSt // Update operator conditions based on pacemaker status switch status.OverallStatus { - case statusError: + case StatusError: return c.setPacemakerDegradedCondition(ctx, status) - case statusHealthy, statusWarning: + case StatusHealthy, StatusWarning: // Both healthy and warning states should clear degraded condition // Warnings are informational (e.g. recent fencing, node count mismatch) and don't indicate degradation - if status.OverallStatus == statusWarning { + if status.OverallStatus == StatusWarning { klog.V(2).Infof("Pacemaker health check has warnings but cluster is operational: %v", status.Warnings) } return c.clearPacemakerDegradedCondition(ctx, status) - case statusUnknown: + case StatusUnknown: // Unknown status means we cannot determine pacemaker health (CR not found, stale, no status, etc.) // Only mark as degraded if we haven't received a valid status in a while (grace period). // Use previous.CRLastUpdated which reflects when we last had valid cluster data. @@ -831,12 +754,12 @@ func (c *HealthCheck) recordHealthCheckEvents(current *HealthStatus, previous *H func (c *HealthCheck) recordHealthTransitionEvents(current *HealthStatus, previous *HealthStatus) { // Determine previous state from the previous HealthStatus we computed. // This is derived from the previous PacemakerCluster CR we processed. - previousWasUnknown := previous == nil || previous.OverallStatus == statusUnknown - previousWasDegraded := previous != nil && previous.OverallStatus == statusError + previousWasUnknown := previous == nil || previous.OverallStatus == StatusUnknown + previousWasDegraded := previous != nil && previous.OverallStatus == StatusError previousHadWarnings := previous != nil && len(previous.Warnings) > 0 // Determine current state - operationallyHealthy := current.OverallStatus == statusHealthy || current.OverallStatus == statusWarning + operationallyHealthy := current.OverallStatus == StatusHealthy || current.OverallStatus == StatusWarning currentHasNoWarnings := len(current.Warnings) == 0 // Record PacemakerHealthy when transitioning to operationally healthy from: @@ -862,10 +785,6 @@ func (c *HealthCheck) recordHealthTransitionEvents(current *HealthStatus, previo } } -// recordWarningEvent records appropriate warning events based on warning type. -// Note: Failed action events (warningPrefixFailedAction) are recorded by the status collector -// directly from pacemaker XML, not by the healthcheck controller. The PacemakerCluster CR -// only contains conditions, not raw failed action history. func (c *HealthCheck) recordWarningEvent(warning string) { switch { case strings.Contains(warning, warningPrefixFencingEvent): diff --git a/pkg/tnf/pkg/pacemaker/healthcheck_test.go b/pkg/tnf/pkg/pacemaker/healthcheck_test.go index c9f2af19ff..ff57c11338 100644 --- a/pkg/tnf/pkg/pacemaker/healthcheck_test.go +++ b/pkg/tnf/pkg/pacemaker/healthcheck_test.go @@ -105,7 +105,7 @@ func TestHealthCheck_getPacemakerStatus(t *testing.T) { LastUpdated: metav1.Now(), }, }, - expectedStatus: statusHealthy, + expectedStatus: StatusHealthy, expectErrors: false, expectWarnings: false, }, @@ -128,7 +128,7 @@ func TestHealthCheck_getPacemakerStatus(t *testing.T) { LastUpdated: metav1.Time{Time: time.Now().Add(-10 * time.Minute)}, // > 5 min threshold }, }, - expectedStatus: statusUnknown, + expectedStatus: StatusUnknown, expectErrors: true, expectWarnings: false, }, @@ -144,7 +144,7 @@ func TestHealthCheck_getPacemakerStatus(t *testing.T) { }, Status: pacmkrv1.PacemakerClusterStatus{}, // Status not populated yet (zero LastUpdated) }, - expectedStatus: statusUnknown, + expectedStatus: StatusUnknown, expectErrors: true, expectWarnings: false, }, @@ -166,7 +166,7 @@ func TestHealthCheck_getPacemakerStatus(t *testing.T) { LastUpdated: metav1.Now(), }, }, - expectedStatus: statusError, + expectedStatus: StatusError, expectErrors: true, expectWarnings: false, }, @@ -189,7 +189,7 @@ func TestHealthCheck_getPacemakerStatus(t *testing.T) { LastUpdated: metav1.Now(), }, }, - expectedStatus: statusError, + expectedStatus: StatusError, expectErrors: true, expectWarnings: false, }, @@ -212,7 +212,7 @@ func TestHealthCheck_getPacemakerStatus(t *testing.T) { LastUpdated: metav1.Now(), }, }, - expectedStatus: statusError, + expectedStatus: StatusError, expectErrors: true, expectWarnings: false, }, @@ -235,7 +235,7 @@ func TestHealthCheck_getPacemakerStatus(t *testing.T) { LastUpdated: metav1.Now(), }, }, - expectedStatus: statusError, + expectedStatus: StatusError, expectErrors: true, expectWarnings: false, }, @@ -296,7 +296,7 @@ func TestHealthCheck_getPacemakerStatus_UnchangedTimestamp(t *testing.T) { current1, _, err1 := controller.getPacemakerStatus(ctx) require.NoError(t, err1, "First getPacemakerStatus should not return an error") require.NotNil(t, current1, "First call should return a status") - require.Equal(t, statusHealthy, current1.OverallStatus, "First call should return healthy status") + require.Equal(t, StatusHealthy, current1.OverallStatus, "First call should return healthy status") // Second call with same timestamp should return nil (no change) current2, _, err2 := controller.getPacemakerStatus(ctx) @@ -313,36 +313,36 @@ func TestHealthCheck_updateOperatorStatus(t *testing.T) { { name: "error_status_sets_degraded", status: &HealthStatus{ - OverallStatus: statusError, + OverallStatus: StatusError, Warnings: []string{"Test warning"}, Errors: []string{"Test error"}, }, previous: &HealthStatus{ - OverallStatus: statusHealthy, + OverallStatus: StatusHealthy, CRLastUpdated: time.Now().Add(-1 * time.Minute), }, }, { name: "healthy_status_clears_degraded", status: &HealthStatus{ - OverallStatus: statusHealthy, + OverallStatus: StatusHealthy, Warnings: []string{}, Errors: []string{}, }, previous: &HealthStatus{ - OverallStatus: statusHealthy, + OverallStatus: StatusHealthy, CRLastUpdated: time.Now().Add(-1 * time.Minute), }, }, { name: "warning_status_clears_degraded", status: &HealthStatus{ - OverallStatus: statusWarning, + OverallStatus: StatusWarning, Warnings: []string{"Test warning"}, Errors: []string{}, }, previous: &HealthStatus{ - OverallStatus: statusHealthy, + OverallStatus: StatusHealthy, CRLastUpdated: time.Now().Add(-1 * time.Minute), }, }, @@ -363,9 +363,9 @@ func TestHealthCheck_recordHealthCheckEvents(t *testing.T) { controller := createTestHealthCheck() // Test with warnings and errors - previous was healthy (not degraded, no warnings, known status) - previousHealthy := &HealthStatus{OverallStatus: statusHealthy, Warnings: []string{}, Errors: []string{}} + previousHealthy := &HealthStatus{OverallStatus: StatusHealthy, Warnings: []string{}, Errors: []string{}} currentWithWarnings := &HealthStatus{ - OverallStatus: statusWarning, + OverallStatus: StatusWarning, Warnings: []string{ "Recent failed resource action: kubelet monitor on master-0 failed", "Recent fencing event: reboot of master-1 success", @@ -378,7 +378,7 @@ func TestHealthCheck_recordHealthCheckEvents(t *testing.T) { // Test with healthy status - previous had warnings currentHealthy := &HealthStatus{ - OverallStatus: statusHealthy, + OverallStatus: StatusHealthy, Warnings: []string{}, Errors: []string{}, } @@ -396,7 +396,7 @@ func TestHealthCheck_eventDeduplication(t *testing.T) { // First recording - Error state (start in Error to test transitions properly) // previousStatus=nil because this is first sync errorStatus := &HealthStatus{ - OverallStatus: statusError, + OverallStatus: StatusError, Warnings: []string{}, Errors: []string{"Test error"}, } @@ -416,7 +416,7 @@ func TestHealthCheck_eventDeduplication(t *testing.T) { // Transition to Warning (operationally healthy) - should record PacemakerHealthy event // previousStatus is errorStatus (degraded), so healthy event should fire warningStatus := &HealthStatus{ - OverallStatus: statusWarning, + OverallStatus: StatusWarning, Warnings: []string{ "Recent failed resource action: kubelet monitor on master-0 failed", }, @@ -434,7 +434,7 @@ func TestHealthCheck_eventDeduplication(t *testing.T) { // Test fencing event deduplication (longer window) fencingStatus := &HealthStatus{ - OverallStatus: statusWarning, + OverallStatus: StatusWarning, Warnings: []string{ "Recent fencing event: reboot of master-1 success", }, @@ -453,7 +453,7 @@ func TestHealthCheck_eventDeduplication(t *testing.T) { // Transition from Warning to Healthy - fires PacemakerWarningsCleared event // previousStatus has warnings, so WarningsCleared should fire healthyStatus := &HealthStatus{ - OverallStatus: statusHealthy, + OverallStatus: StatusHealthy, Warnings: []string{}, Errors: []string{}, } @@ -479,7 +479,7 @@ func TestHealthCheck_eventDeduplication(t *testing.T) { require.Greater(t, afterRecovery, beforeRecovery, "PacemakerHealthy event should be recorded on transition from Error to Healthy") } -func TestHealthCheck_buildHealthStatusFromCR(t *testing.T) { +func TestBuildHealthStatusFromCR(t *testing.T) { tests := []struct { name string cr *pacmkrv1.PacemakerCluster @@ -499,7 +499,7 @@ func TestHealthCheck_buildHealthStatusFromCR(t *testing.T) { }, }, }, - expectedStatus: statusHealthy, + expectedStatus: StatusHealthy, expectErrors: false, }, { @@ -514,7 +514,7 @@ func TestHealthCheck_buildHealthStatusFromCR(t *testing.T) { }, }, }, - expectedStatus: statusError, + expectedStatus: StatusError, expectErrors: true, errorContains: string(pacmkrv1.PacemakerClusterResourceNameKubelet), }, @@ -530,7 +530,7 @@ func TestHealthCheck_buildHealthStatusFromCR(t *testing.T) { }, }, }, - expectedStatus: statusError, + expectedStatus: StatusError, expectErrors: true, errorContains: string(pacmkrv1.PacemakerClusterResourceNameEtcd), }, @@ -545,7 +545,7 @@ func TestHealthCheck_buildHealthStatusFromCR(t *testing.T) { }, }, }, - expectedStatus: statusError, + expectedStatus: StatusError, expectErrors: true, errorContains: "Insufficient nodes", }, @@ -558,7 +558,7 @@ func TestHealthCheck_buildHealthStatusFromCR(t *testing.T) { Nodes: &[]pacmkrv1.PacemakerClusterNodeStatus{}, }, }, - expectedStatus: statusError, + expectedStatus: StatusError, expectErrors: true, errorContains: "No nodes found", }, @@ -567,16 +567,15 @@ func TestHealthCheck_buildHealthStatusFromCR(t *testing.T) { cr: &pacmkrv1.PacemakerCluster{ Status: pacmkrv1.PacemakerClusterStatus{}, // Zero LastUpdated }, - expectedStatus: statusUnknown, + expectedStatus: StatusUnknown, expectErrors: true, - errorContains: "Internal error", + errorContains: "no status populated", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - controller := &HealthCheck{} - status := controller.buildHealthStatusFromCR(tt.cr) + status := BuildHealthStatusFromCR(tt.cr) require.NotNil(t, status, "HealthStatus should not be nil") require.Equal(t, tt.expectedStatus, status.OverallStatus) @@ -601,7 +600,7 @@ func TestHealthCheck_buildHealthStatusFromCR(t *testing.T) { } } -func TestHealthCheck_checkClusterConditions(t *testing.T) { +func TestCheckClusterConditions(t *testing.T) { tests := []struct { name string conditions []metav1.Condition @@ -661,8 +660,7 @@ func TestHealthCheck_checkClusterConditions(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - controller := &HealthCheck{} - pacemakerStatus := &pacmkrv1.PacemakerCluster{ + cr := &pacmkrv1.PacemakerCluster{ Status: pacmkrv1.PacemakerClusterStatus{ Conditions: tt.conditions, Nodes: tt.nodes, @@ -670,7 +668,7 @@ func TestHealthCheck_checkClusterConditions(t *testing.T) { } status := &HealthStatus{Warnings: []string{}, Errors: []string{}} - controller.checkClusterConditions(pacemakerStatus, status) + checkClusterConditions(cr, status) if tt.expectErrors { require.NotEmpty(t, status.Errors, "Should have errors") @@ -684,7 +682,7 @@ func TestHealthCheck_checkClusterConditions(t *testing.T) { } } -func TestHealthCheck_checkNodeStatuses(t *testing.T) { +func TestCheckNodeStatuses(t *testing.T) { tests := []struct { name string nodes *[]pacmkrv1.PacemakerClusterNodeStatus @@ -727,8 +725,7 @@ func TestHealthCheck_checkNodeStatuses(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - controller := &HealthCheck{} - pacemakerStatus := &pacmkrv1.PacemakerCluster{ + cr := &pacmkrv1.PacemakerCluster{ Status: pacmkrv1.PacemakerClusterStatus{ Conditions: createHealthyClusterConditions(), Nodes: tt.nodes, @@ -736,7 +733,7 @@ func TestHealthCheck_checkNodeStatuses(t *testing.T) { } status := &HealthStatus{Warnings: []string{}, Errors: []string{}} - controller.checkNodeStatuses(pacemakerStatus, status) + checkNodeStatuses(cr, status) if tt.expectErrors { require.NotEmpty(t, status.Errors, "Should have errors") @@ -757,19 +754,14 @@ func TestHealthCheck_checkNodeStatuses(t *testing.T) { } } -func TestHealthCheck_checkNodeStatuses_MultipleUnhealthyResources(t *testing.T) { - controller := &HealthCheck{} - - // Create a node with multiple unhealthy resources +func TestCheckNodeStatuses_MultipleUnhealthyResources(t *testing.T) { node := createHealthyNodeStatus("master-0", []string{"192.168.1.10"}) - // Mark node as unhealthy for i := range node.Conditions { if node.Conditions[i].Type == pacmkrv1.NodeHealthyConditionType { node.Conditions[i].Status = metav1.ConditionFalse node.Conditions[i].Reason = pacmkrv1.NodeHealthyReasonUnhealthy } } - // Mark kubelet and etcd as unhealthy for i := range node.Resources { if node.Resources[i].Name == pacmkrv1.PacemakerClusterResourceNameKubelet || node.Resources[i].Name == pacmkrv1.PacemakerClusterResourceNameEtcd { @@ -787,9 +779,7 @@ func TestHealthCheck_checkNodeStatuses_MultipleUnhealthyResources(t *testing.T) } status := &HealthStatus{Warnings: []string{}, Errors: []string{}} - controller.checkNodeStatuses(pacemakerStatus, status) - - // Should have 1 consolidated error for the unhealthy node (includes all resource issues) + checkNodeStatuses(pacemakerStatus, status) require.Len(t, status.Errors, 1, "Should have 1 consolidated error for unhealthy node") // Verify the single error mentions both resources @@ -800,9 +790,7 @@ func TestHealthCheck_checkNodeStatuses_MultipleUnhealthyResources(t *testing.T) require.Contains(t, nodeError, string(pacmkrv1.PacemakerClusterResourceNameEtcd), "Error should mention Etcd") } -func TestHealthCheck_getResourceIssue(t *testing.T) { - controller := &HealthCheck{} - +func TestGetResourceIssue(t *testing.T) { tests := []struct { name string conditions []metav1.Condition @@ -837,7 +825,7 @@ func TestHealthCheck_getResourceIssue(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := controller.getResourceIssue(tt.conditions) + got := getResourceIssue(tt.conditions) require.Equal(t, tt.wantIssue, got) }) } @@ -936,7 +924,7 @@ func TestHealthCheck_eventDeduplication_StatusTransitions(t *testing.T) { // Start with healthy status from Unknown state (first sync or recovering from stale) // previousStatus=nil: PacemakerHealthy event fires on initial healthy status healthyStatus := &HealthStatus{ - OverallStatus: statusHealthy, + OverallStatus: StatusHealthy, Warnings: []string{}, Errors: []string{}, } @@ -951,7 +939,7 @@ func TestHealthCheck_eventDeduplication_StatusTransitions(t *testing.T) { // Transition to Warning - should record warning but not healthy event (already in healthy state) warningStatus := &HealthStatus{ - OverallStatus: statusWarning, + OverallStatus: StatusWarning, Warnings: []string{"Some warning"}, Errors: []string{}, } @@ -969,7 +957,7 @@ func TestHealthCheck_eventDeduplication_StatusTransitions(t *testing.T) { // Transition to Error errorStatus := &HealthStatus{ - OverallStatus: statusError, + OverallStatus: StatusError, Warnings: []string{}, Errors: []string{"Critical error"}, } @@ -995,9 +983,9 @@ func TestHealthCheck_eventDeduplication_CleanupOldEntries(t *testing.T) { controller.recordedEventsMu.Unlock() // Trigger cleanup by recording a new event - previousStatus := &HealthStatus{OverallStatus: statusHealthy, Warnings: []string{}, Errors: []string{}} + previousStatus := &HealthStatus{OverallStatus: StatusHealthy, Warnings: []string{}, Errors: []string{}} currentStatus := &HealthStatus{ - OverallStatus: statusWarning, + OverallStatus: StatusWarning, Warnings: []string{"New warning"}, Errors: []string{}, } @@ -1010,8 +998,7 @@ func TestHealthCheck_eventDeduplication_CleanupOldEntries(t *testing.T) { controller.recordedEventsMu.Unlock() } -func TestHealthCheck_getNodeConditionErrorsAndWarnings(t *testing.T) { - controller := &HealthCheck{} +func TestGetNodeConditionErrorsAndWarnings(t *testing.T) { now := metav1.Now() tests := []struct { @@ -1102,8 +1089,8 @@ func TestHealthCheck_getNodeConditionErrorsAndWarnings(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { // Check errors and warnings using the separate functions - errors := controller.getNodeConditionErrors(tt.conditions) - warnings := controller.getFencingWarnings(tt.conditions) + errors := getNodeConditionErrors(tt.conditions) + warnings := getFencingWarnings(tt.conditions) // Check errors require.Equal(t, len(tt.expectedErrors), len(errors), "Number of errors should match") @@ -1137,7 +1124,7 @@ func TestHealthCheck_getNodeConditionErrorsAndWarnings(t *testing.T) { // TestHealthCheck_FencingRedundancyWarning tests fencing redundancy warnings in various scenarios. // Fencing redundancy degraded (FencingHealthy=False but FencingAvailable=True) should be // reported as a warning, not an error, and should be captured regardless of overall node health. -func TestHealthCheck_FencingRedundancyWarning(t *testing.T) { +func TestCheckNodeConditions_FencingRedundancyWarning(t *testing.T) { tests := []struct { name string nodeHealthy bool // Overall node healthy condition @@ -1176,7 +1163,6 @@ func TestHealthCheck_FencingRedundancyWarning(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - controller := &HealthCheck{} now := metav1.Now() // Build conditions based on test case @@ -1219,7 +1205,7 @@ func TestHealthCheck_FencingRedundancyWarning(t *testing.T) { } status := &HealthStatus{Warnings: []string{}, Errors: []string{}} - controller.checkNodeConditions(node, status) + checkNodeConditions(node, status) // Check warnings if tt.expectWarnings { @@ -1254,12 +1240,12 @@ func TestHealthCheck_WarningsClearedEvent(t *testing.T) { { name: "warning_to_healthy", previousStatus: &HealthStatus{ - OverallStatus: statusWarning, + OverallStatus: StatusWarning, Warnings: []string{"master-0: " + msgFencingRedundancyLost}, Errors: []string{}, }, currentStatus: &HealthStatus{ - OverallStatus: statusHealthy, + OverallStatus: StatusHealthy, Warnings: []string{}, Errors: []string{}, }, @@ -1269,12 +1255,12 @@ func TestHealthCheck_WarningsClearedEvent(t *testing.T) { { name: "error_without_warnings_to_healthy", previousStatus: &HealthStatus{ - OverallStatus: statusError, + OverallStatus: StatusError, Warnings: []string{}, Errors: []string{"Critical error"}, }, currentStatus: &HealthStatus{ - OverallStatus: statusHealthy, + OverallStatus: StatusHealthy, Warnings: []string{}, Errors: []string{}, }, @@ -1284,12 +1270,12 @@ func TestHealthCheck_WarningsClearedEvent(t *testing.T) { { name: "error_with_warnings_to_error_without_warnings", previousStatus: &HealthStatus{ - OverallStatus: statusError, + OverallStatus: StatusError, Warnings: []string{"master-0: " + msgFencingRedundancyLost}, Errors: []string{"Critical error"}, }, currentStatus: &HealthStatus{ - OverallStatus: statusError, + OverallStatus: StatusError, Warnings: []string{}, Errors: []string{"Critical error"}, }, @@ -1299,12 +1285,12 @@ func TestHealthCheck_WarningsClearedEvent(t *testing.T) { { name: "error_with_warnings_to_healthy", previousStatus: &HealthStatus{ - OverallStatus: statusError, + OverallStatus: StatusError, Warnings: []string{"master-0: " + msgFencingRedundancyLost}, Errors: []string{"Critical error"}, }, currentStatus: &HealthStatus{ - OverallStatus: statusHealthy, + OverallStatus: StatusHealthy, Warnings: []string{}, Errors: []string{}, }, diff --git a/pkg/tnf/pkg/tools/events.go b/pkg/tnf/pkg/tools/events.go new file mode 100644 index 0000000000..e3cf095321 --- /dev/null +++ b/pkg/tnf/pkg/tools/events.go @@ -0,0 +1,50 @@ +package tools + +import ( + "context" + "fmt" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" +) + +const ( + setupEventNamespace = "openshift-etcd" + setupJobName = "tnf-setup-job" + setupSourceComponent = "tnf-setup-runner" +) + +// RecordSetupEvent creates a Normal Kubernetes event in openshift-etcd for a +// TNF setup lifecycle milestone. Failures are logged but never returned — +// event recording must not block the setup. +func RecordSetupEvent(ctx context.Context, kubeClient kubernetes.Interface, reason, message string) { + event := &corev1.Event{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("tnf-%s-%d", strings.ToLower(reason), time.Now().UnixNano()), + Namespace: setupEventNamespace, + }, + InvolvedObject: corev1.ObjectReference{ + Kind: "Job", + Name: setupJobName, + Namespace: setupEventNamespace, + APIVersion: "batch/v1", + }, + Reason: reason, + Message: message, + Type: corev1.EventTypeNormal, + Source: corev1.EventSource{Component: setupSourceComponent}, + FirstTimestamp: metav1.Now(), + LastTimestamp: metav1.Now(), + Count: 1, + } + + if _, err := kubeClient.CoreV1().Events(setupEventNamespace).Create(ctx, event, metav1.CreateOptions{}); err != nil { + klog.Warningf("Failed to record %s event: %v", reason, err) + } else { + klog.Infof("Recorded event: %s - %s", reason, message) + } +} diff --git a/pkg/tnf/setup/runner.go b/pkg/tnf/setup/runner.go index 7586c2df5c..6f90330bc0 100644 --- a/pkg/tnf/setup/runner.go +++ b/pkg/tnf/setup/runner.go @@ -83,6 +83,9 @@ func RunTnfSetup() error { return err } + tools.RecordSetupEvent(ctx, kubeClient, "EtcdTransitionAuthCompleted", + "PCS authentication completed on all nodes") + klog.Info("Running TNF setup") // create tnf cluster config @@ -100,12 +103,18 @@ func RunTnfSetup() error { time.Sleep(5 * time.Second) } + tools.RecordSetupEvent(ctx, kubeClient, "EtcdTransitionClusterConfigured", + "Pacemaker cluster configured successfully") + // configure stonith err = pcs.ConfigureFencing(ctx, kubeClient, []string{cfg.NodeName1, cfg.NodeName2}) if err != nil { return err } + tools.RecordSetupEvent(ctx, kubeClient, "EtcdTransitionFencingConfigured", + "STONITH fencing configured successfully") + // Register pacemaker alert agents for fencing taint/untaint. // Tolerate the scripts not being present yet, they are delivered by // MCO which may not have rolled out at this point @@ -125,6 +134,9 @@ func RunTnfSetup() error { return err } + tools.RecordSetupEvent(ctx, kubeClient, "EtcdTransitionEtcdResourceCreated", + "Pacemaker etcd resource agent (podman-etcd) configured") + // configure etcd constraints configured, err = pcs.ConfigureConstraints(ctx) if err != nil { @@ -134,8 +146,11 @@ func RunTnfSetup() error { time.Sleep(5 * time.Second) } + tools.RecordSetupEvent(ctx, kubeClient, "EtcdTransitionConstraintsConfigured", + "Pacemaker ordering and colocation constraints configured") + // Signal CEO that TNF setup is ready for etcd container removal - err = etcd.RemoveStaticContainer(ctx, operatorClient) + err = etcd.RemoveStaticContainer(ctx, operatorClient, kubeClient) if err != nil { return err }