OCPEDGE-2094: Add console notification for pacemaker podman-etcd failures

pkg/tnf/hack/test-transition-events.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/bash
+# Test script to validate transition event format on a live cluster.
+# Creates mock events identical to what our code produces, then verifies
+# they appear correctly in `oc get events`.
+#
+# Usage: ./hack/test-transition-events.sh
+# Requires: oc/kubectl with cluster-admin access
+
+set -euo pipefail
+
+NAMESPACE="openshift-etcd"
+TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+
+events=(
+  "EtcdTransitionAuthCompleted|PCS authentication completed on all nodes"
+  "EtcdTransitionClusterConfigured|Pacemaker cluster configured successfully"
+  "EtcdTransitionFencingConfigured|STONITH fencing configured successfully"
+  "EtcdTransitionEtcdResourceCreated|Pacemaker etcd resource agent (podman-etcd) configured"
+  "EtcdTransitionConstraintsConfigured|Pacemaker ordering and colocation constraints configured"
+  "EtcdTransitionStarted|Etcd transition from CEO-controlled to pacemaker-controlled has started"
+  "EtcdTransitionWaitingForRemoval|Waiting for CEO to remove static etcd container from all nodes"
+  "EtcdTransitionStaticContainerRemoved|Static etcd container removed from all nodes, revision is stable"
+  "EtcdTransitionCompleted|Etcd transition to pacemaker-controlled has completed"
+)
+
+echo "Creating ${#events[@]} test transition events in ${NAMESPACE}..."
+
+for entry in "${events[@]}"; do
+  reason="${entry%%|*}"
+  message="${entry#*|}"
+  name="test-${reason,,}-$(date +%s%N)"
+
+  oc apply -f - <<EOF
+apiVersion: v1
+kind: Event
+metadata:
+  name: ${name}
+  namespace: ${NAMESPACE}
+  labels:
+    tnf-test: "true"
+involvedObject:
+  kind: Job
+  name: tnf-setup-job
+  namespace: ${NAMESPACE}
+  apiVersion: batch/v1
+reason: ${reason}
+message: "${message}"
+type: Normal
+source:
+  component: tnf-setup-runner
+firstTimestamp: "${TIMESTAMP}"
+lastTimestamp: "${TIMESTAMP}"
+count: 1
+EOF
+
+  echo "  Created: ${reason}"
+done
+
+echo ""
+echo "Verifying events..."
+echo ""
+oc get events -n "${NAMESPACE}" --field-selector reason!=="" --sort-by='.lastTimestamp' | grep -i "EtcdTransition" || echo "No EtcdTransition events found!"
+
+echo ""
+echo "Cleanup: oc delete events -n ${NAMESPACE} -l tnf-test=true"
+echo "  (events auto-expire after ~1 hour)"

pkg/tnf/operator/starter.go

@@ -80,7 +80,7 @@ func HandleDualReplicaClusters(
 	if err := runTnfResourceController(ctx, controllerContext, kubeClient, dynamicClient, operatorClient, kubeInformersForNamespaces); err != nil {
 		return false, err
 	}
-	runPacemakerControllers(ctx, controllerContext, operatorClient, kubeClient, etcdInformer)
+	runPacemakerControllers(ctx, controllerContext, operatorClient, kubeClient, dynamicClient, etcdInformer)
 
 	// we need node names for assigning auth and after-setup jobs to specific nodes
 	controlPlaneNodeLister := corev1listers.NewNodeLister(controlPlaneNodeInformer.GetIndexer())
@@ -223,7 +223,7 @@ func runTnfResourceController(ctx context.Context, controllerContext *controller
 	return nil
 }
 
-func runPacemakerControllers(ctx context.Context, controllerContext *controllercmd.ControllerContext, operatorClient v1helpers.StaticPodOperatorClient, kubeClient kubernetes.Interface, etcdInformer operatorv1informers.EtcdInformer) {
+func runPacemakerControllers(ctx context.Context, controllerContext *controllercmd.ControllerContext, operatorClient v1helpers.StaticPodOperatorClient, kubeClient kubernetes.Interface, dynamicClient dynamic.Interface, etcdInformer operatorv1informers.EtcdInformer) {
 	// Wait for external etcd transition before creating any Pacemaker controllers
 	// This runs in a background goroutine to avoid blocking the main thread.
 	go func() {
@@ -327,6 +327,16 @@ func runPacemakerControllers(ctx context.Context, controllerContext *controllerc
 		klog.Infof("starting Pacemaker metrics controller")
 		go metricsController.Run(ctx, 1)
 
+		// Create and start the console notification controller, sharing the same informer
+		klog.Infof("creating Pacemaker console notification controller")
+		notificationController := pacemaker.NewConsoleNotificationController(
+			pacemakerInformer,
+			dynamicClient,
+			controllerContext.EventRecorder,
+		)
+		klog.Infof("starting Pacemaker console notification controller")
+		go notificationController.Run(ctx, 1)
+
 		// Start the status collector CronJob
 		klog.Infof("starting Pacemaker status collector CronJob")
 		runPacemakerStatusCollectorCronJob(ctx, controllerContext, operatorClient, kubeClient)

pkg/tnf/pkg/etcd/etcd.go

@@ -8,13 +8,15 @@ import (
 	operatorv1 "github.com/openshift/api/operator/v1"
 	"github.com/openshift/library-go/pkg/operator/v1helpers"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
 	"k8s.io/klog/v2"
 
 	"github.com/openshift/cluster-etcd-operator/pkg/operator/ceohelpers"
+	"github.com/openshift/cluster-etcd-operator/pkg/tnf/pkg/tools"
 )
 
 // RemoveStaticContainer informs CEO to remove its etcd container
-func RemoveStaticContainer(ctx context.Context, operatorClient v1helpers.StaticPodOperatorClient) error {
+func RemoveStaticContainer(ctx context.Context, operatorClient v1helpers.StaticPodOperatorClient, kubeClient kubernetes.Interface) error {
 	klog.Info("Signaling CEO that TNF setup is ready for etcd container transition")
 
 	_, _, err := v1helpers.UpdateStatus(ctx, operatorClient, v1helpers.UpdateConditionFn(operatorv1.OperatorCondition{
@@ -28,8 +30,11 @@ func RemoveStaticContainer(ctx context.Context, operatorClient v1helpers.StaticP
 		return fmt.Errorf("error while updating ExternalEtcdReadyForTransition operator condition: %w", err)
 	}
 
+	tools.RecordSetupEvent(ctx, kubeClient, "EtcdTransitionStarted",
+		"Etcd transition from CEO-controlled to pacemaker-controlled has started")
+
 	// Wait for CEO to respond by removing static containers
-	err = waitForStaticContainerRemoved(ctx, operatorClient)
+	err = waitForStaticContainerRemoved(ctx, operatorClient, kubeClient)
 	if err != nil {
 		klog.Error(err, "Failed to wait for etcd container transition")
 		return err
@@ -39,15 +44,21 @@ func RemoveStaticContainer(ctx context.Context, operatorClient v1helpers.StaticP
 }
 
 // waitForStaticContainerRemoved waits until the static etcd container has been removed
-func waitForStaticContainerRemoved(ctx context.Context, operatorClient v1helpers.StaticPodOperatorClient) error {
+func waitForStaticContainerRemoved(ctx context.Context, operatorClient v1helpers.StaticPodOperatorClient, kubeClient kubernetes.Interface) error {
 	klog.Info("Wait for static etcd removed")
 
+	tools.RecordSetupEvent(ctx, kubeClient, "EtcdTransitionWaitingForRemoval",
+		"Waiting for CEO to remove static etcd container from all nodes")
+
 	// the container is removed when all nodes run the latest revision
 	err := WaitForStableRevision(ctx, operatorClient)
 	if err != nil {
 		return err
 	}
 
+	tools.RecordSetupEvent(ctx, kubeClient, "EtcdTransitionStaticContainerRemoved",
+		"Static etcd container removed from all nodes, revision is stable")
+
 	// Update the operator status to indicate that the transition has completed.
 	// As soon as the etcd container is removed, this operator won't be able to update this status
 	// unless the etcd container is restarted by the pacemaker resource agent.
@@ -58,7 +69,14 @@ func waitForStaticContainerRemoved(ctx context.Context, operatorClient v1helpers
 		Message: "pacemaker's resource agent is now running the etcd container",
 	}))
 
-	return err
+	if err != nil {
+		return err
+	}
+
+	tools.RecordSetupEvent(ctx, kubeClient, "EtcdTransitionCompleted",
+		"Etcd transition to pacemaker-controlled has completed, pacemaker's resource agent is now running the etcd container")
+
+	return nil
 }
 
 // WaitForStableRevision waits until all nodes run the latest available revision

pkg/tnf/pkg/etcd/etcd_test.go

@@ -8,54 +8,47 @@ import (
 	"github.com/openshift/library-go/pkg/operator/v1helpers"
 	"github.com/stretchr/testify/require"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes/fake"
 
 	"github.com/openshift/cluster-etcd-operator/pkg/operator/ceohelpers"
 )
 
-type args struct {
-	ctx            context.Context
-	operatorClient v1helpers.StaticPodOperatorClient
-}
-
 func TestRemoveStaticContainer(t *testing.T) {
-	tests := []struct {
-		name    string
-		args    args
-		wantErr bool
-	}{
-		{
-			name:    "sets ExternalEtcdReadyForTransition condition",
-			args:    getArgs(),
-			wantErr: false,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if err := RemoveStaticContainer(tt.args.ctx, tt.args.operatorClient); (err != nil) != tt.wantErr {
-				t.Errorf("RemoveStaticContainer() error = %v, wantErr %v", err, tt.wantErr)
-			}
+	ctx := context.Background()
+	operatorClient := newFakeOperatorClient()
+	kubeClient := fake.NewSimpleClientset()
 
-			// Verify that the operator condition was set
-			_, status, _, err := tt.args.operatorClient.GetStaticPodOperatorState()
-			require.NoError(t, err, "Failed to get static pod operator state")
-			isSet := v1helpers.IsOperatorConditionTrue(status.Conditions, ceohelpers.OperatorConditionExternalEtcdReadyForTransition)
-			require.True(t, isSet, "Expected ReadyForEtcdContainerRemoval condition to be set to True")
-		})
-	}
-}
+	err := RemoveStaticContainer(ctx, operatorClient, kubeClient)
+	require.NoError(t, err)
 
-func getArgs() args {
-	ctx := context.Background()
+	_, status, _, err := operatorClient.GetStaticPodOperatorState()
+	require.NoError(t, err)
+	require.True(t, v1helpers.IsOperatorConditionTrue(status.Conditions, ceohelpers.OperatorConditionExternalEtcdReadyForTransition))
+	require.True(t, v1helpers.IsOperatorConditionTrue(status.Conditions, ceohelpers.OperatorConditionExternalEtcdHasCompletedTransition))
 
-	etcd := &v1.Etcd{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: "cluster",
-		},
+	events, err := kubeClient.CoreV1().Events("openshift-etcd").List(ctx, metav1.ListOptions{})
+	require.NoError(t, err)
+
+	expectedReasons := map[string]bool{
+		"EtcdTransitionStarted":                true,
+		"EtcdTransitionWaitingForRemoval":      true,
+		"EtcdTransitionStaticContainerRemoved": true,
+		"EtcdTransitionCompleted":              true,
 	}
 
-	// Create a fake operator client with node statuses that indicate etcd container has been removed
-	fakeOperatorClient := v1helpers.NewFakeStaticPodOperatorClient(
-		&etcd.Spec.StaticPodOperatorSpec,
+	require.Len(t, events.Items, len(expectedReasons))
+	for _, event := range events.Items {
+		require.True(t, expectedReasons[event.Reason], "unexpected event reason: %s", event.Reason)
+		require.Equal(t, "Normal", event.Type)
+		require.Equal(t, "tnf-setup-runner", event.Source.Component)
+		delete(expectedReasons, event.Reason)
+	}
+	require.Empty(t, expectedReasons, "missing events: %v", expectedReasons)
+}
+
+func newFakeOperatorClient() v1helpers.StaticPodOperatorClient {
+	return v1helpers.NewFakeStaticPodOperatorClient(
+		&v1.StaticPodOperatorSpec{},
 		&v1.StaticPodOperatorStatus{
 			OperatorStatus: v1.OperatorStatus{
 				LatestAvailableRevision: 1,
@@ -70,9 +63,4 @@ func getArgs() args {
 		nil,
 		nil,
 	)
-
-	return args{
-		ctx:            ctx,
-		operatorClient: fakeOperatorClient,
-	}
 }