AAP-46619 Get pod container status

.gitignore

@@ -29,4 +29,5 @@ venv
 /vendor
 pkg/services/.lock
 pkg/workceptor/status
-pkg/workceptor/status.lock
+pkg/workceptor/status.lock
+**/__debug_*

pkg/workceptor/kubernetes.go

@@ -177,6 +177,8 @@ var ErrPodFailed = fmt.Errorf("pod failed to start")
 // ErrImagePullBackOff is returned when the image for the container in the Pod cannot be pulled.
 var ErrImagePullBackOff = fmt.Errorf("container failed to start")
 
+const WorkerContainerName = "worker"
+
 // podRunningAndReady is a completion criterion for pod ready to be attached to.
 func podRunningAndReady(kw KubeUnit) func(event watch.Event) (bool, error) {
 	imagePullBackOffRetries := 3
@@ -249,7 +251,7 @@ func (kw *KubeUnit) kubeLoggingConnectionHandler(timestamps bool, sinceTime time
 	podNamespace := kw.Pod.Namespace
 	podName := kw.Pod.Name
 	podOptions := &corev1.PodLogOptions{
-		Container: "worker",
+		Container: WorkerContainerName,
 		Follow:    true,
 	}
 	if timestamps {
@@ -486,7 +488,7 @@ func (kw *KubeUnit) CreatePod(env map[string]string) error {
 		foundWorker := false
 		spec = &pod.Spec
 		for i := range spec.Containers {
-			if spec.Containers[i].Name == "worker" {
+			if spec.Containers[i].Name == WorkerContainerName {
 				spec.Containers[i].Stdin = true
 				spec.Containers[i].StdinOnce = true
 				foundWorker = true
@@ -517,7 +519,7 @@ func (kw *KubeUnit) CreatePod(env map[string]string) error {
 		}
 		spec = &corev1.PodSpec{
 			Containers: []corev1.Container{{
-				Name:      "worker",
+				Name:      WorkerContainerName,
 				Image:     ked.Image,
 				Command:   command,
 				Args:      params,
@@ -601,7 +603,7 @@ func (kw *KubeUnit) CreatePod(env map[string]string) error {
 	if err == ErrPodCompleted {
 		// Hao: shouldn't we also call kw.Cancel() in these cases?
 		for _, cstat := range kw.Pod.Status.ContainerStatuses {
-			if cstat.Name == "worker" {
+			if cstat.Name == WorkerContainerName {
 				if cstat.State.Terminated != nil && cstat.State.Terminated.ExitCode != 0 {
 					return fmt.Errorf("container failed with exit code %d: %s", cstat.State.Terminated.ExitCode, cstat.State.Terminated.Message)
 				}
@@ -632,7 +634,7 @@ func (kw *KubeUnit) CreatePod(env map[string]string) error {
 			}
 
 			for _, cstat := range kw.Pod.Status.ContainerStatuses {
-				if cstat.Name == "worker" {
+				if cstat.Name == WorkerContainerName {
 					if cstat.State.Waiting != nil {
 						return fmt.Errorf("%s, %s", err.Error(), cstat.State.Waiting.Reason)
 					}
@@ -732,7 +734,7 @@ func (kw *KubeUnit) runWorkUsingLogger() {
 
 		req.VersionedParams(
 			&corev1.PodExecOptions{
-				Container: "worker",
+				Container: WorkerContainerName,
 				Stdin:     true,
 				Stdout:    false,
 				Stderr:    false,

pkg/workceptor/pod.go

@@ -0,0 +1,92 @@
+package workceptor
+
+import (
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+)
+
+type KubePodStateHelper interface {
+	PodHealthy(pod *corev1.Pod, containerName string) (bool, error)
+	PodContainerHealthy(pod *corev1.Pod, containerName string) (bool, error)
+}
+
+// PodContainerHealthy checks if the pod has successfully completed its application logic.
+// this is called after podInfrastructureSuccess has confirmed the pod is in a terminal state.
+func (kw KubeUnit) PodContainerHealthy(pod *corev1.Pod, containerName string) (bool, error) {
+	if pod == nil {
+		return false, fmt.Errorf("pod is nil")
+	}
+
+	var foundContainer *corev1.ContainerStatus = nil
+
+	for i, cs := range pod.Status.ContainerStatuses {
+		if cs.Name == containerName {
+			foundContainer = &pod.Status.ContainerStatuses[i]
+
+			break
+		}
+	}
+	if foundContainer == nil {
+		return false, fmt.Errorf("pod does not contain container %s", containerName)
+	}
+
+	state := foundContainer.State
+
+	// Check if container is running and ready
+	if state.Running != nil {
+		return foundContainer.Ready, nil // Use Ready field for health
+	}
+
+	// Check if container terminated successfully
+	if state.Terminated != nil {
+		if state.Terminated.ExitCode == 0 {
+			return true, nil // Successfully completed
+		}
+
+		return false, fmt.Errorf("container %s failed with exit code %d: %s %s",
+			containerName, state.Terminated.ExitCode, state.Terminated.Reason, state.Terminated.Message)
+	}
+
+	// Container is waiting - usually not healthy yet
+	if state.Waiting != nil {
+		// Check if it's a problematic waiting state
+		reason := state.Waiting.Reason
+		if reason == "ImagePullBackOff" || reason == "ErrImagePull" ||
+			reason == "CrashLoopBackOff" || reason == "CreateContainerConfigError" {
+			return false, fmt.Errorf("container %s in error state: %s %s", containerName, reason, state.Waiting.Message)
+		}
+		// Normal waiting states like "ContainerCreating", "PodInitializing"
+		return false, nil // Not healthy yet, but not an error
+	}
+
+	return false, fmt.Errorf("container %s in unknown state: %v", containerName, state)
+}
+
+// PodHealthy checks if the pod and container are in a healthy state.
+func (kw KubeUnit) PodHealthy(pod *corev1.Pod, containerName string) (bool, error) {
+	if pod == nil {
+		return false, fmt.Errorf("pod is nil")
+	}
+
+	var containerDiag string = ""
+	containerOk, containerError := kw.PodContainerHealthy(pod, containerName)
+	if containerError != nil {
+		containerDiag = fmt.Sprintf(" %v", containerError)
+	}
+
+	switch pod.Status.Phase {
+	case corev1.PodFailed:
+		podError := fmt.Errorf("pod failed with reason: %s", pod.Status.Reason)
+		if pod.Status.Message != "" {
+			podError = fmt.Errorf("%s message: %s", podError, pod.Status.Message)
+		}
+
+		return false, fmt.Errorf("%s%s", podError, containerDiag)
+
+	case corev1.PodSucceeded, corev1.PodRunning, corev1.PodPending:
+		return containerOk, containerError
+	default:
+		return false, fmt.Errorf("unknown phase: %s%s", pod.Status.Phase, containerDiag)
+	}
+}