diff --git a/.gitignore b/.gitignore index 4ca36a88b..a2c016e3b 100644 --- a/.gitignore +++ b/.gitignore @@ -29,4 +29,5 @@ venv /vendor pkg/services/.lock pkg/workceptor/status -pkg/workceptor/status.lock \ No newline at end of file +pkg/workceptor/status.lock +**/__debug_* \ No newline at end of file diff --git a/pkg/workceptor/kubernetes.go b/pkg/workceptor/kubernetes.go index 096a113b5..9dbbaf7a5 100644 --- a/pkg/workceptor/kubernetes.go +++ b/pkg/workceptor/kubernetes.go @@ -177,6 +177,8 @@ var ErrPodFailed = fmt.Errorf("pod failed to start") // ErrImagePullBackOff is returned when the image for the container in the Pod cannot be pulled. var ErrImagePullBackOff = fmt.Errorf("container failed to start") +const WorkerContainerName = "worker" + // podRunningAndReady is a completion criterion for pod ready to be attached to. func podRunningAndReady(kw KubeUnit) func(event watch.Event) (bool, error) { imagePullBackOffRetries := 3 @@ -249,7 +251,7 @@ func (kw *KubeUnit) kubeLoggingConnectionHandler(timestamps bool, sinceTime time podNamespace := kw.Pod.Namespace podName := kw.Pod.Name podOptions := &corev1.PodLogOptions{ - Container: "worker", + Container: WorkerContainerName, Follow: true, } if timestamps { @@ -486,7 +488,7 @@ func (kw *KubeUnit) CreatePod(env map[string]string) error { foundWorker := false spec = &pod.Spec for i := range spec.Containers { - if spec.Containers[i].Name == "worker" { + if spec.Containers[i].Name == WorkerContainerName { spec.Containers[i].Stdin = true spec.Containers[i].StdinOnce = true foundWorker = true @@ -517,7 +519,7 @@ func (kw *KubeUnit) CreatePod(env map[string]string) error { } spec = &corev1.PodSpec{ Containers: []corev1.Container{{ - Name: "worker", + Name: WorkerContainerName, Image: ked.Image, Command: command, Args: params, @@ -601,7 +603,7 @@ func (kw *KubeUnit) CreatePod(env map[string]string) error { if err == ErrPodCompleted { // Hao: shouldn't we also call kw.Cancel() in these cases? for _, cstat := range kw.Pod.Status.ContainerStatuses { - if cstat.Name == "worker" { + if cstat.Name == WorkerContainerName { if cstat.State.Terminated != nil && cstat.State.Terminated.ExitCode != 0 { return fmt.Errorf("container failed with exit code %d: %s", cstat.State.Terminated.ExitCode, cstat.State.Terminated.Message) } @@ -632,7 +634,7 @@ func (kw *KubeUnit) CreatePod(env map[string]string) error { } for _, cstat := range kw.Pod.Status.ContainerStatuses { - if cstat.Name == "worker" { + if cstat.Name == WorkerContainerName { if cstat.State.Waiting != nil { return fmt.Errorf("%s, %s", err.Error(), cstat.State.Waiting.Reason) } @@ -732,7 +734,7 @@ func (kw *KubeUnit) runWorkUsingLogger() { req.VersionedParams( &corev1.PodExecOptions{ - Container: "worker", + Container: WorkerContainerName, Stdin: true, Stdout: false, Stderr: false, diff --git a/pkg/workceptor/pod.go b/pkg/workceptor/pod.go new file mode 100644 index 000000000..6975bcfb3 --- /dev/null +++ b/pkg/workceptor/pod.go @@ -0,0 +1,92 @@ +package workceptor + +import ( + "fmt" + + corev1 "k8s.io/api/core/v1" +) + +type KubePodStateHelper interface { + PodHealthy(pod *corev1.Pod, containerName string) (bool, error) + PodContainerHealthy(pod *corev1.Pod, containerName string) (bool, error) +} + +// PodContainerHealthy checks if the pod has successfully completed its application logic. +// this is called after podInfrastructureSuccess has confirmed the pod is in a terminal state. +func (kw KubeUnit) PodContainerHealthy(pod *corev1.Pod, containerName string) (bool, error) { + if pod == nil { + return false, fmt.Errorf("pod is nil") + } + + var foundContainer *corev1.ContainerStatus = nil + + for i, cs := range pod.Status.ContainerStatuses { + if cs.Name == containerName { + foundContainer = &pod.Status.ContainerStatuses[i] + + break + } + } + if foundContainer == nil { + return false, fmt.Errorf("pod does not contain container %s", containerName) + } + + state := foundContainer.State + + // Check if container is running and ready + if state.Running != nil { + return foundContainer.Ready, nil // Use Ready field for health + } + + // Check if container terminated successfully + if state.Terminated != nil { + if state.Terminated.ExitCode == 0 { + return true, nil // Successfully completed + } + + return false, fmt.Errorf("container %s failed with exit code %d: %s %s", + containerName, state.Terminated.ExitCode, state.Terminated.Reason, state.Terminated.Message) + } + + // Container is waiting - usually not healthy yet + if state.Waiting != nil { + // Check if it's a problematic waiting state + reason := state.Waiting.Reason + if reason == "ImagePullBackOff" || reason == "ErrImagePull" || + reason == "CrashLoopBackOff" || reason == "CreateContainerConfigError" { + return false, fmt.Errorf("container %s in error state: %s %s", containerName, reason, state.Waiting.Message) + } + // Normal waiting states like "ContainerCreating", "PodInitializing" + return false, nil // Not healthy yet, but not an error + } + + return false, fmt.Errorf("container %s in unknown state: %v", containerName, state) +} + +// PodHealthy checks if the pod and container are in a healthy state. +func (kw KubeUnit) PodHealthy(pod *corev1.Pod, containerName string) (bool, error) { + if pod == nil { + return false, fmt.Errorf("pod is nil") + } + + var containerDiag string = "" + containerOk, containerError := kw.PodContainerHealthy(pod, containerName) + if containerError != nil { + containerDiag = fmt.Sprintf(" %v", containerError) + } + + switch pod.Status.Phase { + case corev1.PodFailed: + podError := fmt.Errorf("pod failed with reason: %s", pod.Status.Reason) + if pod.Status.Message != "" { + podError = fmt.Errorf("%s message: %s", podError, pod.Status.Message) + } + + return false, fmt.Errorf("%s%s", podError, containerDiag) + + case corev1.PodSucceeded, corev1.PodRunning, corev1.PodPending: + return containerOk, containerError + default: + return false, fmt.Errorf("unknown phase: %s%s", pod.Status.Phase, containerDiag) + } +} diff --git a/pkg/workceptor/pod_test.go b/pkg/workceptor/pod_test.go new file mode 100644 index 000000000..3ca164b7f --- /dev/null +++ b/pkg/workceptor/pod_test.go @@ -0,0 +1,505 @@ +package workceptor_test + +import ( + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var podSuccess = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "test-pod", + }, + Status: corev1.PodStatus{ + Phase: corev1.PodSucceeded, + ContainerStatuses: []corev1.ContainerStatus{ + { + Name: "worker", + State: corev1.ContainerState{ + Waiting: nil, + Running: nil, + Terminated: &corev1.ContainerStateTerminated{ + ExitCode: 0, + Reason: "Success", + }, + }, + }, + }, + }, +} + +var podInfraError = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "infra-error-pod", + }, + Status: corev1.PodStatus{ + Phase: corev1.PodFailed, + Reason: "OOMKilled", + ContainerStatuses: []corev1.ContainerStatus{ + { + Name: "worker", + State: corev1.ContainerState{ + Terminated: &corev1.ContainerStateTerminated{ + ExitCode: 137, + Reason: "OOMKill", + }, + }, + }, + }, + }, +} + +var podInfraErrorWithMessage = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "infra-error-pod", + }, + Status: corev1.PodStatus{ + Phase: corev1.PodFailed, + Reason: "Pod OOMKilled", + Message: "The pod was killed because it ran out of memory", + ContainerStatuses: []corev1.ContainerStatus{ + { + Name: "worker", + State: corev1.ContainerState{ + Terminated: &corev1.ContainerStateTerminated{ + ExitCode: 137, + Reason: "Container OOMKill", + }, + }, + }, + }, + }, +} + +var podAppError = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "app-error-pod", + }, + Status: corev1.PodStatus{ + Phase: corev1.PodFailed, + Reason: "Error", + ContainerStatuses: []corev1.ContainerStatus{ + { + Name: "worker", + State: corev1.ContainerState{ + Terminated: &corev1.ContainerStateTerminated{ + ExitCode: 1, + Reason: "Error", + }, + }, + }, + }, + }, +} + +var podPending = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "pending-pod", + }, + Status: corev1.PodStatus{ + Phase: corev1.PodPending, + ContainerStatuses: []corev1.ContainerStatus{ + { + Name: "worker", + State: corev1.ContainerState{ + Waiting: &corev1.ContainerStateWaiting{ + Reason: "ContainerCreating", + Message: "Container is being created", + }, + }, + }, + }, + }, +} + +var podUnknownPhase = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "unknown-phase-pod", + }, + Status: corev1.PodStatus{ + Phase: "NotARealPhase", + ContainerStatuses: []corev1.ContainerStatus{ + { + Name: "worker", + State: corev1.ContainerState{ + Waiting: &corev1.ContainerStateWaiting{ + Reason: "ContainerCreating", + Message: "Container is being created", + }, + }, + }, + }, + }, +} + +var podMultipleContainers = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "multi-container-pod", + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + ContainerStatuses: []corev1.ContainerStatus{ + { + Name: "worker", + State: corev1.ContainerState{ + Running: &corev1.ContainerStateRunning{ + StartedAt: metav1.Now(), + }, + }, + Ready: true, + }, + { + Name: "helper", + State: corev1.ContainerState{ + Running: &corev1.ContainerStateRunning{ + StartedAt: metav1.Now(), + }, + }, + Ready: true, + }, + }, + }, +} + +var podImagePullBackOff = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "pod-image-pull-backoff", + }, + Status: corev1.PodStatus{ + Phase: corev1.PodPending, + ContainerStatuses: []corev1.ContainerStatus{ + { + Name: "worker", + State: corev1.ContainerState{ + Waiting: &corev1.ContainerStateWaiting{ + Reason: "ImagePullBackOff", + Message: "Back-off pulling image", + }, + }, + }, + }, + }, +} + +var podCreateContainerConfigError = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "pod-Create-Container-ConfigErrorf", + }, + Status: corev1.PodStatus{ + Phase: corev1.PodPending, + ContainerStatuses: []corev1.ContainerStatus{ + { + Name: "worker", + State: corev1.ContainerState{ + Waiting: &corev1.ContainerStateWaiting{ + Reason: "CreateContainerConfigError", + Message: "Create container error", + }, + }, + }, + }, + }, +} + +var podErrImagePull = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "pod-error-image-pull", + }, + Status: corev1.PodStatus{ + Phase: corev1.PodPending, + ContainerStatuses: []corev1.ContainerStatus{ + { + Name: "worker", + State: corev1.ContainerState{ + Waiting: &corev1.ContainerStateWaiting{ + Reason: "ErrImagePull", + Message: "Error when pulling image", + }, + }, + }, + }, + }, +} + +var podCrashLoopBackOff = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "pod-crash-loop-backoff", + }, + Status: corev1.PodStatus{ + Phase: corev1.PodPending, + ContainerStatuses: []corev1.ContainerStatus{ + { + Name: "worker", + State: corev1.ContainerState{ + Waiting: &corev1.ContainerStateWaiting{ + Reason: "CrashLoopBackOff", + Message: "Error when starting image", + }, + }, + }, + }, + }, +} + +var podCreating = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "pod-creating", + }, + Status: corev1.PodStatus{ + Phase: corev1.PodPending, + ContainerStatuses: []corev1.ContainerStatus{ + { + Name: "worker", + State: corev1.ContainerState{ + Waiting: &corev1.ContainerStateWaiting{ + Reason: "ContainerCreating", + Message: "Container is being created", + }, + }, + }, + }, + }, +} + +func TestPodHeathy(t *testing.T) { + kw, err := startNetceptorNodeWithWorkceptor() + if err != nil { + t.Fatal(err) + } + + tests := []struct { + name string + pod *corev1.Pod + container string + wantOk bool + wantErr bool + wantError string + }{ + { + name: "pod err image pull", + pod: podErrImagePull, + container: "worker", + wantOk: false, + wantErr: true, + wantError: "container worker in error state: ErrImagePull Error when pulling image", + }, + { + name: "backoff image backoff pod", + pod: podImagePullBackOff, + container: "worker", + wantOk: false, + wantErr: true, + wantError: "container worker in error state: ImagePullBackOff Back-off pulling image", + }, + { + name: "Create Container Config Error pod", + pod: podCreateContainerConfigError, + container: "worker", + wantOk: false, + wantErr: true, + wantError: "container worker in error state: CreateContainerConfigError Create container error", + }, + { + name: "crash loop backoff pod", + pod: podCrashLoopBackOff, + container: "worker", + wantOk: false, + wantErr: true, + wantError: "container worker in error state: CrashLoopBackOff Error when starting image", + }, + { + name: "pod creating", + pod: podCreating, + container: "worker", + wantOk: false, + wantErr: false, + }, + { + name: "nil pod", + pod: nil, + container: "worker", + wantOk: false, + wantErr: true, + wantError: "pod is nil", + }, + { + name: "pod not terminated", + pod: podPending, + container: "worker", + wantOk: false, + wantErr: false, + }, + { + name: "container missing", + pod: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "no-container-pod", Namespace: "default"}}, + container: "worker", + wantOk: false, + wantErr: true, + wantError: "pod does not contain container worker", + }, + { + name: "container healthy", + pod: podSuccess, + container: "worker", + wantOk: true, + wantErr: false, + }, + { + name: "pod unknown phase", + pod: podUnknownPhase, + container: "worker", + wantOk: false, + wantErr: true, + wantError: "unknown phase: NotARealPhase", + }, + { + name: "pod with multiple containers", + pod: podMultipleContainers, + container: "worker", + wantOk: true, + wantErr: false, + }, + { + name: "pod with multiple containers, different container", + pod: podMultipleContainers, + container: "helper", + wantOk: true, + wantErr: false, + }, + { + name: "pod with oomkill error", + pod: podInfraError, + container: "worker", + wantOk: false, + wantErr: true, + wantError: "pod failed with reason: OOMKilled container worker failed with exit code 137: OOMKill", + }, + { + name: "pod with application error", + pod: podAppError, + container: "worker", + wantOk: false, + wantErr: true, + wantError: "pod failed with reason: Error container worker failed with exit code 1: Error ", + }, + { + name: "pod with oomkill error and message", + pod: podInfraErrorWithMessage, + container: "worker", + wantOk: false, + wantErr: true, + wantError: "pod failed with reason: Pod OOMKilled message: The pod was killed because it ran out of memory container worker failed with exit code 137: Container OOMKill ", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ok, err := kw.PodHealthy(tt.pod, tt.container) + if ok != tt.wantOk || (err != nil) != tt.wantErr { + t.Errorf("Failed %s case: ok=%v wantok=%v err=%v", tt.name, ok, tt.wantOk, err) + } + if err != nil && tt.wantErr == false { + t.Errorf("Expected error message got '%s'", err.Error()) + } + if tt.wantErr { + if err == nil { + t.Errorf("Expected error message '%s', got nil error", tt.wantError) + } else if !strings.Contains(err.Error(), tt.wantError) { + t.Errorf("Expected error message '%s', got '%s'", tt.wantError, err.Error()) + } + } + if tt.wantError == "" && err != nil { + t.Errorf("Unexpected error for %s case: %v", tt.name, err) + } + }) + } +} + +func TestPodContainerHealthy(t *testing.T) { + kw, err := startNetceptorNodeWithWorkceptor() + if err != nil { + t.Fatal(err) + } + + tests := []struct { + name string + pod *corev1.Pod + container string + wantOk bool + wantErr bool + wantError string + }{ + { + name: "nil pod", + pod: nil, + container: "worker", + wantOk: false, + wantErr: true, + wantError: "pod is nil", + }, + { + name: "pod not terminated", + pod: podPending, + container: "worker", + wantOk: false, + wantErr: false, + }, + { + name: "container missing", + pod: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "no-container-pod", Namespace: "default"}}, + container: "worker", + wantOk: false, + wantErr: true, + wantError: "pod does not contain container worker", + }, + { + name: "container healthy", + pod: podSuccess, + container: "worker", + wantOk: true, + wantErr: false, + }, + { + name: "pod ImagePullBackOff", + pod: podImagePullBackOff, + container: "worker", + wantOk: false, + wantErr: true, + wantError: "container worker in error state: ImagePullBackOff Back-off pulling image", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ok, err := kw.PodContainerHealthy(tt.pod, tt.container) + if ok != tt.wantOk || (err != nil) != tt.wantErr { + t.Errorf("Failed %s case: ok=%v wantok=%v err=%v", tt.name, ok, tt.wantOk, err) + } + if err != nil && tt.wantErr == false { + t.Errorf("Expected error message got '%s'", err.Error()) + } + if tt.wantErr { + if err == nil { + t.Errorf("Expected error message '%s', got nil error", tt.wantError) + } else if !strings.Contains(err.Error(), tt.wantError) { + t.Errorf("Expected error message '%s', got '%s'", tt.wantError, err.Error()) + } + } + if tt.wantError == "" && err != nil { + t.Errorf("Unexpected error for %s case: %v", tt.name, err) + } + }) + } +}