From 2d385364a1f73471f4531253cce4459174b930f5 Mon Sep 17 00:00:00 2001 From: Lisa Rashidi-Ranjbar Date: Wed, 4 Jun 2025 14:21:53 -0700 Subject: [PATCH] Add a pod status check after EOF detection --- pkg/workceptor/kubernetes.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/pkg/workceptor/kubernetes.go b/pkg/workceptor/kubernetes.go index 54b225f5d..bba4c3f92 100644 --- a/pkg/workceptor/kubernetes.go +++ b/pkg/workceptor/kubernetes.go @@ -386,6 +386,27 @@ func (kw *KubeUnit) KubeLoggingWithReconnect(streamWait *sync.WaitGroup, stdout podName, ) + // check container status again + pod, err := kw.KubeAPIWrapperInstance.Get(kw.GetContext(), kw.clientset, podNamespace, podName, metav1.GetOptions{}) + if err != nil { + kw.GetWorkceptor().nc.GetLogger().Error(err.Error()) + kw.UpdateBasicStatus(WorkStateFailed, err.Error(), stdout.Size()) + } + + // kw.GetWorkceptor().nc.GetLogger().Debug(pod.Status.String()) + // wait for container status to terminate + for pod.Status.ContainerStatuses[0].State.Terminated == nil { + pod, _ = kw.KubeAPIWrapperInstance.Get(kw.GetContext(), kw.clientset, podNamespace, podName, metav1.GetOptions{}) + } + + // if the pod's container finished unsuccessfully - add an error + if pod.Status.ContainerStatuses[0].State.Terminated != nil && pod.Status.ContainerStatuses[0].State.Terminated.ExitCode != 0 { + errmsg := fmt.Sprintf("pod exited unexpectedly reason: %s (%d)", + pod.Status.ContainerStatuses[0].State.Terminated.Reason, pod.Status.ContainerStatuses[0].State.Terminated.ExitCode) + kw.GetWorkceptor().nc.GetLogger().Error(errmsg) + kw.UpdateBasicStatus(WorkStateFailed, errmsg, stdout.Size()) + } + return }