From 7f577b090b7349f0759f5927c21c692f32af2773 Mon Sep 17 00:00:00 2001 From: eriknordmark Date: Tue, 21 Jul 2026 22:59:23 -0700 Subject: [PATCH 1/2] kubeapi: log CDI upload-pod teardown state on rollout failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When RolloutDiskToPVC exhausts its upload retries the app volume stays in CREATING_VOLUME, and the existing failure diagnosis walks only the data PVC, its PV, and the Longhorn engine. That path does not surface the case where the CDI upload pod reaches Ready but is then torn down — the data PVC's CDI annotations show pod.phase Failed / ContainerStatusUnknown — while its local-path scratch PVC is left stuck Terminating under the pvc-protection finalizer, so the upload never completes and the volume never leaves CREATING_VOLUME. Log the upload pod phase, the CDI teardown annotations on the data PVC, and the scratch PVC's phase / deletion-pending / storage class / finalizers before the existing checks, so this wedge is diagnosable from the device's own logs. Diagnostic only; no behavior change. Signed-off-by: eriknordmark Co-Authored-By: Claude Opus 4.8 --- pkg/pillar/kubeapi/vitoapiserver.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/pkg/pillar/kubeapi/vitoapiserver.go b/pkg/pillar/kubeapi/vitoapiserver.go index 78f0292455f..39cb0c311c8 100644 --- a/pkg/pillar/kubeapi/vitoapiserver.go +++ b/pkg/pillar/kubeapi/vitoapiserver.go @@ -513,6 +513,26 @@ func RolloutDiskToPVC(ctx context.Context, log *base.LogObject, exists bool, return transientf("PVC Upload for pvc:%s attempts to upload image failed, upload pod:%s does not exist", pvcName, cdiUploadPodName) } uploadNodeName := pod.Spec.NodeName + // 3b. The upload pod can reach Ready yet get torn down by CDI (pod.phase Failed / + // ContainerStatusUnknown) with the local-path scratch PVC left stuck Terminating — + // a wedge the data-vol/PV/engine checks below do not surface, so log it explicitly. + log.Noticef("RolloutDiskToPVC pvc:%s upload pod:%s phase:%s cdi.pod.phase:%q cdi.running.reason:%q cdi.running.msg:%q", + pvcName, cdiUploadPodName, pod.Status.Phase, + pvc.ObjectMeta.Annotations["cdi.kubevirt.io/storage.pod.phase"], + pvc.ObjectMeta.Annotations["cdi.kubevirt.io/storage.condition.running.reason"], + pvc.ObjectMeta.Annotations["cdi.kubevirt.io/storage.condition.running.message"]) + scratchName := pvcName + "-scratch" + if scratchPvc, scratchErr := PVCGet(scratchName, log); scratchErr != nil { + log.Noticef("RolloutDiskToPVC pvc:%s scratch PVC %s not found: %v", pvcName, scratchName, scratchErr) + } else { + scratchSC := "" + if scratchPvc.Spec.StorageClassName != nil { + scratchSC = *scratchPvc.Spec.StorageClassName + } + log.Noticef("RolloutDiskToPVC pvc:%s scratch PVC %s phase:%s terminating:%t storageClass:%q finalizers:%v", + pvcName, scratchName, scratchPvc.Status.Phase, + scratchPvc.ObjectMeta.DeletionTimestamp != nil, scratchSC, scratchPvc.ObjectMeta.Finalizers) + } // 4. Did the PVC claim get a backing pv? lhVol, err := lhVolGet(pvName) if err != nil { From c5901fe63a254210c20793db51d9504d51c67beb Mon Sep 17 00:00:00 2001 From: eriknordmark Date: Wed, 22 Jul 2026 11:18:19 -0700 Subject: [PATCH 2/2] kubeapi: name the unmet WaitForKubernetes gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On EVE-k the boot-time WaitForKubernetes poll gates on node-Ready, KubeVirt-CR-Available, and (optionally) Longhorn. On the 20-minute timeout it returned only "timed out waiting for the condition", and domainmgr then logged a hardcoded "kubevirt not ready" regardless of which check was actually failing — so an app wedged at INSTALLED while the node or Longhorn (not kubevirt) was the laggard was mislabeled and undiagnosable from the log alone. Record the last unmet sub-check and attach it to the returned error, and drop domainmgr's fixed label for a neutral "not satisfied", so the specific gate rides in the logged error. Diagnostic only: gating and retry behavior are unchanged, and all callers (volumemgr, domainmgr, zedkube) only log the error. Signed-off-by: eriknordmark Co-Authored-By: Claude Opus 4.8 --- pkg/pillar/cmd/domainmgr/domainmgr.go | 2 +- pkg/pillar/kubeapi/kubeapi.go | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/pkg/pillar/cmd/domainmgr/domainmgr.go b/pkg/pillar/cmd/domainmgr/domainmgr.go index 8ac5e6b9643..3e3579555e0 100644 --- a/pkg/pillar/cmd/domainmgr/domainmgr.go +++ b/pkg/pillar/cmd/domainmgr/domainmgr.go @@ -719,7 +719,7 @@ func Run(ps *pubsub.PubSub, loggerArg *logrus.Logger, logArg *base.LogObject, ar break } if waitForKubevirtFlag { - log.Warnf("Domainmgr: WaitForKubernetes kubevirt not ready: %v, retrying", err) + log.Warnf("Domainmgr: WaitForKubernetes not satisfied: %v, retrying", err) continue } log.Errorf("Domainmgr: WaitForKubernetes error %v", err) diff --git a/pkg/pillar/kubeapi/kubeapi.go b/pkg/pillar/kubeapi/kubeapi.go index ae8b495e6fb..5e2fbfcab9e 100644 --- a/pkg/pillar/kubeapi/kubeapi.go +++ b/pkg/pillar/kubeapi/kubeapi.go @@ -230,22 +230,28 @@ func WaitForKubernetes(agentName string, ps *pubsub.PubSub, stillRunning *time.T // nodeName is the caller-supplied EVE-k node name (from EdgeNodeInfo.DeviceName), // not os.Hostname(). var nodeReadyErr error + // Record the last failing sub-check; PollImmediate's timeout error alone does not say which of node/kubevirt/longhorn was unmet. + var lastUnmet error doneCh := make(chan struct{}, 1) go func() { nodeReadyErr = wait.PollImmediate(time.Second, time.Minute*20, func() (bool, error) { if err := nodeReadyByName(client, nodeName); err != nil { + lastUnmet = fmt.Errorf("node not ready: %w", err) return false, nil } if opts.WaitForKubevirt { if err := waitForKubevirtReady(config); err != nil { + lastUnmet = fmt.Errorf("kubevirt not ready: %w", err) return false, nil } } if opts.WaitForLonghorn { if err := checkLonghornReady(client, nodeName); err != nil { + lastUnmet = fmt.Errorf("longhorn not ready: %w", err) return false, nil } } + lastUnmet = nil return true, nil }) doneCh <- struct{}{} @@ -261,6 +267,9 @@ func WaitForKubernetes(agentName string, ps *pubsub.PubSub, stillRunning *time.T }) watches = append(watches, alsoWatch...) pubsub.MultiChannelWatch(watches) + if nodeReadyErr != nil && lastUnmet != nil { + return fmt.Errorf("%w (last unmet condition: %v)", nodeReadyErr, lastUnmet) + } return nodeReadyErr }