pillar: diagnose stuck EVE-k app bring-up (CDI + readiness gate)

pkg/pillar/cmd/domainmgr/domainmgr.go

@@ -719,7 +719,7 @@ func Run(ps *pubsub.PubSub, loggerArg *logrus.Logger, logArg *base.LogObject, ar
 				break
 			}
 			if waitForKubevirtFlag {
-				log.Warnf("Domainmgr: WaitForKubernetes kubevirt not ready: %v, retrying", err)
+				log.Warnf("Domainmgr: WaitForKubernetes not satisfied: %v, retrying", err)
 				continue
 			}
 			log.Errorf("Domainmgr: WaitForKubernetes error %v", err)

pkg/pillar/kubeapi/kubeapi.go

@@ -230,22 +230,28 @@ func WaitForKubernetes(agentName string, ps *pubsub.PubSub, stillRunning *time.T
 	// nodeName is the caller-supplied EVE-k node name (from EdgeNodeInfo.DeviceName),
 	// not os.Hostname().
 	var nodeReadyErr error
+	// Record the last failing sub-check; PollImmediate's timeout error alone does not say which of node/kubevirt/longhorn was unmet.
+	var lastUnmet error
 	doneCh := make(chan struct{}, 1)
 	go func() {
 		nodeReadyErr = wait.PollImmediate(time.Second, time.Minute*20, func() (bool, error) {
 			if err := nodeReadyByName(client, nodeName); err != nil {
+				lastUnmet = fmt.Errorf("node not ready: %w", err)
 				return false, nil
 			}
 			if opts.WaitForKubevirt {
 				if err := waitForKubevirtReady(config); err != nil {
+					lastUnmet = fmt.Errorf("kubevirt not ready: %w", err)
 					return false, nil
 				}
 			}
 			if opts.WaitForLonghorn {
 				if err := checkLonghornReady(client, nodeName); err != nil {
+					lastUnmet = fmt.Errorf("longhorn not ready: %w", err)
 					return false, nil
 				}
 			}
+			lastUnmet = nil
 			return true, nil
 		})
 		doneCh <- struct{}{}
@@ -261,6 +267,9 @@ func WaitForKubernetes(agentName string, ps *pubsub.PubSub, stillRunning *time.T
 	})
 	watches = append(watches, alsoWatch...)
 	pubsub.MultiChannelWatch(watches)
+	if nodeReadyErr != nil && lastUnmet != nil {
+		return fmt.Errorf("%w (last unmet condition: %v)", nodeReadyErr, lastUnmet)
+	}
 	return nodeReadyErr
 }
 

pkg/pillar/kubeapi/vitoapiserver.go

@@ -513,6 +513,26 @@ func RolloutDiskToPVC(ctx context.Context, log *base.LogObject, exists bool,
 		return transientf("PVC Upload for pvc:%s attempts to upload image failed, upload pod:%s does not exist", pvcName, cdiUploadPodName)
 	}
 	uploadNodeName := pod.Spec.NodeName
+	// 3b. The upload pod can reach Ready yet get torn down by CDI (pod.phase Failed /
+	// ContainerStatusUnknown) with the local-path scratch PVC left stuck Terminating —
+	// a wedge the data-vol/PV/engine checks below do not surface, so log it explicitly.
+	log.Noticef("RolloutDiskToPVC pvc:%s upload pod:%s phase:%s cdi.pod.phase:%q cdi.running.reason:%q cdi.running.msg:%q",
+		pvcName, cdiUploadPodName, pod.Status.Phase,
+		pvc.ObjectMeta.Annotations["cdi.kubevirt.io/storage.pod.phase"],
+		pvc.ObjectMeta.Annotations["cdi.kubevirt.io/storage.condition.running.reason"],
+		pvc.ObjectMeta.Annotations["cdi.kubevirt.io/storage.condition.running.message"])
+	scratchName := pvcName + "-scratch"
+	if scratchPvc, scratchErr := PVCGet(scratchName, log); scratchErr != nil {
+		log.Noticef("RolloutDiskToPVC pvc:%s scratch PVC %s not found: %v", pvcName, scratchName, scratchErr)
+	} else {
+		scratchSC := ""
+		if scratchPvc.Spec.StorageClassName != nil {
+			scratchSC = *scratchPvc.Spec.StorageClassName
+		}
+		log.Noticef("RolloutDiskToPVC pvc:%s scratch PVC %s phase:%s terminating:%t storageClass:%q finalizers:%v",
+			pvcName, scratchName, scratchPvc.Status.Phase,
+			scratchPvc.ObjectMeta.DeletionTimestamp != nil, scratchSC, scratchPvc.ObjectMeta.Finalizers)
+	}
 	// 4. Did the PVC claim get a backing pv?
 	lhVol, err := lhVolGet(pvName)
 	if err != nil {