diff --git a/pkg/pillar/cmd/domainmgr/domainmgr.go b/pkg/pillar/cmd/domainmgr/domainmgr.go index 8ac5e6b9643..3e3579555e0 100644 --- a/pkg/pillar/cmd/domainmgr/domainmgr.go +++ b/pkg/pillar/cmd/domainmgr/domainmgr.go @@ -719,7 +719,7 @@ func Run(ps *pubsub.PubSub, loggerArg *logrus.Logger, logArg *base.LogObject, ar break } if waitForKubevirtFlag { - log.Warnf("Domainmgr: WaitForKubernetes kubevirt not ready: %v, retrying", err) + log.Warnf("Domainmgr: WaitForKubernetes not satisfied: %v, retrying", err) continue } log.Errorf("Domainmgr: WaitForKubernetes error %v", err) diff --git a/pkg/pillar/kubeapi/kubeapi.go b/pkg/pillar/kubeapi/kubeapi.go index ae8b495e6fb..5e2fbfcab9e 100644 --- a/pkg/pillar/kubeapi/kubeapi.go +++ b/pkg/pillar/kubeapi/kubeapi.go @@ -230,22 +230,28 @@ func WaitForKubernetes(agentName string, ps *pubsub.PubSub, stillRunning *time.T // nodeName is the caller-supplied EVE-k node name (from EdgeNodeInfo.DeviceName), // not os.Hostname(). var nodeReadyErr error + // Record the last failing sub-check; PollImmediate's timeout error alone does not say which of node/kubevirt/longhorn was unmet. + var lastUnmet error doneCh := make(chan struct{}, 1) go func() { nodeReadyErr = wait.PollImmediate(time.Second, time.Minute*20, func() (bool, error) { if err := nodeReadyByName(client, nodeName); err != nil { + lastUnmet = fmt.Errorf("node not ready: %w", err) return false, nil } if opts.WaitForKubevirt { if err := waitForKubevirtReady(config); err != nil { + lastUnmet = fmt.Errorf("kubevirt not ready: %w", err) return false, nil } } if opts.WaitForLonghorn { if err := checkLonghornReady(client, nodeName); err != nil { + lastUnmet = fmt.Errorf("longhorn not ready: %w", err) return false, nil } } + lastUnmet = nil return true, nil }) doneCh <- struct{}{} @@ -261,6 +267,9 @@ func WaitForKubernetes(agentName string, ps *pubsub.PubSub, stillRunning *time.T }) watches = append(watches, alsoWatch...) pubsub.MultiChannelWatch(watches) + if nodeReadyErr != nil && lastUnmet != nil { + return fmt.Errorf("%w (last unmet condition: %v)", nodeReadyErr, lastUnmet) + } return nodeReadyErr } diff --git a/pkg/pillar/kubeapi/vitoapiserver.go b/pkg/pillar/kubeapi/vitoapiserver.go index 78f0292455f..39cb0c311c8 100644 --- a/pkg/pillar/kubeapi/vitoapiserver.go +++ b/pkg/pillar/kubeapi/vitoapiserver.go @@ -513,6 +513,26 @@ func RolloutDiskToPVC(ctx context.Context, log *base.LogObject, exists bool, return transientf("PVC Upload for pvc:%s attempts to upload image failed, upload pod:%s does not exist", pvcName, cdiUploadPodName) } uploadNodeName := pod.Spec.NodeName + // 3b. The upload pod can reach Ready yet get torn down by CDI (pod.phase Failed / + // ContainerStatusUnknown) with the local-path scratch PVC left stuck Terminating — + // a wedge the data-vol/PV/engine checks below do not surface, so log it explicitly. + log.Noticef("RolloutDiskToPVC pvc:%s upload pod:%s phase:%s cdi.pod.phase:%q cdi.running.reason:%q cdi.running.msg:%q", + pvcName, cdiUploadPodName, pod.Status.Phase, + pvc.ObjectMeta.Annotations["cdi.kubevirt.io/storage.pod.phase"], + pvc.ObjectMeta.Annotations["cdi.kubevirt.io/storage.condition.running.reason"], + pvc.ObjectMeta.Annotations["cdi.kubevirt.io/storage.condition.running.message"]) + scratchName := pvcName + "-scratch" + if scratchPvc, scratchErr := PVCGet(scratchName, log); scratchErr != nil { + log.Noticef("RolloutDiskToPVC pvc:%s scratch PVC %s not found: %v", pvcName, scratchName, scratchErr) + } else { + scratchSC := "" + if scratchPvc.Spec.StorageClassName != nil { + scratchSC = *scratchPvc.Spec.StorageClassName + } + log.Noticef("RolloutDiskToPVC pvc:%s scratch PVC %s phase:%s terminating:%t storageClass:%q finalizers:%v", + pvcName, scratchName, scratchPvc.Status.Phase, + scratchPvc.ObjectMeta.DeletionTimestamp != nil, scratchSC, scratchPvc.ObjectMeta.Finalizers) + } // 4. Did the PVC claim get a backing pv? lhVol, err := lhVolGet(pvName) if err != nil {