fix(ops): fail rebuild fast when the referenced backup is missing

pkg/operations/rebuild_instance.go

@@ -596,6 +596,9 @@ func (r rebuildInstanceOpsHandler) prepareInplaceRebuildHelper(reqCtx intctrluti
 		// prepare backup infos
 		backup = &dpv1alpha1.Backup{}
 		if err = cli.Get(reqCtx.Ctx, client.ObjectKey{Name: rebuildInstance.BackupName, Namespace: opsRes.Cluster.Namespace}, backup); err != nil {
+			if apierrors.IsNotFound(err) {
+				return nil, intctrlutil.NewFatalError(fmt.Sprintf(`the backup "%s" is not found`, rebuildInstance.BackupName))
+			}
 			return nil, err
 		}
 		if !slices.Contains([]string{string(dpv1alpha1.BackupTypeFull), string(dpv1alpha1.BackupTypeIncremental)}, backup.Labels[dptypes.BackupTypeLabelKey]) {
@@ -607,6 +610,8 @@ func (r rebuildInstanceOpsHandler) prepareInplaceRebuildHelper(reqCtx intctrluti
 		if backup.Status.BackupMethod == nil {
 			return nil, intctrlutil.NewFatalError(fmt.Sprintf(`the backupMethod of the backup "%s" can not be empty`, rebuildInstance.BackupName))
 		}
+		// NOTE: an ActionSet is a cluster-scoped resource supplied by the addon and can be
+		// temporarily absent (addon installation/upgrade), so a NotFound here stays retryable.
 		actionSet, err = dputils.GetActionSetByName(reqCtx, cli, backup.Status.BackupMethod.ActionSetName)
 		if err != nil {
 			return nil, err

pkg/operations/rebuild_instance_test.go

@@ -869,6 +869,84 @@ var _ = Describe("OpsUtil functions", func() {
 			testRebuildInstanceWithBackup(true)
 		})
 
+		It("test rebuild instance in place when the backup does not exist", func() {
+			By("init operations resources with a nonexistent backup")
+			opsRes := prepareOpsRes("backup-not-exist-"+randomStr, true)
+			_ = testapps.MockInstanceSetComponent(&testCtx, clusterName, defaultCompName)
+			opsRes.OpsRequest.Status.Phase = opsv1alpha1.OpsRunningPhase
+			reqCtx := intctrlutil.RequestCtx{Ctx: testCtx.Ctx}
+			handler := rebuildInstanceOpsHandler{}
+
+			By("expect the missing backup to fail the instances instead of returning a retryable error")
+			_, _, err := handler.ReconcileAction(reqCtx, k8sClient, opsRes)
+			Expect(err).ShouldNot(HaveOccurred())
+			progressDetails := opsRes.OpsRequest.Status.Components[defaultCompName].ProgressDetails
+			Expect(progressDetails).Should(HaveLen(rebuildInstanceCount))
+			for _, detail := range progressDetails {
+				Expect(detail.Status).Should(Equal(opsv1alpha1.FailedProgressStatus))
+				Expect(detail.Message).Should(ContainSubstring("not found"))
+			}
+
+			By("expect the opsRequest to fail on the next reconciliation")
+			opsPhase, _, err := handler.ReconcileAction(reqCtx, k8sClient, opsRes)
+			Expect(err).ShouldNot(HaveOccurred())
+			Expect(opsPhase).Should(Equal(opsv1alpha1.OpsFailedPhase))
+		})
+
+		It("test rebuild instance in place keeps retrying while the actionSet of the backup is absent", func() {
+			By("init operations resources with a backup that references a not-yet-installed actionSet")
+			lateActionSetName := "actionset-late-" + randomStr
+			backup := testdp.NewBackupFactory(testCtx.DefaultNamespace, testdp.BackupName).
+				SetBackupPolicyName(testdp.BackupPolicyName).
+				SetBackupMethod(testdp.BackupMethodName).
+				AddLabels(dptypes.BackupTypeLabelKey, string(dpv1alpha1.BackupTypeFull)).
+				Create(&testCtx).GetObject()
+			Expect(testapps.ChangeObjStatus(&testCtx, backup, func() {
+				backup.Status.Phase = dpv1alpha1.BackupPhaseCompleted
+				backup.Status.BackupMethod = &dpv1alpha1.BackupMethod{
+					Name:          backup.Spec.BackupMethod,
+					ActionSetName: lateActionSetName,
+					TargetVolumes: &dpv1alpha1.TargetVolumeInfo{
+						VolumeMounts: []corev1.VolumeMount{
+							{Name: testapps.DataVolumeName, MountPath: "/test"},
+						},
+					},
+				}
+			})).Should(Succeed())
+			opsRes := prepareOpsRes(backup.Name, true)
+			_ = testapps.MockInstanceSetComponent(&testCtx, clusterName, defaultCompName)
+			opsRes.OpsRequest.Status.Phase = opsv1alpha1.OpsRunningPhase
+			reqCtx := intctrlutil.RequestCtx{Ctx: testCtx.Ctx}
+			handler := rebuildInstanceOpsHandler{}
+
+			By("expect the absent actionSet to be a retryable error instead of failing the instances")
+			opsPhase, _, err := handler.ReconcileAction(reqCtx, k8sClient, opsRes)
+			Expect(err).Should(HaveOccurred())
+			Expect(err.Error()).Should(ContainSubstring("not found"))
+			Expect(intctrlutil.IsTargetError(err, intctrlutil.ErrorTypeFatal)).Should(BeFalse())
+			Expect(opsPhase).Should(Equal(opsv1alpha1.OpsRunningPhase))
+			for _, detail := range opsRes.OpsRequest.Status.Components[defaultCompName].ProgressDetails {
+				Expect(detail.Status).ShouldNot(Equal(opsv1alpha1.FailedProgressStatus))
+			}
+
+			By("expect the rebuild to proceed once the actionSet is installed")
+			actionSet := testapps.CreateCustomizedObj(&testCtx, "backup/actionset.yaml",
+				&dpv1alpha1.ActionSet{}, testapps.WithName(lateActionSetName))
+			Expect(testapps.ChangeObjStatus(&testCtx, actionSet, func() {
+				actionSet.Status.Phase = dpv1alpha1.AvailablePhase
+				actionSet.Status.ObservedGeneration = actionSet.Generation
+			})).Should(Succeed())
+			opsPhase, _, err = handler.ReconcileAction(reqCtx, k8sClient, opsRes)
+			Expect(err).ShouldNot(HaveOccurred())
+			Expect(opsPhase).Should(Equal(opsv1alpha1.OpsRunningPhase))
+			restoreList := &dpv1alpha1.RestoreList{}
+			Expect(k8sClient.List(ctx, restoreList, client.MatchingLabels{
+				constant.OpsRequestNameLabelKey:      opsRes.OpsRequest.Name,
+				constant.OpsRequestNamespaceLabelKey: opsRes.OpsRequest.Namespace,
+			}, client.InNamespace(opsRes.OpsRequest.Namespace))).Should(Succeed())
+			Expect(restoreList.Items).Should(HaveLen(rebuildInstanceCount))
+		})
+
 		It("rebuild instance with horizontal scaling", func() {
 			By("init operations resources ")
 			opsRes, _, _ := initOperationsResources(compDefName, clusterName)