diff --git a/pkg/operations/rebuild_instance.go b/pkg/operations/rebuild_instance.go index 7721f03a788..4010d59961f 100644 --- a/pkg/operations/rebuild_instance.go +++ b/pkg/operations/rebuild_instance.go @@ -596,6 +596,9 @@ func (r rebuildInstanceOpsHandler) prepareInplaceRebuildHelper(reqCtx intctrluti // prepare backup infos backup = &dpv1alpha1.Backup{} if err = cli.Get(reqCtx.Ctx, client.ObjectKey{Name: rebuildInstance.BackupName, Namespace: opsRes.Cluster.Namespace}, backup); err != nil { + if apierrors.IsNotFound(err) { + return nil, intctrlutil.NewFatalError(fmt.Sprintf(`the backup "%s" is not found`, rebuildInstance.BackupName)) + } return nil, err } if !slices.Contains([]string{string(dpv1alpha1.BackupTypeFull), string(dpv1alpha1.BackupTypeIncremental)}, backup.Labels[dptypes.BackupTypeLabelKey]) { @@ -607,6 +610,8 @@ func (r rebuildInstanceOpsHandler) prepareInplaceRebuildHelper(reqCtx intctrluti if backup.Status.BackupMethod == nil { return nil, intctrlutil.NewFatalError(fmt.Sprintf(`the backupMethod of the backup "%s" can not be empty`, rebuildInstance.BackupName)) } + // NOTE: an ActionSet is a cluster-scoped resource supplied by the addon and can be + // temporarily absent (addon installation/upgrade), so a NotFound here stays retryable. actionSet, err = dputils.GetActionSetByName(reqCtx, cli, backup.Status.BackupMethod.ActionSetName) if err != nil { return nil, err diff --git a/pkg/operations/rebuild_instance_test.go b/pkg/operations/rebuild_instance_test.go index 5712917f7be..361f128a83b 100644 --- a/pkg/operations/rebuild_instance_test.go +++ b/pkg/operations/rebuild_instance_test.go @@ -869,6 +869,84 @@ var _ = Describe("OpsUtil functions", func() { testRebuildInstanceWithBackup(true) }) + It("test rebuild instance in place when the backup does not exist", func() { + By("init operations resources with a nonexistent backup") + opsRes := prepareOpsRes("backup-not-exist-"+randomStr, true) + _ = testapps.MockInstanceSetComponent(&testCtx, clusterName, defaultCompName) + opsRes.OpsRequest.Status.Phase = opsv1alpha1.OpsRunningPhase + reqCtx := intctrlutil.RequestCtx{Ctx: testCtx.Ctx} + handler := rebuildInstanceOpsHandler{} + + By("expect the missing backup to fail the instances instead of returning a retryable error") + _, _, err := handler.ReconcileAction(reqCtx, k8sClient, opsRes) + Expect(err).ShouldNot(HaveOccurred()) + progressDetails := opsRes.OpsRequest.Status.Components[defaultCompName].ProgressDetails + Expect(progressDetails).Should(HaveLen(rebuildInstanceCount)) + for _, detail := range progressDetails { + Expect(detail.Status).Should(Equal(opsv1alpha1.FailedProgressStatus)) + Expect(detail.Message).Should(ContainSubstring("not found")) + } + + By("expect the opsRequest to fail on the next reconciliation") + opsPhase, _, err := handler.ReconcileAction(reqCtx, k8sClient, opsRes) + Expect(err).ShouldNot(HaveOccurred()) + Expect(opsPhase).Should(Equal(opsv1alpha1.OpsFailedPhase)) + }) + + It("test rebuild instance in place keeps retrying while the actionSet of the backup is absent", func() { + By("init operations resources with a backup that references a not-yet-installed actionSet") + lateActionSetName := "actionset-late-" + randomStr + backup := testdp.NewBackupFactory(testCtx.DefaultNamespace, testdp.BackupName). + SetBackupPolicyName(testdp.BackupPolicyName). + SetBackupMethod(testdp.BackupMethodName). + AddLabels(dptypes.BackupTypeLabelKey, string(dpv1alpha1.BackupTypeFull)). + Create(&testCtx).GetObject() + Expect(testapps.ChangeObjStatus(&testCtx, backup, func() { + backup.Status.Phase = dpv1alpha1.BackupPhaseCompleted + backup.Status.BackupMethod = &dpv1alpha1.BackupMethod{ + Name: backup.Spec.BackupMethod, + ActionSetName: lateActionSetName, + TargetVolumes: &dpv1alpha1.TargetVolumeInfo{ + VolumeMounts: []corev1.VolumeMount{ + {Name: testapps.DataVolumeName, MountPath: "/test"}, + }, + }, + } + })).Should(Succeed()) + opsRes := prepareOpsRes(backup.Name, true) + _ = testapps.MockInstanceSetComponent(&testCtx, clusterName, defaultCompName) + opsRes.OpsRequest.Status.Phase = opsv1alpha1.OpsRunningPhase + reqCtx := intctrlutil.RequestCtx{Ctx: testCtx.Ctx} + handler := rebuildInstanceOpsHandler{} + + By("expect the absent actionSet to be a retryable error instead of failing the instances") + opsPhase, _, err := handler.ReconcileAction(reqCtx, k8sClient, opsRes) + Expect(err).Should(HaveOccurred()) + Expect(err.Error()).Should(ContainSubstring("not found")) + Expect(intctrlutil.IsTargetError(err, intctrlutil.ErrorTypeFatal)).Should(BeFalse()) + Expect(opsPhase).Should(Equal(opsv1alpha1.OpsRunningPhase)) + for _, detail := range opsRes.OpsRequest.Status.Components[defaultCompName].ProgressDetails { + Expect(detail.Status).ShouldNot(Equal(opsv1alpha1.FailedProgressStatus)) + } + + By("expect the rebuild to proceed once the actionSet is installed") + actionSet := testapps.CreateCustomizedObj(&testCtx, "backup/actionset.yaml", + &dpv1alpha1.ActionSet{}, testapps.WithName(lateActionSetName)) + Expect(testapps.ChangeObjStatus(&testCtx, actionSet, func() { + actionSet.Status.Phase = dpv1alpha1.AvailablePhase + actionSet.Status.ObservedGeneration = actionSet.Generation + })).Should(Succeed()) + opsPhase, _, err = handler.ReconcileAction(reqCtx, k8sClient, opsRes) + Expect(err).ShouldNot(HaveOccurred()) + Expect(opsPhase).Should(Equal(opsv1alpha1.OpsRunningPhase)) + restoreList := &dpv1alpha1.RestoreList{} + Expect(k8sClient.List(ctx, restoreList, client.MatchingLabels{ + constant.OpsRequestNameLabelKey: opsRes.OpsRequest.Name, + constant.OpsRequestNamespaceLabelKey: opsRes.OpsRequest.Namespace, + }, client.InNamespace(opsRes.OpsRequest.Namespace))).Should(Succeed()) + Expect(restoreList.Items).Should(HaveLen(rebuildInstanceCount)) + }) + It("rebuild instance with horizontal scaling", func() { By("init operations resources ") opsRes, _, _ := initOperationsResources(compDefName, clusterName)