diff --git a/CHANGELOG.md b/CHANGELOG.md index b9d158529..eab67260b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Show periodic ASCII Box cleanup progress and preserve the deletion phase and last observed status on timeout, while retaining incomplete cleanup claims. [PR 1789](https://github.com/openclaw/crabbox/pull/1789). Thanks @shunkakinoki. - Avoid unnecessary Git metadata lookups during configuration loading, lease claim refreshes, and sync planning while preserving repository and credential trust boundaries. [PR 1783](https://github.com/openclaw/crabbox/pull/1783). Thanks @steipete. - Preserve finish-submission and receipt-verification errors, attempt counts, and recovery guidance when terminal run recording times out, without changing retry limits or receipt verification. - Closed and joined lease-owned SSH connection masters after confirmed brokered deletion, preserving native connection reuse and lease/host-key isolation while retaining failed local cleanup for a local-only retry. [PR 1774](https://github.com/openclaw/crabbox/pull/1774). Thanks @steipete. diff --git a/docs/providers/ascii-box.md b/docs/providers/ascii-box.md index b682bbe73..5cc3b2f59 100644 --- a/docs/providers/ascii-box.md +++ b/docs/providers/ascii-box.md @@ -115,6 +115,15 @@ BOX_ORG lookups, and cancellation retain the claim without recording completion. The shared native CLI SSH key is retained. +Cleanup reports its current native-call phase and elapsed time at roughly +ten-second intervals, including while a native command is blocked. A remaining +budget is shown only when that command's context has a deadline; progress does +not extend it or impose a new whole-command timeout. Claim-lock waits and +best-effort remote teardown are outside this native-call progress reporter. +Deletion-wait failures retain the exact operation and its last validated status +in the error. Native command capture is capped at 8 MiB per stream; oversized or +incomplete output is an error, never evidence of completed deletion. + If this release observes a valid native deletion acceptance but cannot finish waiting because of a timeout, cancellation, or operation lookup failure, it durably records the exact operation ID and its claim binding before returning diff --git a/internal/providers/asciibox/backend.go b/internal/providers/asciibox/backend.go index 2e1dfb73a..b5729c417 100644 --- a/internal/providers/asciibox/backend.go +++ b/internal/providers/asciibox/backend.go @@ -111,6 +111,7 @@ func (b *backend) Acquire(ctx context.Context, req AcquireRequest) (LeaseTarget, func (b *backend) rollbackBox(ctx context.Context, client api, leaseID string, box boxData, claim LeaseClaim, exists bool) error { cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), boxReleaseTimeout) defer cancel() + cleanupCtx = withBoxCleanupProgress(cleanupCtx, b.rt.Stderr) if exists { if claim.LeaseID != leaseID || box.ID != box.createdID || !concreteBoxID(box.createdID) { return exit(2, "ascii-box rollback has no matching original publication identity") @@ -139,6 +140,9 @@ func (b *backend) rollbackBox(ctx context.Context, client api, leaseID string, b } func (b *backend) Resolve(ctx context.Context, req ResolveRequest) (LeaseTarget, error) { + if req.ReleaseOnly { + ctx = withBoxCleanupProgress(ctx, b.rt.Stderr) + } cfg, err := b.configForRun() if err != nil { return LeaseTarget{}, err @@ -316,6 +320,7 @@ func (b *backend) ReleaseLease(ctx context.Context, req ReleaseLeaseRequest) err } ctx, cancel := context.WithTimeout(ctx, boxReleaseTimeout) defer cancel() + ctx = withBoxCleanupProgress(ctx, b.rt.Stderr) return releaseClaimedBox(ctx, client, claim, func(box boxData) { if req.GuardedRemoteCleanup != nil { lease := req.Lease diff --git a/internal/providers/asciibox/backend_test.go b/internal/providers/asciibox/backend_test.go index 441c63381..f3c108dc9 100644 --- a/internal/providers/asciibox/backend_test.go +++ b/internal/providers/asciibox/backend_test.go @@ -86,6 +86,11 @@ func TestClientUsesOfficialAsciiBoxCLI(t *testing.T) { if !reflect.DeepEqual(runner.commands, want) { t.Fatalf("commands=%v want=%v", runner.commands, want) } + for _, req := range runner.requests { + if req.MaxCapturedOutputBytes <= 0 || req.MaxCapturedOutputBytes > 8<<20 || req.DisableOutputCapture || req.Stdout != nil || req.Stderr != nil { + t.Fatalf("native command must use bounded, non-streaming capture: %+v", req.Args) + } + } for _, env := range runner.env { if !hasEnv(env, "BOX_API_KEY=box_key") { t.Fatal("child environment missing the synthetic BOX_API_KEY") @@ -286,6 +291,135 @@ func TestReleaseBoxPendingOperationHonorsDeadline(t *testing.T) { } } +func TestReleaseBoxReportsLastDeletionStatusWhenNativeLookupTimesOut(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) + defer cancel() + lookups := 0 + runner := &releaseCommandRunner{configPath: filepath.Join(t.TempDir(), "config.json"), outcomes: map[string][]commandOutcome{ + "stop": {{result: LocalCommandResult{}}}, + "delete": {deletionOutcome(testDeletionID, "bx_guard", "box", "pending")}, + "deletion": { + deletionOutcome(testDeletionID, "bx_guard", "box", "blocked"), + {err: context.DeadlineExceeded}, + }, + }, onAction: func(action string) { + if action == "deletion" { + lookups++ + if lookups == 2 { + <-ctx.Done() + } + } + }} + c := &client{apiKey: "box_key", apiURL: "https://ascii.dev", cliPath: "box", home: t.TempDir(), runner: runner, releasePollInterval: time.Nanosecond} + err := c.ReleaseBox(ctx, "bx_guard", func(context.Context) error { return nil }) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("lost deadline cause: %v", err) + } + for _, want := range []string{"phase=deletion-operation", testDeletionID, "last_observed_status=blocked", "retaining claim"} { + if !strings.Contains(err.Error(), want) { + t.Fatalf("missing %q in %v", want, err) + } + } + var incomplete *boxDeletionIncompleteError + if !errors.As(err, &incomplete) || incomplete.operation.ID != testDeletionID || incomplete.operation.Status != "pending" { + t.Fatalf("lost original accepted operation: %v", err) + } +} + +func TestBoxCleanupProgressReportsDuringNativeCallAndJoins(t *testing.T) { + output := make(boxProgressOutput, 32) + ctx, cancel := context.WithTimeout(withBoxCleanupProgress(context.Background(), output), time.Second) + defer cancel() + ctx.Value(boxCleanupProgressKey{}).(*boxCleanupProgress).interval = 5 * time.Millisecond + entered := make(chan struct{}) + runner := boxCommandRunnerFunc(func(ctx context.Context, req LocalCommandRequest) (LocalCommandResult, error) { + close(entered) + <-ctx.Done() + return LocalCommandResult{Stdout: "native output must not become progress"}, ctx.Err() + }) + c := &client{cliPath: "box", runner: runner} + done := make(chan error, 1) + go func() { _, err := c.runPrepared(ctx, "delete", "bx_guard", "--yes"); done <- err }() + <-entered + for range 2 { + select { + case line := <-output: + if !strings.Contains(line, "phase=native-delete") || !strings.Contains(line, "remaining=") || strings.Contains(line, "native output") { + t.Fatalf("unexpected progress: %s", line) + } + case <-time.After(time.Second): + t.Fatal("no progress while native command was blocked") + } + } + cancel() + if err := <-done; !errors.Is(err, context.Canceled) { + t.Fatalf("native cancellation lost: %v", err) + } + for len(output) > 0 { + <-output + } + select { + case line := <-output: + t.Fatalf("progress after native command returned: %s", line) + case <-time.After(15 * time.Millisecond): + } +} + +func TestBoxCleanupProgressRetainsCadenceAcrossFastPolls(t *testing.T) { + output := make(boxProgressOutput, 32) + ctx := withBoxCleanupProgress(context.Background(), output) + ctx.Value(boxCleanupProgressKey{}).(*boxCleanupProgress).interval = 5 * time.Millisecond + c := &client{cliPath: "box", runner: boxCommandRunnerFunc(func(context.Context, LocalCommandRequest) (LocalCommandResult, error) { + return LocalCommandResult{}, nil + })} + deadline := time.Now().Add(time.Second) + for len(output) < 2 && time.Now().Before(deadline) { + if _, err := c.runPrepared(ctx, "deletion", "status", testDeletionID); err != nil { + t.Fatal(err) + } + time.Sleep(2 * time.Millisecond) + } + if len(output) < 2 { + t.Fatal("fast native calls reset progress cadence") + } + for len(output) > 0 { + if line := <-output; !strings.Contains(line, "phase=deletion-operation") || strings.Contains(line, "remaining=-") { + t.Fatalf("unexpected progress: %s", line) + } + } +} + +func TestNativeCaptureErrorCannotAuthorizeCleanupJSON(t *testing.T) { + for _, action := range []string{"status", "list", "deletion"} { + t.Run(action, func(t *testing.T) { + configPath := filepath.Join(t.TempDir(), "config.json") + runner := boxCommandRunnerFunc(func(_ context.Context, req LocalCommandRequest) (LocalCommandResult, error) { + current := boxCLIAction(req.Args) + result := LocalCommandResult{Stdout: fmt.Sprintf(`{"config":{"path":%q}}`, configPath)} + if current == "list" { + result.Stdout = `{"boxes":[],"pageInfo":{"hasMore":false}}` + } else if current == "deletion" { + result = deletionOutcome(testDeletionID, "bx_guard", "box", "completed").result + } + if current == action { + return result, errors.New("captured command output exceeded limit") + } + return result, nil + }) + c := &client{apiKey: "box_key", apiURL: "https://ascii.dev", cliPath: "box", home: t.TempDir(), runner: runner} + var err error + if action == "deletion" { + _, err = c.GetDeletionOperation(context.Background(), "bx_guard", testDeletionID) + } else { + _, err = c.ListBoxes(context.Background(), true) + } + if err == nil { + t.Fatal("valid-looking JSON from failed capture was accepted") + } + }) + } +} + func TestAsciiBoxBaseURLValidation(t *testing.T) { for _, test := range []struct { name string @@ -865,6 +999,7 @@ func (f *fakeAPI) GetDeletionOperation(_ context.Context, targetID, operationID type fakeCommandRunner struct { commands []string + requests []LocalCommandRequest env [][]string configPath string newStdout string @@ -921,6 +1056,7 @@ func snapshotGuardOutcome() commandOutcome { } func (r *fakeCommandRunner) Run(_ context.Context, req LocalCommandRequest) (LocalCommandResult, error) { + r.requests = append(r.requests, req) r.commands = append(r.commands, strings.Join(append([]string{req.Name}, req.Args...), " ")) r.env = append(r.env, req.Env) joined := strings.Join(req.Args, " ") @@ -960,6 +1096,19 @@ func (r *fakeCommandRunner) Run(_ context.Context, req LocalCommandRequest) (Loc } } +type boxCommandRunnerFunc func(context.Context, LocalCommandRequest) (LocalCommandResult, error) + +func (f boxCommandRunnerFunc) Run(ctx context.Context, req LocalCommandRequest) (LocalCommandResult, error) { + return f(ctx, req) +} + +type boxProgressOutput chan string + +func (out boxProgressOutput) Write(data []byte) (int, error) { + out <- string(data) + return len(data), nil +} + func hasEnv(env []string, want string) bool { for _, value := range env { if value == want { diff --git a/internal/providers/asciibox/cleanup_progress.go b/internal/providers/asciibox/cleanup_progress.go new file mode 100644 index 000000000..2153b5351 --- /dev/null +++ b/internal/providers/asciibox/cleanup_progress.go @@ -0,0 +1,94 @@ +package asciibox + +import ( + "context" + "fmt" + "io" + "sync" + "time" +) + +type boxCleanupProgressKey struct{} +type boxCleanupPhaseKey struct{} + +type boxCleanupProgress struct { + mu sync.Mutex + writer io.Writer + started time.Time + last time.Time + interval time.Duration +} + +func withBoxCleanupProgress(ctx context.Context, writer io.Writer) context.Context { + if writer == nil || ctx.Value(boxCleanupProgressKey{}) != nil { + return ctx + } + return context.WithValue(ctx, boxCleanupProgressKey{}, &boxCleanupProgress{ + writer: writer, started: time.Now(), interval: 10 * time.Second, + }) +} + +func boxCleanupPhaseContext(ctx context.Context, phase string) context.Context { + return context.WithValue(ctx, boxCleanupPhaseKey{}, phase) +} + +func boxCommandPhase(args []string) string { + if len(args) > 0 { + switch args[0] { + case "stop": + return "native-stop" + case "delete": + return "native-delete" + case "extend": + return "snapshot-recovery" + case "deletion": + return "deletion-operation" + } + } + return "configuration" +} + +func startBoxCommandProgress(ctx context.Context, phase string) func() { + progress, _ := ctx.Value(boxCleanupProgressKey{}).(*boxCleanupProgress) + if progress == nil { + return func() {} + } + if override, ok := ctx.Value(boxCleanupPhaseKey{}).(string); ok { + phase = override + } + write := func() { + progress.mu.Lock() + defer progress.mu.Unlock() + now := time.Now() + if ctx.Err() != nil || !progress.last.IsZero() && now.Sub(progress.last) < progress.interval { + return + } + progress.last = now + remaining := "deadline=none" + if deadline, ok := ctx.Deadline(); ok { + remaining = fmt.Sprintf("remaining=%s", max(time.Duration(0), deadline.Sub(now)).Round(time.Second)) + } + fmt.Fprintf(progress.writer, "ascii-box cleanup phase=%s elapsed=%s %s; waiting for native CLI\n", phase, now.Sub(progress.started).Round(time.Second), remaining) + } + // Shared cadence also reports fast polls whose individual timers never fire. + write() + stop := make(chan struct{}) + joined := make(chan struct{}) + go func() { + defer close(joined) + ticker := time.NewTicker(progress.interval) + defer ticker.Stop() + for { + select { + case <-stop: + return + case <-ctx.Done(): + return + case <-ticker.C: + write() + } + } + }() + // Join before returning to core or its guarded-cleanup writer. + return func() { close(stop); <-joined } +} diff --git a/internal/providers/asciibox/client.go b/internal/providers/asciibox/client.go index 4101f0246..bf36fef9f 100644 --- a/internal/providers/asciibox/client.go +++ b/internal/providers/asciibox/client.go @@ -38,6 +38,9 @@ type client struct { releasePollInterval time.Duration } +// Native inventories may be large, but partial output cannot prove cleanup. +const boxCommandOutputLimit = 8 << 20 + type createRequest struct { TTL time.Duration } @@ -190,6 +193,7 @@ func (c *client) PrepareSSH(ctx context.Context, id string) error { } func (c *client) GetBox(ctx context.Context, id string) (boxData, error) { + ctx = boxCleanupPhaseContext(ctx, "ownership-check") result, err := c.run(ctx, "info", id) if err != nil { return boxData{}, fmt.Errorf("ascii-box CLI info failed: %s", c.formatError(result, err)) @@ -202,6 +206,7 @@ func (c *client) GetBox(ctx context.Context, id string) (boxData, error) { } func (c *client) ListBoxes(ctx context.Context, requireComplete bool) ([]boxData, error) { + ctx = boxCleanupPhaseContext(ctx, "inventory-confirmation") result, err := c.run(ctx, "list", "--all") if err != nil { return nil, fmt.Errorf("ascii-box CLI list failed: %s", c.formatError(result, err)) @@ -255,6 +260,7 @@ func (c *client) releaseAfterSnapshotGuard( } recoveryCtx, cancel := context.WithTimeout(ctx, 2*time.Minute) defer cancel() + recoveryCtx = boxCleanupPhaseContext(recoveryCtx, "snapshot-recovery") if err := validate(recoveryCtx); err != nil { return err @@ -384,6 +390,7 @@ func validateBoxDeletionOperation(operation boxDeletionOperation, targetID, oper } func (c *client) GetDeletionOperation(ctx context.Context, targetID, operationID string) (boxDeletionOperation, error) { + ctx = boxCleanupPhaseContext(ctx, "deletion-operation") if !concreteBoxID(targetID) || !boxDeletionIDRE.MatchString(operationID) { return boxDeletionOperation{}, fmt.Errorf("ascii-box deletion lookup requires exact Box and operation IDs") } @@ -408,7 +415,10 @@ func (c *client) waitForDeletion(ctx context.Context, targetID, output string) ( accepted := operation defer func() { if resultErr != nil { - resultErr = &boxDeletionIncompleteError{operation: accepted, err: resultErr} + resultErr = &boxDeletionIncompleteError{operation: accepted, err: fmt.Errorf( + "ascii-box cleanup phase=deletion-operation operation=%s last_observed_status=%s; retaining claim: %w", + accepted.ID, operation.Status, resultErr, + )} } }() operationID := operation.ID @@ -420,25 +430,26 @@ func (c *client) waitForDeletion(ctx context.Context, targetID, output string) ( defer ticker.Stop() for { if err := ctx.Err(); err != nil { - return fmt.Errorf("ascii-box deletion operation %s did not complete; retaining claim: %w", operationID, err) + return err } if operation.Status == "completed" { return nil } select { case <-ctx.Done(): - return fmt.Errorf("ascii-box deletion operation %s did not complete; retaining claim: %w", operationID, ctx.Err()) + return ctx.Err() case <-ticker.C: } if err := ctx.Err(); err != nil { - return fmt.Errorf("ascii-box deletion operation %s did not complete; retaining claim: %w", operationID, err) + return err } // Accepted deletion hides normal Box reads, so poll only its exact // operation. Native exit zero alone can still mean pending or blocked. - operation, err = c.GetDeletionOperation(ctx, targetID, operationID) + nextOperation, err := c.GetDeletionOperation(ctx, targetID, operationID) if err != nil { return err } + operation = nextOperation } } @@ -504,10 +515,13 @@ func (c *client) runPreparedWithEnv(ctx context.Context, env []string, args ...s argv = append(argv, "--api-url", c.apiURL) } argv = append(argv, args...) + stopProgress := startBoxCommandProgress(ctx, boxCommandPhase(args)) + defer stopProgress() return c.runner.Run(ctx, LocalCommandRequest{ - Name: c.cliPath, - Args: argv, - Env: env, + Name: c.cliPath, + Args: argv, + Env: env, + MaxCapturedOutputBytes: boxCommandOutputLimit, }) } @@ -531,11 +545,7 @@ func (c *client) ensureConfig(ctx context.Context) error { ctx, cancel = context.WithTimeout(ctx, 30*time.Second) defer cancel() } - result, err := c.runner.Run(ctx, LocalCommandRequest{ - Name: c.cliPath, - Args: []string{"--no-update", "--json", "--org", blank(c.org, "personal"), "--api-url", c.apiURL, "status"}, - Env: c.env(), - }) + result, err := c.runPrepared(ctx, "status") if err != nil { return fmt.Errorf("ascii-box CLI status failed: %s", c.formatError(result, err)) } diff --git a/internal/providers/asciibox/ownership.go b/internal/providers/asciibox/ownership.go index 870445dcc..1f38f6b99 100644 --- a/internal/providers/asciibox/ownership.go +++ b/internal/providers/asciibox/ownership.go @@ -284,14 +284,14 @@ func releaseExactBox(ctx context.Context, client api, expected boxData, beforeRe // native deletion completion followed by complete inventory is finalization. for { if err := ctx.Err(); err != nil { - return err + return fmt.Errorf("ascii-box cleanup phase=inventory-confirmation; retaining claim: %w", err) } boxes, err := client.ListBoxes(ctx, true) - if err != nil { - return fmt.Errorf("ascii-box deletion confirmation; retaining claim: %w", err) + if ctxErr := ctx.Err(); ctxErr != nil { + return fmt.Errorf("ascii-box cleanup phase=inventory-confirmation; retaining claim: %w", ctxErr) } - if err := ctx.Err(); err != nil { - return err + if err != nil { + return fmt.Errorf("ascii-box cleanup phase=inventory-confirmation; retaining claim: %w", err) } found := false for _, box := range boxes { @@ -307,7 +307,7 @@ func releaseExactBox(ctx context.Context, client api, expected boxData, beforeRe } select { case <-ctx.Done(): - return ctx.Err() + return fmt.Errorf("ascii-box cleanup phase=inventory-confirmation; retaining claim: %w", ctx.Err()) case <-time.After(250 * time.Millisecond): } } @@ -320,16 +320,16 @@ func exactBoxForRelease(ctx context.Context, client api, expected boxData) (boxD if expected.deletionOperationID != "" { operation, err := client.GetDeletionOperation(ctx, expected.ID, expected.deletionOperationID) if ctxErr := ctx.Err(); ctxErr != nil { - return boxData{}, false, ctxErr + return boxData{}, false, fmt.Errorf("ascii-box cleanup phase=deletion-operation; retaining claim: %w", ctxErr) } if err != nil { - return boxData{}, false, fmt.Errorf("ascii-box deletion operation lookup; retaining claim: %w", err) + return boxData{}, false, fmt.Errorf("ascii-box cleanup phase=deletion-operation lookup; retaining claim: %w", err) } if err := validateBoxDeletionOperation(operation, expected.ID, expected.deletionOperationID); err != nil { return boxData{}, false, err } if operation.Status != "completed" { - return boxData{}, false, exit(2, "ascii-box deletion operation %s is %s; retaining claim", operation.ID, operation.Status) + return boxData{}, false, exit(2, "ascii-box cleanup phase=deletion-operation operation=%s last_observed_status=%s; retaining claim", operation.ID, operation.Status) } // Recheck the recorded operation inside the release fence; a reference // or an earlier resolution read is not completion authority. @@ -353,10 +353,10 @@ func exactBoxForRelease(ctx context.Context, client api, expected boxData) (boxD } boxes, listErr := client.ListBoxes(ctx, true) if ctxErr := ctx.Err(); ctxErr != nil { - return boxData{}, false, ctxErr + return boxData{}, false, fmt.Errorf("ascii-box cleanup phase=inventory-confirmation; retaining claim: %w", ctxErr) } if listErr != nil { - return boxData{}, false, fmt.Errorf("ascii-box absence confirmation; retaining claim: %w", listErr) + return boxData{}, false, fmt.Errorf("ascii-box cleanup phase=inventory-confirmation; retaining claim: %w", listErr) } for _, box := range boxes { if box.ID != expected.ID { diff --git a/internal/providers/asciibox/ownership_test.go b/internal/providers/asciibox/ownership_test.go index b6d70f2a1..4827a97db 100644 --- a/internal/providers/asciibox/ownership_test.go +++ b/internal/providers/asciibox/ownership_test.go @@ -312,8 +312,8 @@ func TestReleasePendingDeletionSurvivesTimeoutAndRetries(t *testing.T) { pending := assertPendingDeletionRetained(t, claim, testDeletionID) runner.outcomes["deletion"] = []commandOutcome{deletionOutcome(testDeletionID, claim.CloudID, "box", "blocked")} commandCount := len(runner.commands) - if _, err := b.Resolve(context.Background(), ResolveRequest{ID: claim.LeaseID, ReleaseOnly: true}); err == nil { - t.Fatal("blocked operation was treated as completed") + if _, err := b.Resolve(context.Background(), ResolveRequest{ID: claim.LeaseID, ReleaseOnly: true}); err == nil || !strings.Contains(err.Error(), "phase=deletion-operation") || !strings.Contains(err.Error(), "last_observed_status=blocked") { + t.Fatalf("blocked operation lost its diagnostic or was treated as completed: %v", err) } assertClaimRetained(t, pending) for _, command := range runner.commands[commandCount:] { @@ -472,7 +472,7 @@ func TestReleaseRejectsCancellationDuringAbsenceCheck(t *testing.T) { cancel() return []boxData{}, nil } - if err := b.ReleaseLease(ctx, ReleaseLeaseRequest{Lease: lease}); !errors.Is(err, context.Canceled) { + if err := b.ReleaseLease(ctx, ReleaseLeaseRequest{Lease: lease}); !errors.Is(err, context.Canceled) || !strings.Contains(err.Error(), "phase=inventory-confirmation") { t.Fatalf("release err=%v, want cancellation", err) } assertClaimRetained(t, claim) @@ -499,27 +499,34 @@ func TestReleaseRetriesCompletedDeletionAfterConfirmationFailure(t *testing.T) { } func TestReleaseRecordsCompletionWhenConfirmationCanceled(t *testing.T) { - b, f, claim, lease := ownedFixture(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - f.listHook = func() ([]boxData, error) { - cancel() - return []boxData{}, nil - } - if err := b.ReleaseLease(ctx, ReleaseLeaseRequest{Lease: lease}); !errors.Is(err, context.Canceled) { - t.Fatalf("release err=%v, want canceled confirmation", err) - } - assertCompletedDeletionRetained(t, claim) - f.listHook = nil - lease, err := b.Resolve(context.Background(), ResolveRequest{ID: claim.LeaseID, ReleaseOnly: true}) - if err != nil { - t.Fatal(err) - } - if err := b.ReleaseLease(context.Background(), ReleaseLeaseRequest{Lease: lease}); err != nil { - t.Fatal(err) - } - if len(f.deletedIDs) != 1 { - t.Fatalf("retry duplicated native deletion: %v", f.deletedIDs) + for _, failedResponse := range []bool{false, true} { + t.Run(fmt.Sprintf("failed-response=%t", failedResponse), func(t *testing.T) { + b, f, claim, lease := ownedFixture(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + f.listHook = func() ([]boxData, error) { + cancel() + if failedResponse { + return nil, errors.New("native inventory call failed after cancellation") + } + return []boxData{}, nil + } + if err := b.ReleaseLease(ctx, ReleaseLeaseRequest{Lease: lease}); !errors.Is(err, context.Canceled) || !strings.Contains(err.Error(), "phase=inventory-confirmation") { + t.Fatalf("release err=%v, want canceled confirmation", err) + } + assertCompletedDeletionRetained(t, claim) + f.listHook = nil + lease, err := b.Resolve(context.Background(), ResolveRequest{ID: claim.LeaseID, ReleaseOnly: true}) + if err != nil { + t.Fatal(err) + } + if err := b.ReleaseLease(context.Background(), ReleaseLeaseRequest{Lease: lease}); err != nil { + t.Fatal(err) + } + if len(f.deletedIDs) != 1 { + t.Fatalf("retry duplicated native deletion: %v", f.deletedIDs) + } + }) } }