Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions packages/daemon/src/lib/agent/agent-session.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1053,6 +1053,16 @@ export class AgentSession
return fired;
}

/**
* True only after the cooldown banner's Cancel. Used by the manual Resume
* path to detect a parked, banner-cancelled session whose consumed turn must
* be re-spawned — narrower than the raw pause flag, which is also true while
* an auto-retry is actively starting.
*/
isRateLimitBannerCancelled(): boolean {
return this.rateLimitWatchdog.isRateLimitBannerCancelled();
}

/**
* Get current rate limit watchdog state (for RPC responses).
*/
Expand Down
38 changes: 37 additions & 1 deletion packages/daemon/src/lib/agent/rate-limit-watchdog.ts
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,16 @@ export class RateLimitWatchdog {
* successful start.
*/
private startupExhausted = false;
/**
* True only after the cooldown banner's Cancel (`cancel(notifyResume=false)`)
* — the user stopped the auto-retry but the task must stay paused, so the
* consumed turn is parked and `retryNow` can't re-fire (no timer, not
* startup-exhausted). The manual Resume path uses this banner-only signal
* (NOT `paused`, which is also true mid-`fireCooldownRetry` while an auto-retry
* is actively starting) to decide to re-spawn the execution. Cleared on the
* next pause/resume/new episode/reset.
*/
private bannerCancelled = false;
/**
* The user-message UUID the current episode is tracking. A genuinely new
* user turn (different UUID) starts a fresh episode — clears triedKeys,
Expand Down Expand Up @@ -280,6 +290,7 @@ export class RateLimitWatchdog {
this.retryCount = 0;
this.startupRetries = 0;
this.startupExhausted = false;
this.bannerCancelled = false;
}

// Mark the CURRENT (failed) provider+model as tried so we never re-select it.
Expand Down Expand Up @@ -680,9 +691,14 @@ export class RateLimitWatchdog {
// Cancel via cancelRateLimitRetry). There, resuming would falsely signal
// active work and the ensuing idle transition can be misread as successful
// node completion, advancing the workflow past a failed turn. The banner
// path passes notifyResume=false so the task stays paused/blocked.
// path passes notifyResume=false so the task stays paused/blocked, and sets
// bannerCancelled so a later manual Resume knows the consumed turn is parked
// and re-spawns the execution (paused alone is too broad a signal — see
// isRateLimitBannerCancelled).
if (notifyResume) {
this.notifyResume();
} else {
this.bannerCancelled = true;
}
}

Expand Down Expand Up @@ -771,6 +787,7 @@ export class RateLimitWatchdog {
this.retryCount = 0;
this.startupRetries = 0;
this.startupExhausted = false;
this.bannerCancelled = false;
this.lastUserMessage = null;
this.lastErrorMessage = '';
this.triedKeys.clear();
Expand All @@ -792,8 +809,26 @@ export class RateLimitWatchdog {
return this.cooldownTimer !== null;
}

/**
* True only after the cooldown banner's Cancel (`cancel(false)`). The manual
* Resume path combines this with `retryNow()` returning false to detect a
* parked, banner-cancelled session whose consumed turn must be re-spawned.
*
* This is intentionally a NARROWER signal than `paused`: `paused` is also
* true (with `retryNow()` false) during the brief window after the cooldown
* timer fires while `fireCooldownRetry` is mid-await — i.e. an auto-retry is
* actively starting. Gating the respawn on `paused` would respawn a session
* whose retry is already in flight (wasteful, discards the in-flight retry).
* `bannerCancelled` is set only by the banner path, so the respawn fires only
* for genuinely parked sessions.
*/
isRateLimitBannerCancelled(): boolean {
return this.bannerCancelled;
}

private notifyPause(payload: RateLimitPausePayload): void {
this.paused = true;
this.bannerCancelled = false; // a fresh pause supersedes any prior banner cancel
try {
this.deps.notifyPause?.(payload);
} catch (err) {
Expand All @@ -804,6 +839,7 @@ export class RateLimitWatchdog {
private notifyResume(): void {
if (!this.paused) return;
this.paused = false;
this.bannerCancelled = false; // the pause is resolved
try {
this.deps.notifyResume?.();
} catch (err) {
Expand Down
23 changes: 15 additions & 8 deletions packages/daemon/src/lib/space/runtime/space-runtime.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5711,15 +5711,22 @@ export class SpaceRuntime {
for (const sessionId of liveSessionIds) {
// If the live session is paused in a rate/usage-limit cooldown, break it
// out immediately so the manual Resume re-runs the turn now instead of
// sitting idle until the watchdog timer fires at resetAt.
// sitting idle until the watchdog timer fires at resetAt. A banner-
// cancelled session (cooldown banner's Cancel) can't be broken out via
// retryNow, so resumeRateLimitedSubSession re-spawns its execution; in
// that case the session is stopped and a fresh one is spawned by the next
// tick (with MCP tools attached at spawn), so skip the prepare step.
const tam = this.config.taskAgentManager;
if (tam && typeof tam.resumeRateLimitedSubSession === 'function') {
await tam.resumeRateLimitedSubSession(sessionId).catch((err: unknown) => {
log.warn(
`Workflow resume: failed to resume rate-limited session ${sessionId}: ${err instanceof Error ? err.message : String(err)}`
);
});
}
const resumeOutcome: 'retried' | 'respawned' | 'noop' =
tam && typeof tam.resumeRateLimitedSubSession === 'function'
? await tam.resumeRateLimitedSubSession(sessionId).catch((err: unknown) => {
log.warn(
`Workflow resume: failed to resume rate-limited session ${sessionId}: ${err instanceof Error ? err.message : String(err)}`
);
return 'noop' as const;
})
: 'noop';
if (resumeOutcome === 'respawned') continue;
const prepared =
(await this.config.taskAgentManager?.prepareSubSessionForWorkflowResume(sessionId)) ?? true;
if (!prepared) {
Expand Down
Loading