diff --git a/packages/daemon/scripts/recover-messages.ts b/packages/daemon/scripts/recover-messages.ts index ea28b30ae2..ef3b196cc6 100644 --- a/packages/daemon/scripts/recover-messages.ts +++ b/packages/daemon/scripts/recover-messages.ts @@ -20,6 +20,7 @@ import { readFileSync } from 'fs'; import { Database } from '../src/storage/sqlite-compat'; +import { SDKMessageRepository } from '../src/storage/repositories/sdk-message-repository'; const dbPath = process.argv[2]; @@ -159,8 +160,15 @@ const insertMessage = db.prepare(` `); let messagesInserted = 0; +// Every matched session is recomputed on every run, even if this invocation +// inserts no new rows for it: a prior interrupted run may have inserted rows +// but exited before Step 6, leaving the counter stale, and the existing-UUID +// `continue` below would then skip the session entirely. recomputeVisibleMessageCount +// is idempotent, so recomputing all matched sessions each run is safe. +const touchedSessions = new Set(); for (const [kaiSessionId, msgs] of messagesByKaiSession.entries()) { + touchedSessions.add(kaiSessionId); for (const msg of msgs) { if (existingMessageIds.has(msg.uuid)) continue; @@ -280,6 +288,7 @@ for (const [sdkSessionId, msgs] of orphansBySDKSession.entries()) { null ); orphanSessionsCreated++; + touchedSessions.add(newSessionId); // Insert messages for this new session for (const msg of msgs) { @@ -306,6 +315,25 @@ for (const [sdkSessionId, msgs] of orphansBySDKSession.entries()) { console.log(` Orphan sessions created: ${orphanSessionsCreated}`); console.log(` Orphan messages restored: ${orphanMessagesInserted}`); +// Step 6: Recompute visible_message_count for every touched session. +console.log('\nStep 6: Recomputing visible_message_count for recovered sessions...'); +// The inserts above bypass SDKMessageRepository, so reuse its shared badge +// predicate to recompute the maintained counter (no-op on a schema that +// doesn't carry the visible_message_count column yet — the migration backfill +// covers that). Deliberately no hardcoded migration number here so this comment +// survives future renumbers. +const recoverRepo = new SDKMessageRepository(db); +let recomputed = 0; +for (const sid of touchedSessions) { + try { + recoverRepo.recomputeVisibleMessageCount(sid); + recomputed++; + } catch { + // Skip recompute errors + } +} +console.log(` Recomputed ${recomputed} session(s)`); + // Final summary const finalSessions = db.query('SELECT COUNT(*) as count FROM sessions').get() as { count: number }; const finalMessages = db.query('SELECT COUNT(*) as count FROM sdk_messages').get() as { diff --git a/packages/daemon/src/lib/rpc-handlers/live-query-handlers.ts b/packages/daemon/src/lib/rpc-handlers/live-query-handlers.ts index e6a7ed65f9..a977bacbe5 100644 --- a/packages/daemon/src/lib/rpc-handlers/live-query-handlers.ts +++ b/packages/daemon/src/lib/rpc-handlers/live-query-handlers.ts @@ -2781,8 +2781,11 @@ function mapSessionRow(row: Record): Record { /** * Render-hidden rows excluded before applying transcript pagination limits. - * Shared by `messages.bySession` and the `spaceSessions.bySpace` messageCount - * subquery so the unread count only reflects visible top-level rows. + * Used by `messages.bySession` to cap the visible transcript window. The + * `spaceSessions.bySpace` badge count now reads the maintained + * `sessions.visible_message_count` column instead of a correlated COUNT(*), but + * the same visibility predicate is enforced there incrementally by + * SDKMessageRepository (and backfilled by migration 177). */ const EXCLUDED_FROM_PAGINATION_SQL_LIST = toSqlStringList([ ...HIDDEN_SYSTEM_SUBTYPES, @@ -2795,14 +2798,12 @@ SELECT s.title as title, s.status as status, s.processing_state as processingState, - -- Mirror the messages.bySession top-level visibility predicate so the count - -- reflects what the user actually sees: top-level rows only (subagent rows - -- ride on their parent turn), non-deferred user rows, and non-hidden - -- subtypes. A best-effort approximation of the visible transcript. - (SELECT COUNT(*) FROM sdk_messages sm WHERE sm.session_id = s.id - AND sm.parent_tool_use_id IS NULL - AND (sm.message_type != 'user' OR COALESCE(sm.send_status, 'consumed') IN ('consumed', 'failed')) - AND sm.message_subtype_norm NOT IN (${EXCLUDED_FROM_PAGINATION_SQL_LIST})) as messageCount, + -- Read the maintained counter directly instead of a correlated COUNT(*) over + -- sdk_messages per session (previously ~92ms warm for dev-neokai, re-run every + -- 150ms debounce). SDKMessageRepository keeps this in sync with the same + -- visibility predicate (top-level rows, non-deferred user rows, non-hidden + -- subtypes) on every sdk_messages mutation. + s.visible_message_count as messageCount, (unixepoch(s.last_active_at) - 0) * 1000 as lastActiveAt FROM sessions s INNER JOIN spaces sp ON sp.id = ? @@ -2817,9 +2818,9 @@ ORDER BY s.last_active_at DESC, s.id DESC * `processingState` is the persisted JSON-serialised `AgentProcessingState` * (mirrors `mapSessionRow` for the global sessions list); the web client parses * it via `session-status.ts`'s `parseProcessingState`. `messageCount` is the - * per-session visible SDK message total (deferred/enqueued user rows are - * excluded, matching the `messages.bySession` transcript view), coerced to a - * number so the sidebar can drive an unread badge the same way global chat + * maintained `sessions.visible_message_count` counter (deferred/enqueued user + * rows are excluded, matching the `messages.bySession` transcript view), coerced + * to a number so the sidebar can drive an unread badge the same way global chat * sessions do. */ function mapSpaceSessionRow(row: Record): Record { diff --git a/packages/daemon/src/lib/space/runtime/space-runtime.ts b/packages/daemon/src/lib/space/runtime/space-runtime.ts index 3968b8cc72..8cc363566d 100644 --- a/packages/daemon/src/lib/space/runtime/space-runtime.ts +++ b/packages/daemon/src/lib/space/runtime/space-runtime.ts @@ -1103,7 +1103,7 @@ export class SpaceRuntime { */ private getSdkMessageRepo(): SDKMessageRepository { if (!this.sdkMessageRepo) { - this.sdkMessageRepo = new SDKMessageRepository(this.config.db); + this.sdkMessageRepo = new SDKMessageRepository(this.config.db, this.config.reactiveDb); } return this.sdkMessageRepo; } diff --git a/packages/daemon/src/storage/index.ts b/packages/daemon/src/storage/index.ts index ecf74c7b50..8cb6f5ef14 100644 --- a/packages/daemon/src/storage/index.ts +++ b/packages/daemon/src/storage/index.ts @@ -140,7 +140,7 @@ export class Database { this.shortIdAllocator = new ShortIdAllocator(db); const shortIdAllocator = this.shortIdAllocator; this.sessionRepo = new SessionRepository(db); - this.sdkMessageRepo = new SDKMessageRepository(db); + this.sdkMessageRepo = new SDKMessageRepository(db, reactiveDb); this.settingsRepo = new SettingsRepository(db); this.githubMappingRepo = new GitHubMappingRepository(db); this.inboxItemRepo = new InboxItemRepository(db); diff --git a/packages/daemon/src/storage/reactive-database.ts b/packages/daemon/src/storage/reactive-database.ts index 0f21075b50..4cfff79ad5 100644 --- a/packages/daemon/src/storage/reactive-database.ts +++ b/packages/daemon/src/storage/reactive-database.ts @@ -68,8 +68,11 @@ export interface ReactiveDatabase { /** * Manually notify that a table has changed. * Used for tables whose writes bypass the proxy (e.g., direct SQL via external repos). + * Pass a `scope` when known so the change can batch cleanly inside a reactive + * transaction — an unscoped notify forces the whole flushed batch to undefined + * scope, poisoning properly-scoped writes in the same batch. */ - notifyChange(table: string): void; + notifyChange(table: string, scope?: TableChangeScope): void; } // Mapping from facade method name to table name + optional scope extractor. @@ -408,8 +411,8 @@ export function createReactiveDatabase(db: Database): ReactiveDatabase { pendingTableScopes.clear(); } }, - notifyChange(table: string): void { - incrementAndEmit(table); + notifyChange(table: string, scope?: TableChangeScope): void { + incrementAndEmit(table, scope); }, }; return reactiveDb as ReactiveDatabase; diff --git a/packages/daemon/src/storage/repositories/sdk-message-repository.ts b/packages/daemon/src/storage/repositories/sdk-message-repository.ts index d516f66677..ef76f373f9 100644 --- a/packages/daemon/src/storage/repositories/sdk-message-repository.ts +++ b/packages/daemon/src/storage/repositories/sdk-message-repository.ts @@ -12,6 +12,7 @@ import { generateUUID } from '@hyperneo/shared'; import type { MessageOrigin, HyperNeoActionMessage, ChatMessage } from '@hyperneo/shared'; import type { SDKMessage } from '@hyperneo/shared/sdk'; import { HIDDEN_SYSTEM_SUBTYPES } from '@hyperneo/shared/sdk/type-guards'; +import type { ReactiveDatabase } from '../reactive-database'; import { Logger } from '../../lib/logger'; import { buildFtsQuery, @@ -56,6 +57,41 @@ const EXCLUDED_FROM_LAST_MESSAGE_SQL_LIST = toSqlStringList([ 'model_refusal_fallback', ]); +/** + * Subtypes excluded from the space-sessions visible-message badge — the same + * set the former `spaceSessions.bySpace` correlated COUNT(*) subquery dropped. + * Used by {@link isVisibleBadgeRow} so the maintained + * `sessions.visible_message_count` counter can never drift from the predicate + * it replaces. + */ +const BADGE_HIDDEN_SUBTYPES = new Set([...HIDDEN_SYSTEM_SUBTYPES, 'thinking_tokens']); + +/** + * Does a row with these persisted column values count toward the space-sessions + * visible-message badge? Mirrors the predicate the `spaceSessions.bySpace` + * correlated subquery evaluated inline: top-level only (no `parent_tool_use_id`), + * non-deferred user rows (`consumed`/`failed`), and non-hidden subtypes. + * + * Pure function of the columns as stored, so {@link SDKMessageRepository} can + * decide at INSERT time whether to increment the maintained counter without + * re-querying the row. + */ +function isVisibleBadgeRow(opts: { + parentToolUseId: string | null; + messageType: string; + messageSubtype: string | null; + sendStatus: SendStatus | null; +}): boolean { + if (opts.parentToolUseId !== null) return false; + if (BADGE_HIDDEN_SUBTYPES.has(opts.messageSubtype ?? '')) return false; + if (opts.messageType === 'user') { + // NULL send_status (SDK/action rows) coalesces to 'consumed' — visible. + const status = opts.sendStatus ?? 'consumed'; + return status === 'consumed' || status === 'failed'; + } + return true; +} + function isOlderThanMessageSearchTtl(value: string | number | null | undefined): boolean { if (value === null || value === undefined) return false; const timestamp = typeof value === 'number' ? value : Date.parse(value); @@ -163,7 +199,10 @@ export function extractReplacementEdges(message: SDKMessage): SDKMessageReplacem export class SDKMessageRepository { private logger = new Logger('Database'); - constructor(private db: BunDatabase) {} + constructor( + private db: BunDatabase, + private reactiveDb?: ReactiveDatabase + ) {} private hasMessageSearchIndex(): boolean { return this.tableExists('message_search_content'); @@ -487,6 +526,13 @@ export class SDKMessageRepository { const messageSubtype = 'subtype' in message ? (message.subtype as string) : null; const timestamp = new Date().toISOString(); const taskId = this.resolveTaskIdForSession(sessionId); + const parentToolUseId = extractParentToolUseId(message); + const countsTowardsBadge = isVisibleBadgeRow({ + parentToolUseId, + messageType, + messageSubtype, + sendStatus: null, + }); const stmt = this.db.prepare( `INSERT INTO sdk_messages ( @@ -507,12 +553,16 @@ export class SDKMessageRepository { origin ?? null, computeIsRenderable(message), computeIsTerminal(message), - extractParentToolUseId(message), + parentToolUseId, taskId, ]; stmt.run(...values, extractSdkUuid(message)); this.saveReplacementEdges(id, sessionId, taskId, message); + if (countsTowardsBadge) this.bumpVisibleMessageCount(sessionId, 1); })(); + // Notify before the fallible search-index work so an FTS throw can't strand + // the badge update — the counter is already committed with the tx above. + if (countsTowardsBadge) this.notifySessionsChanged(sessionId); this.deleteSupersededMessageSearchRows(sessionId, message); this.upsertMessageSearchRow(id); return true; @@ -1052,6 +1102,89 @@ export class SDKMessageRepository { return result.count; } + // --------------------------------------------------------------------------- + // sessions.visible_message_count maintenance + // --------------------------------------------------------------------------- + // + // `visible_message_count` is a maintained counter that lets + // `spaceSessions.bySpace` read the badge count directly instead of running a + // correlated COUNT(*) over sdk_messages for every session on every poll. The + // predicate mirrors {@link isVisibleBadgeRow} (and the former subquery): + // top-level rows, non-deferred user rows (consumed/failed), non-hidden + // subtypes. + // + // INSERT paths increment by visibility (O(1)); structural mutations that can + // flip or remove visible rows (send_status transitions, rewind deletes) + // recompute the affected session(s) authoritatively from sdk_messages. + + private visibleMessageCountReady: boolean | null = null; + + /** True only on a schema that carries the column (post-migration / fresh). */ + private supportsVisibleMessageCount(): boolean { + if (this.visibleMessageCountReady === null) { + this.visibleMessageCountReady = + this.tableExists('sessions') && this.tableHasColumn('sessions', 'visible_message_count'); + } + return this.visibleMessageCountReady; + } + + /** Adjust the counter by `delta` for one session (no-op if unsupported). */ + private bumpVisibleMessageCount(sessionId: string, delta: number): void { + if (delta === 0 || !this.supportsVisibleMessageCount()) return; + this.db + .prepare(`UPDATE sessions SET visible_message_count = visible_message_count + ? WHERE id = ?`) + .run(delta, sessionId); + } + + /** + * Recompute the counter for one session from its current sdk_messages rows. + * Used after mutations that can change visibility in bulk (send_status + * transitions, rewind deletes) where an incremental delta would be fragile. + * Also the shared entry point for callers that bypass the repository and write + * `sdk_messages` directly (e.g. `scripts/recover-messages.ts`) — so they reuse + * this predicate instead of re-literalizing it. Returns true if the counter + * actually changed (so callers can gate the + * reactive notification). + */ + recomputeVisibleMessageCount(sessionId: string): boolean { + if (!this.supportsVisibleMessageCount()) return false; + const row = this.db + .prepare( + `SELECT COUNT(*) AS n FROM sdk_messages + WHERE session_id = ? + AND parent_tool_use_id IS NULL + AND (message_type != 'user' + OR COALESCE(send_status, 'consumed') IN ('consumed', 'failed')) + AND COALESCE(message_subtype, '') NOT IN (${EXCLUDED_FROM_PAGINATION_SQL_LIST})` + ) + .get(sessionId) as { n: number } | undefined; + const count = row?.n ?? 0; + // Only write (and report a change) when the value actually differs. + const result = this.db + .prepare( + `UPDATE sessions SET visible_message_count = ? WHERE id = ? AND visible_message_count != ?` + ) + .run(count, sessionId, count); + return result.changes > 0; + } + + /** + * Notify the reactive layer that a session's visible-message counter changed. + * `spaceSessions.bySpace` depends on `sessions` (not `sdk_messages` — the + * correlated COUNT(*) is gone), so without this the live badge would never + * re-evaluate when messages arrive. Called AFTER the mutation transaction + * commits so re-evaluation sees committed state; the LiveQuery debounce + * coalesces bursts during a streaming turn. + * + * The `sessionId` scope is mandatory: without it the `sessions` change would + * be unscoped, and when batched inside a reactive transaction with a scoped + * `sdk_messages` write it would force the whole flushed batch to undefined + * scope (see `flushPendingTables`), poisoning scope filtering. + */ + private notifySessionsChanged(sessionId: string): void { + this.reactiveDb?.notifyChange('sessions', { sessionId }); + } + // ============================================================================ // Message Query Mode operations // ============================================================================ @@ -1082,6 +1215,13 @@ export class SDKMessageRepository { const messageSubtype = 'subtype' in message ? (message.subtype as string) : null; const timestamp = new Date().toISOString(); const taskId = this.resolveTaskIdForSession(sessionId); + const parentToolUseId = extractParentToolUseId(message); + const countsTowardsBadge = isVisibleBadgeRow({ + parentToolUseId, + messageType, + messageSubtype, + sendStatus, + }); const stmt = this.db.prepare( `INSERT INTO sdk_messages ( @@ -1103,12 +1243,14 @@ export class SDKMessageRepository { origin ?? null, computeIsRenderable(message), computeIsTerminal(message), - extractParentToolUseId(message), + parentToolUseId, taskId, ]; stmt.run(...values, extractSdkUuid(message)); this.saveReplacementEdges(id, sessionId, taskId, message); + if (countsTowardsBadge) this.bumpVisibleMessageCount(sessionId, 1); })(); + if (countsTowardsBadge) this.notifySessionsChanged(sessionId); this.upsertMessageSearchRow(id); return id; } @@ -1200,10 +1342,33 @@ export class SDKMessageRepository { // Use parameterized query to prevent SQL injection const placeholders = messageIds.map(() => '?').join(','); + // A send_status transition can flip a user row's badge visibility + // (deferred/enqueued -> consumed/failed), so capture the affected sessions + // and recompute their counters after the update. + const affectedSessions = this.supportsVisibleMessageCount() + ? (this.db + .prepare( + `SELECT DISTINCT session_id AS sid FROM sdk_messages WHERE id IN (${placeholders})` + ) + .all(...messageIds) as Array<{ sid: string }>) + : []; const stmt = this.db.prepare( `UPDATE sdk_messages SET send_status = ? WHERE id IN (${placeholders})` ); - stmt.run(newStatus, ...messageIds); + // Wrap the status update + counter recompute in one transaction so an FTS + // throw (upsertMessageSearchRow, below) can't leave the counter stale. FTS + // stays outside the transaction (best-effort, as today). + const changedSessions: string[] = []; + this.db.transaction(() => { + stmt.run(newStatus, ...messageIds); + for (const { sid } of affectedSessions) { + if (this.recomputeVisibleMessageCount(sid)) changedSessions.push(sid); + } + })(); + // Notify per session (a status flip can touch multiple sessions) before the + // fallible search-index work; the per-session scope keeps a reactive-tx + // flush compatible with the scoped sdk_messages write in the same batch. + for (const sid of changedSessions) this.notifySessionsChanged(sid); for (const messageId of messageIds) this.upsertMessageSearchRow(messageId); } @@ -1250,21 +1415,28 @@ export class SDKMessageRepository { } const message = JSON.parse(row.sdk_message) as { uuid?: string }; - const result = this.db - .prepare( - `DELETE FROM sdk_messages + const deleteStmt = this.db.prepare( + `DELETE FROM sdk_messages WHERE session_id = ? AND id = ? AND message_type = 'user' AND send_status IN ('deferred', 'enqueued')` - ) - .run(sessionId, messageId); + ); + // Wrap DELETE + counter recompute in one transaction (FTS cleanup below is + // best-effort, outside the tx) so an FTS throw can't leave the counter stale. + let deleted = false; + this.db.transaction(() => { + deleted = deleteStmt.run(sessionId, messageId).changes > 0; + if (deleted) this.recomputeVisibleMessageCount(sessionId); + })(); - if (result.changes === 0) { + if (!deleted) { return null; } this.deleteMessageSearchRow(row.id); + // No notifySessionsChanged(): the deleted row was deferred/enqueued + // (invisible), so the badge count is unchanged. return { dbId: row.id, uuid: message.uuid ?? '', @@ -1300,9 +1472,17 @@ export class SDKMessageRepository { .prepare(`SELECT id FROM sdk_messages WHERE session_id = ? AND timestamp > ?`) .all(sessionId, isoTimestamp) as Array<{ id: string }>; const stmt = this.db.prepare(`DELETE FROM sdk_messages WHERE session_id = ? AND timestamp > ?`); - const result = stmt.run(sessionId, isoTimestamp); + // Wrap DELETE + counter recompute in one transaction (FTS cleanup below is + // best-effort, outside the tx) so an FTS throw can't leave the counter stale. + let deleted = 0; + let badgeChanged = false; + this.db.transaction(() => { + deleted = stmt.run(sessionId, isoTimestamp).changes; + badgeChanged = this.recomputeVisibleMessageCount(sessionId); + })(); + if (badgeChanged) this.notifySessionsChanged(sessionId); for (const row of rows) this.deleteMessageSearchRow(row.id); - return result.changes; + return deleted; } /** @@ -1323,9 +1503,17 @@ export class SDKMessageRepository { const stmt = this.db.prepare( `DELETE FROM sdk_messages WHERE session_id = ? AND timestamp >= ?` ); - const result = stmt.run(sessionId, isoTimestamp); + // Wrap DELETE + counter recompute in one transaction (FTS cleanup below is + // best-effort, outside the tx) so an FTS throw can't leave the counter stale. + let deleted = 0; + let badgeChanged = false; + this.db.transaction(() => { + deleted = stmt.run(sessionId, isoTimestamp).changes; + badgeChanged = this.recomputeVisibleMessageCount(sessionId); + })(); + if (badgeChanged) this.notifySessionsChanged(sessionId); for (const row of rows) this.deleteMessageSearchRow(row.id); - return result.changes; + return deleted; } /** @@ -1575,6 +1763,15 @@ export class SDKMessageRepository { saveHyperNeoActionMessage(sessionId: string, message: HyperNeoActionMessage): string { const id = generateUUID(); const timestamp = new Date(message.timestamp).toISOString(); + const taskId = this.resolveTaskIdForSession(sessionId); + // Action rows are top-level, non-user, and use `message.action` as the + // subtype — visible unless that action happens to be a hidden subtype. + const countsTowardsBadge = isVisibleBadgeRow({ + parentToolUseId: null, + messageType: 'hyperneo_action', + messageSubtype: message.action, + sendStatus: null, + }); const values = [ id, @@ -1583,17 +1780,24 @@ export class SDKMessageRepository { message.action, JSON.stringify(message), timestamp, - this.resolveTaskIdForSession(sessionId), + taskId, ]; - this.db - .prepare( - `INSERT INTO sdk_messages ( - id, session_id, message_type, message_subtype, sdk_message, timestamp, task_id, - sdk_uuid, replacement_metadata_normalized - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 1)` - ) - .run(...values, message.uuid); + const insertStmt = this.db.prepare( + `INSERT INTO sdk_messages ( + id, session_id, message_type, message_subtype, sdk_message, timestamp, task_id, + sdk_uuid, replacement_metadata_normalized + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 1)` + ); + + // Wrap insert + counter bump in one transaction so a failure between them + // can't leave the counter under-counted — matches saveSDKMessage / + // saveUserMessage. upsertMessageSearchRow stays outside (FTS, best-effort). + this.db.transaction(() => { + insertStmt.run(...values, message.uuid); + if (countsTowardsBadge) this.bumpVisibleMessageCount(sessionId, 1); + })(); + if (countsTowardsBadge) this.notifySessionsChanged(sessionId); this.upsertMessageSearchRow(id); return id; } diff --git a/packages/daemon/src/storage/schema/index.ts b/packages/daemon/src/storage/schema/index.ts index 0182f57684..2188357108 100644 --- a/packages/daemon/src/storage/schema/index.ts +++ b/packages/daemon/src/storage/schema/index.ts @@ -163,7 +163,13 @@ export function createTables(db: BunDatabase): void { archived_at TEXT, parent_id TEXT, type TEXT DEFAULT 'worker' CHECK(type IN ('worker', 'room_chat', 'planner', 'coder', 'leader', 'general', 'lobby', 'spaces_global', 'space_task_agent', 'space_chat')), - session_context TEXT + session_context TEXT, + -- Maintained counter of visible top-level SDK messages (the badge + -- predicate: non-subagent, non-deferred user, non-hidden subtype). Read + -- directly by spaceSessions.bySpace instead of a correlated COUNT(*) + -- per session per poll. Maintained by SDKMessageRepository on every + -- sdk_messages mutation. See migration 177 for the backfill. + visible_message_count INTEGER NOT NULL DEFAULT 0 ) `); diff --git a/packages/daemon/src/storage/schema/migrations.ts b/packages/daemon/src/storage/schema/migrations.ts index 3e6525c363..56a589113a 100644 --- a/packages/daemon/src/storage/schema/migrations.ts +++ b/packages/daemon/src/storage/schema/migrations.ts @@ -23,6 +23,7 @@ import { resolveLegacyShape, type ArtifactShape, } from '@hyperneo/shared'; +import { HIDDEN_SYSTEM_SUBTYPES } from '@hyperneo/shared/sdk/type-guards'; import { createEvolutionTables } from './evolution'; import { createLongHorizonAgentTables } from './long-horizon-agents'; import { migrateLegacyLongHorizonAgentData } from '../../lib/space/agents/legacy-long-horizon-migration'; @@ -865,6 +866,14 @@ export function runMigrations(db: BunDatabase, createBackup: () => void): void { // backfill; see runMigration176. (Renumbered from 171/172/174/175 — dev // shipped those for other backfills + index drops.) run(migrationMarkerKey(176), () => runMigration176(db)); + + // Migration 177: Maintain sessions.visible_message_count so the + // space-sessions badge (spaceSessions.bySpace) reads a column instead of + // running a correlated COUNT(*) over sdk_messages for every session on every + // (150ms-debounced) re-evaluation. Adds the column and backfills it from the + // current visible totals; SDKMessageRepository maintains it thereafter. + // (Renumbered 169→170→171→175→176→177 as dev shipped intervening migrations.) + run(migrationMarkerKey(177), () => runMigration177(db)); } function migrationMarkerKey(version: number): string { @@ -11777,3 +11786,45 @@ export function runMigration176(db: BunDatabase): void { ); } } + +/** + * Migration 177: Add `sessions.visible_message_count` and backfill it. + * + * The space-sessions sidebar badge (spaceSessions.bySpace) used to compute a + * correlated COUNT(*) over sdk_messages for every session in a space on every + * (150ms-debounced) re-evaluation — ~92ms warm for dev-neokai, scaling with + * sessions × messages-per-session. This migration introduces a maintained + * counter on sessions that the query reads directly instead. + * + * The backfill predicate mirrors the former subquery exactly: top-level rows + * (parent_tool_use_id IS NULL), non-deferred user rows + * (send_status IN ('consumed', 'failed')), and non-hidden subtypes (the + * HIDDEN_SYSTEM_SUBTYPES set plus 'thinking_tokens'). SDKMessageRepository + * maintains the same predicate incrementally after this. + * + * Idempotent: re-running just recomputes the same totals. (Renumbered + * 169→170→171→175→176→177 as dev shipped intervening migrations.) + */ +export function runMigration177(db: BunDatabase): void { + if (!tableExists(db, 'sessions')) return; + if (!tableHasColumn(db, 'sessions', 'visible_message_count')) { + db.exec(`ALTER TABLE sessions ADD COLUMN visible_message_count INTEGER NOT NULL DEFAULT 0`); + } + if (!tableExists(db, 'sdk_messages')) return; + + const excludedSubtypes = [...HIDDEN_SYSTEM_SUBTYPES, 'thinking_tokens'] + .map((s) => `'${s.replace(/'/g, "''")}'`) + .join(', '); + + db.exec(` + UPDATE sessions + SET visible_message_count = COALESCE(( + SELECT COUNT(*) FROM sdk_messages sm + WHERE sm.session_id = sessions.id + AND sm.parent_tool_use_id IS NULL + AND (sm.message_type != 'user' + OR COALESCE(sm.send_status, 'consumed') IN ('consumed', 'failed')) + AND COALESCE(sm.message_subtype, '') NOT IN (${excludedSubtypes}) + ), 0) + `); +} diff --git a/packages/daemon/tests/unit/2-handlers/rpc-handlers/live-query-handlers.test.ts b/packages/daemon/tests/unit/2-handlers/rpc-handlers/live-query-handlers.test.ts index b8f5cc2368..21f3c2c42d 100644 --- a/packages/daemon/tests/unit/2-handlers/rpc-handlers/live-query-handlers.test.ts +++ b/packages/daemon/tests/unit/2-handlers/rpc-handlers/live-query-handlers.test.ts @@ -4607,7 +4607,9 @@ describe('NAMED_QUERY_REGISTRY', () => { test('row mapping carries processingState and messageCount like global sessions', () => { // The scope-filter cases above reuse a spaces-only DB; exercising the - // SELECT + mapRow needs the sessions + sdk_messages tables too. + // SELECT + mapRow needs the sessions table too. messageCount now comes + // from the maintained sessions.visible_message_count column, NOT a + // correlated COUNT(*) over sdk_messages. scopedDb.exec(` CREATE TABLE sessions ( id TEXT PRIMARY KEY, @@ -4615,46 +4617,19 @@ describe('NAMED_QUERY_REGISTRY', () => { status TEXT, processing_state TEXT, last_active_at TEXT, - type TEXT + type TEXT, + visible_message_count INTEGER NOT NULL DEFAULT 0 ) `); scopedDb.exec(` - CREATE TABLE sdk_messages ( - id TEXT, - session_id TEXT, - message_type TEXT, - message_subtype TEXT, - message_subtype_norm TEXT GENERATED ALWAYS AS (COALESCE(message_subtype, '')) VIRTUAL, - send_status TEXT, - parent_tool_use_id TEXT, - timestamp TEXT - ) - `); - scopedDb.exec(` - INSERT INTO sessions (id, title, status, processing_state, last_active_at, type) + INSERT INTO sessions (id, title, status, processing_state, last_active_at, type, visible_message_count) VALUES ('existing-1', 'My session', 'active', '{"status":"processing","phase":"thinking"}', - '2026-07-31 12:00:00', 'worker') + '2026-07-31 12:00:00', 'worker', 2) `); scopedDb.exec(` - INSERT INTO sessions (id, title, status, processing_state, last_active_at, type) - VALUES ('existing-2', 'Quiet session', 'active', NULL, '2026-07-31 12:00:00', 'worker') - `); - scopedDb.exec(` - INSERT INTO sdk_messages - (id, session_id, message_type, message_subtype, send_status, parent_tool_use_id, timestamp) - VALUES - -- visible top-level rows - ('m1', 'existing-1', 'assistant', NULL, NULL, NULL, '2026-07-31 12:00:01'), - ('m2', 'existing-1', 'user', NULL, 'consumed', NULL, '2026-07-31 12:00:02'), - -- hidden by send_status (deferred / enqueued) - ('m3', 'existing-1', 'user', NULL, 'deferred', NULL, '2026-07-31 12:00:03'), - ('m4', 'existing-1', 'user', NULL, 'enqueued', NULL, '2026-07-31 12:00:04'), - -- hidden by subtype (excluded from pagination) - ('m5', 'existing-1', 'system', 'session_state_changed', NULL, NULL, '2026-07-31 12:00:05'), - ('m6', 'existing-1', 'system', 'thinking_tokens', NULL, NULL, '2026-07-31 12:00:06'), - -- hidden because it's a subagent row (non-top-level) - ('m7', 'existing-1', 'assistant', NULL, NULL, 'toolu_1', '2026-07-31 12:00:07') + INSERT INTO sessions (id, title, status, processing_state, last_active_at, type, visible_message_count) + VALUES ('existing-2', 'Quiet session', 'active', NULL, '2026-07-31 12:00:00', 'worker', 0) `); const entry = NAMED_QUERY_REGISTRY.get('spaceSessions.bySpace')!; @@ -4663,9 +4638,7 @@ describe('NAMED_QUERY_REGISTRY', () => { const row = mapped.find((r) => r.id === 'existing-1')!; expect(row.processingState).toBe('{"status":"processing","phase":"thinking"}'); - // Only the two visible top-level rows (m1 assistant + m2 consumed user) - // count; deferred/enqueued, hidden subtypes, and subagent rows are all - // excluded by the transcript visibility predicate. + // The badge reads the maintained counter directly. expect(row.messageCount).toBe(2); // A sibling session with no messages + no processing state reports // 0 / undefined (not null), matching the global sessions shape. @@ -4673,6 +4646,60 @@ describe('NAMED_QUERY_REGISTRY', () => { expect(other.messageCount).toBe(0); expect(other.processingState).toBeUndefined(); }); + + test('messageCount is decoupled from sdk_messages — no per-session COUNT(*)', () => { + // The whole point of the maintained counter: the query must NOT count + // sdk_messages inline. Adding message rows cannot move the badge, which + // proves the correlated subquery is gone; only an explicit update to the + // maintained column moves it. (The visibility predicate that used to live + // here is now enforced incrementally by SDKMessageRepository and covered + // by its own tests.) + scopedDb.exec(` + CREATE TABLE sessions ( + id TEXT PRIMARY KEY, + title TEXT, + status TEXT, + processing_state TEXT, + last_active_at TEXT, + type TEXT, + visible_message_count INTEGER NOT NULL DEFAULT 0 + ) + `); + scopedDb.exec(` + CREATE TABLE sdk_messages ( + id TEXT, + session_id TEXT, + message_type TEXT, + message_subtype TEXT, + message_subtype_norm TEXT GENERATED ALWAYS AS (COALESCE(message_subtype, '')) VIRTUAL, + send_status TEXT, + parent_tool_use_id TEXT, + timestamp TEXT + ) + `); + scopedDb.exec(` + INSERT INTO sessions (id, title, status, processing_state, last_active_at, type, visible_message_count) + VALUES ('existing-1', 'My session', 'active', NULL, '2026-07-31 12:00:00', 'worker', 5) + `); + + const entry = NAMED_QUERY_REGISTRY.get('spaceSessions.bySpace')!; + const readCount = (): number => { + const rows = scopedDb.prepare(entry.sql).all(SPACE_ID) as Record[]; + const mapped = entry.mapRow ? rows.map(entry.mapRow) : rows; + return Number(mapped.find((r) => r.id === 'existing-1')!.messageCount); + }; + + expect(readCount()).toBe(5); + // Pile on visible-looking message rows — the badge must not budge. + scopedDb.exec(` + INSERT INTO sdk_messages (id, session_id, message_type, message_subtype, send_status, parent_tool_use_id, timestamp) + VALUES ('m1', 'existing-1', 'assistant', NULL, NULL, NULL, '2026-07-31 12:00:01') + `); + expect(readCount()).toBe(5); + // Only an explicit update to the maintained column moves it. + scopedDb.exec(`UPDATE sessions SET visible_message_count = 6 WHERE id = 'existing-1'`); + expect(readCount()).toBe(6); + }); }); }); }); diff --git a/packages/daemon/tests/unit/4-space-storage/storage/migrations/migration-177_test.ts b/packages/daemon/tests/unit/4-space-storage/storage/migrations/migration-177_test.ts new file mode 100644 index 0000000000..dd4550f732 --- /dev/null +++ b/packages/daemon/tests/unit/4-space-storage/storage/migrations/migration-177_test.ts @@ -0,0 +1,180 @@ +/** + * Migration 177 Tests — `sessions.visible_message_count` counter + backfill. + * + * The space-sessions badge (`spaceSessions.bySpace`) used to run a correlated + * COUNT(*) over `sdk_messages` per session. Migration 177 replaces that with a + * maintained column. These tests cover only the migration itself (column add + + * one-time backfill); the incremental maintenance is covered by the + * SDKMessageRepository suite. + * + * Covers: + * - Pre-M177 schema: the column is added and backfilled from the badge + * predicate (top-level rows, non-deferred user rows, non-hidden subtypes). + * - Idempotent re-run recomputes the same totals. + * - Fresh, fully-migrated DB carries the column from createTables. + * - Missing-table guards (empty DB; sessions without sdk_messages). + */ + +import { describe, test, expect, beforeEach, afterEach } from 'bun:test'; +import { mkdirSync, rmSync } from 'node:fs'; +import { join } from 'node:path'; +import { Database as BunDatabase } from 'bun:sqlite'; +import { createTables } from '../../../../../src/storage/schema'; +import { runMigration177, runMigrations } from '../../../../../src/storage/schema/migrations.ts'; + +function columnNames(db: BunDatabase, table: string): string[] { + const rows = db.prepare(`PRAGMA table_info('${table}')`).all() as Array<{ name: string }>; + return rows.map((r) => r.name); +} + +/** Minimal pre-M177 shape: sessions without visible_message_count + sdk_messages. */ +function seedPreM177Schema(db: BunDatabase): void { + db.exec(` + CREATE TABLE sessions ( + id TEXT PRIMARY KEY, + title TEXT NOT NULL + ); + CREATE TABLE sdk_messages ( + id TEXT PRIMARY KEY, + session_id TEXT NOT NULL, + message_type TEXT NOT NULL, + message_subtype TEXT, + send_status TEXT, + parent_tool_use_id TEXT, + timestamp TEXT NOT NULL + ); + `); +} + +function insertMessage( + db: BunDatabase, + id: string, + sessionId: string, + type: string, + opts: { subtype?: string | null; sendStatus?: string | null; parent?: string | null } = {} +): void { + db.prepare( + `INSERT INTO sdk_messages (id, session_id, message_type, message_subtype, send_status, parent_tool_use_id, timestamp) + VALUES (?, ?, ?, ?, ?, ?, ?)` + ).run( + id, + sessionId, + type, + opts.subtype ?? null, + opts.sendStatus ?? null, + opts.parent ?? null, + '2026-01-01T00:00:00Z' + ); +} + +function visibleCount(db: BunDatabase, sessionId: string): number { + return ( + db.prepare(`SELECT visible_message_count AS n FROM sessions WHERE id = ?`).get(sessionId) as { + n: number; + } + ).n; +} + +describe('Migration 177: sessions.visible_message_count counter + backfill', () => { + let testDir: string; + let db: BunDatabase; + + beforeEach(() => { + testDir = join( + process.cwd(), + 'tmp', + 'test-migration-177', + `test-${Date.now()}-${Math.random()}` + ); + mkdirSync(testDir, { recursive: true }); + db = new BunDatabase(join(testDir, 'test.db')); + db.exec('PRAGMA foreign_keys = ON'); + }); + + afterEach(() => { + try { + db.close(); + } catch { + // ignore + } + try { + rmSync(testDir, { recursive: true, force: true }); + } catch { + // ignore + } + }); + + describe('pre-M177 schema — add column + backfill', () => { + beforeEach(() => { + seedPreM177Schema(db); + db.prepare(`INSERT INTO sessions (id, title) VALUES (?, '')`).run('s1'); + db.prepare(`INSERT INTO sessions (id, title) VALUES (?, '')`).run('s2'); // no messages + // s1 visible rows + insertMessage(db, 'a1', 's1', 'assistant'); + insertMessage(db, 'a2', 's1', 'user', { sendStatus: 'consumed' }); + insertMessage(db, 'a3', 's1', 'user', { sendStatus: 'failed' }); + // s1 invisible rows + insertMessage(db, 'a4', 's1', 'user', { sendStatus: 'deferred' }); + insertMessage(db, 'a5', 's1', 'user', { sendStatus: 'enqueued' }); + insertMessage(db, 'a6', 's1', 'system', { subtype: 'session_state_changed' }); + insertMessage(db, 'a7', 's1', 'system', { subtype: 'thinking_tokens' }); + insertMessage(db, 'a8', 's1', 'assistant', { parent: 'toolu_1' }); // subagent + }); + + test('adds the NOT NULL DEFAULT 0 column', () => { + expect(columnNames(db, 'sessions')).not.toContain('visible_message_count'); + runMigration177(db); + expect(columnNames(db, 'sessions')).toContain('visible_message_count'); + }); + + test('backfills the badge predicate (3 visible for s1, 0 for s2)', () => { + runMigration177(db); + // assistant + consumed user + failed user; deferred/enqueued, hidden + // subtypes, and the subagent row are all excluded. + expect(visibleCount(db, 's1')).toBe(3); + expect(visibleCount(db, 's2')).toBe(0); + }); + + test('is idempotent — a second run recomputes the same totals', () => { + runMigration177(db); + const after1 = { s1: visibleCount(db, 's1'), s2: visibleCount(db, 's2') }; + expect(() => runMigration177(db)).not.toThrow(); + const after2 = { s1: visibleCount(db, 's1'), s2: visibleCount(db, 's2') }; + expect(after2).toEqual(after1); + }); + }); + + describe('fresh DB (all migrations applied)', () => { + beforeEach(() => { + // Real daemon init order: bootstrap tables, then run migrations. + createTables(db); + runMigrations(db, () => {}); + }); + + test('sessions carries visible_message_count from createTables', () => { + expect(columnNames(db, 'sessions')).toContain('visible_message_count'); + }); + + test('re-running migration 177 recomputes without error', () => { + db.prepare( + `INSERT INTO sessions (id, title, created_at, last_active_at, status, config, metadata) + VALUES ('s1', '', '2026-01-01', '2026-01-01', 'active', '{}', '{}')` + ).run(); + runMigration177(db); + expect(visibleCount(db, 's1')).toBe(0); + }); + }); + + describe('missing tables — no-op guards', () => { + test('runMigration177 on an empty DB does not throw', () => { + expect(() => runMigration177(db)).not.toThrow(); + }); + + test('runMigration177 skips backfill when sessions exists but sdk_messages does not', () => { + db.exec(`CREATE TABLE sessions (id TEXT PRIMARY KEY, title TEXT NOT NULL)`); + db.prepare(`INSERT INTO sessions (id, title) VALUES ('s1', '')`).run(); + expect(() => runMigration177(db)).not.toThrow(); + expect(columnNames(db, 'sessions')).toContain('visible_message_count'); + }); + }); +}); diff --git a/packages/daemon/tests/unit/4-space-storage/storage/recover-messages.test.ts b/packages/daemon/tests/unit/4-space-storage/storage/recover-messages.test.ts new file mode 100644 index 0000000000..3937558649 --- /dev/null +++ b/packages/daemon/tests/unit/4-space-storage/storage/recover-messages.test.ts @@ -0,0 +1,146 @@ +/** + * recover-messages script — regression test for the interrupted-rerun path. + * + * `touchedSessions` must include EVERY matched session on every run, not just + * sessions that inserted a new row this invocation. Otherwise a prior run that + * inserted rows but exited before the recompute step leaves the counter stale, + * and a rerun skips the session entirely (the existing-UUID `continue` fires + * before it's tracked). Since `spaceSessions.bySpace` reads only the counter, + * the badge stays stale for the recovered session. + * + * This test runs the script as a subprocess against a temp DB whose session + * already has its (visible) messages inserted with a stale counter of 0 — i.e. + * a rerun with no new inserts — and asserts the counter is repaired. + */ + +import { describe, test, expect, beforeEach, afterEach } from 'bun:test'; +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { Database } from '../../../../src/storage/sqlite-compat'; + +const SCRIPT = resolve( + join( + dirname(fileURLToPath(import.meta.url)), + '..', + '..', + '..', + '..', + 'scripts', + 'recover-messages.ts' + ) +); + +/** A visible recovered message (top-level, consumed user / assistant). */ +function recoveredMessage(uuid: string, type: string): string { + return JSON.stringify({ + type, + uuid, + session_id: 'sdk-S', + message: { role: type, content: [{ type: 'text', text: 'recovered' }] }, + }); +} + +describe('recover-messages script — interrupted-rerun recompute', () => { + let dir: string; + let dbPath: string; + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'recover-msg-')); + dbPath = join(dir, 'test.db'); + + const db = new Database(dbPath); + // sessions carries the full column set the script's `insertSession` + // statement prepares against (it prepares eagerly in Step 5, even with + // zero orphans), plus the maintained visible_message_count. sdk_messages + // holds the rows a prior (interrupted) run inserted. + db.exec(` + CREATE TABLE sessions ( + id TEXT PRIMARY KEY, + title TEXT NOT NULL, + workspace_path TEXT, + created_at TEXT NOT NULL, + last_active_at TEXT NOT NULL, + status TEXT NOT NULL, + config TEXT NOT NULL, + metadata TEXT NOT NULL, + is_worktree INTEGER DEFAULT 0, + worktree_path TEXT, + main_repo_path TEXT, + worktree_branch TEXT, + git_branch TEXT, + sdk_session_id TEXT, + available_commands TEXT, + processing_state TEXT, + archived_at TEXT, + visible_message_count INTEGER NOT NULL DEFAULT 0 + ); + CREATE TABLE sdk_messages ( + id TEXT PRIMARY KEY, + session_id TEXT NOT NULL, + message_type TEXT NOT NULL, + message_subtype TEXT, + sdk_message TEXT NOT NULL, + timestamp TEXT NOT NULL, + send_status TEXT, + parent_tool_use_id TEXT + ); + `); + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO sessions (id, title, created_at, last_active_at, status, config, metadata, sdk_session_id) + VALUES (?, '', ?, ?, 'active', '{}', '{}', ?)` + ).run('sess-S', now, now, 'sdk-S'); + // Prior-run bypass inserts — visible rows, but the counter was never + // maintained, so it stays at the default 0 (stale). + const insert = db.prepare( + `INSERT INTO sdk_messages (id, session_id, message_type, message_subtype, sdk_message, timestamp, send_status, parent_tool_use_id) + VALUES (?, ?, ?, NULL, ?, ?, 'consumed', NULL)` + ); + insert.run( + 'm1', + 'sess-S', + 'assistant', + recoveredMessage('m1', 'assistant'), + '2026-01-01T00:00:00Z' + ); + insert.run('m2', 'sess-S', 'user', recoveredMessage('m2', 'user'), '2026-01-01T00:00:01Z'); + expect( + ( + db + .prepare(`SELECT visible_message_count AS n FROM sessions WHERE id = ?`) + .get('sess-S') as { + n: number; + } + ).n + ).toBe(0); + db.close(); + }); + + afterEach(() => { + try { + rmSync(dir, { recursive: true, force: true }); + } catch { + // ignore + } + }); + + test('rerun recomputes a session a prior interrupted run left stale (no new inserts)', () => { + // All UUIDs already exist, so this run inserts nothing for sess-S — yet the + // counter must still be recomputed. Fails if touchedSessions only tracks + // current-run inserts. + execFileSync('bun', [SCRIPT, dbPath], { + stdio: ['ignore', 'ignore', 'pipe'], + }); + + const db = new Database(dbPath); + const row = db + .prepare(`SELECT visible_message_count AS n FROM sessions WHERE id = ?`) + .get('sess-S') as { n: number }; + db.close(); + // assistant + consumed user → 2 visible rows. + expect(row.n).toBe(2); + }, 30000); +}); diff --git a/packages/daemon/tests/unit/4-space-storage/storage/sdk-message-repository-live-query.test.ts b/packages/daemon/tests/unit/4-space-storage/storage/sdk-message-repository-live-query.test.ts new file mode 100644 index 0000000000..c5c4517e17 --- /dev/null +++ b/packages/daemon/tests/unit/4-space-storage/storage/sdk-message-repository-live-query.test.ts @@ -0,0 +1,155 @@ +/** + * SDKMessageRepository → LiveQueryEngine reactivity for spaceSessions.bySpace. + * + * Regression test for the visible_message_count counter. Now that the query + * reads a maintained sessions column instead of a correlated COUNT(*) over + * sdk_messages, its table-deps no longer include sdk_messages — so a message + * save must trigger re-evaluation via an explicit notifyChange('sessions') + * emitted by SDKMessageRepository. Without it the live badge would never + * refresh when messages arrive (the P1 from review round 2 on #2358). + * + * Design mirrors goal-repository-live-query.test.ts: wire the reactive layer, + * subscribe to the real spaceSessions.bySpace SQL, write through the repo, and + * await a microtask flush before asserting the LiveQuery delta. + */ + +import { describe, test, expect, beforeEach, afterEach } from 'bun:test'; +import { Database as BunDatabase } from '../../../../src/storage/sqlite-compat'; +import { createReactiveDatabase } from '../../../../src/storage/reactive-database'; +import { LiveQueryEngine } from '../../../../src/storage/live-query'; +import { SDKMessageRepository } from '../../../../src/storage/repositories/sdk-message-repository'; +import { NAMED_QUERY_REGISTRY } from '../../../../src/lib/rpc-handlers/live-query-handlers'; +import { createSpaceTables } from '../../helpers/space-test-db'; +import type { ReactiveDatabase } from '../../../../src/storage/reactive-database'; +import type { QueryDiff } from '../../../../src/storage/live-query'; +import type { SDKMessage } from '@hyperneo/shared/sdk'; + +interface SpaceSessionRow { + id: string; + messageCount: number; +} + +const SPACE_ID = 'space-reactivity'; +const SESSION_ID = 'sess-reactivity'; + +function createAssistantMessage(content: string): SDKMessage { + return { + type: 'assistant', + message: { role: 'assistant', content: [{ type: 'text', text: content }] }, + } as SDKMessage; +} + +describe('SDKMessageRepository → LiveQueryEngine reactivity (spaceSessions.bySpace)', () => { + let bunDb: BunDatabase; + let reactiveDb: ReactiveDatabase; + let engine: LiveQueryEngine; + let repo: SDKMessageRepository; + let sql: string; + + beforeEach(() => { + bunDb = new BunDatabase(':memory:'); + createSpaceTables(bunDb); + reactiveDb = createReactiveDatabase({ getDatabase: () => bunDb } as never); + engine = new LiveQueryEngine(bunDb, reactiveDb); + repo = new SDKMessageRepository(bunDb, reactiveDb); + + sql = NAMED_QUERY_REGISTRY.get('spaceSessions.bySpace')!.sql; + + const now = Date.now(); + const iso = new Date(now).toISOString(); + bunDb + .prepare( + `INSERT INTO spaces (id, slug, workspace_path, name, session_ids, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?)` + ) + .run( + SPACE_ID, + 'reactivity', + '/ws/reactivity', + 'Reactivity', + JSON.stringify([SESSION_ID]), + now, + now + ); + bunDb + .prepare( + `INSERT INTO sessions (id, title, created_at, last_active_at, status, config, metadata) + VALUES (?, '', ?, ?, 'active', '{}', '{}')` + ) + .run(SESSION_ID, iso, iso); + }); + + afterEach(() => { + engine.dispose(); + bunDb.close(); + }); + + test('a visible SDK message save re-evaluates the badge with the new count', async () => { + const diffs: QueryDiff[] = []; + engine.subscribe(sql, [SPACE_ID], (diff) => diffs.push(diff)); + + // Snapshot: the one session is present with count 0. + expect(diffs).toHaveLength(1); + expect(diffs[0].type).toBe('snapshot'); + expect(diffs[0].rows?.[0]?.messageCount).toBe(0); + + repo.saveSDKMessage(SESSION_ID, createAssistantMessage('hello')); + + // The engine re-evaluates on the reactive 'sessions' change in a microtask. + await Promise.resolve(); + await Promise.resolve(); + + // A delta fired and the session's count moved 0 → 1. The row already existed + // in the snapshot, so the change surfaces in `updated` (keyed by session id). + expect(diffs).toHaveLength(2); + expect(diffs[1].type).toBe('delta'); + expect(diffs[1].updated?.[0]?.id).toBe(SESSION_ID); + expect(diffs[1].updated?.[0]?.messageCount).toBe(1); + }); + + test('an invisible (subagent) save does not re-evaluate the badge', async () => { + const diffs: QueryDiff[] = []; + engine.subscribe(sql, [SPACE_ID], (diff) => diffs.push(diff)); + expect(diffs).toHaveLength(1); // snapshot only + + // Subagent row (parent_tool_use_id set) is invisible → counter unchanged → + // no notifyChange('sessions') → no re-evaluation. + repo.saveSDKMessage(SESSION_ID, { + type: 'assistant', + parent_tool_use_id: 'toolu_1', + message: { role: 'assistant', content: [{ type: 'text', text: 'sub' }] }, + } as SDKMessage); + + await Promise.resolve(); + await Promise.resolve(); + + expect(diffs).toHaveLength(1); + }); + + test('a send_status flip into visibility re-evaluates the badge', async () => { + const diffs: QueryDiff[] = []; + engine.subscribe(sql, [SPACE_ID], (diff) => diffs.push(diff)); + expect(diffs[0].rows?.[0]?.messageCount).toBe(0); + + // Save a deferred user message (invisible) → no re-eval. + const id = repo.saveUserMessage( + SESSION_ID, + { + type: 'user', + message: { role: 'user', content: [{ type: 'text', text: 'queued' }] }, + } as SDKMessage, + 'deferred' + ); + await Promise.resolve(); + await Promise.resolve(); + expect(diffs).toHaveLength(1); + + // Flip to consumed → becomes visible → counter +1 → re-eval. + repo.updateMessageStatus([id], 'consumed'); + await Promise.resolve(); + await Promise.resolve(); + + expect(diffs).toHaveLength(2); + expect(diffs[1].updated?.[0]?.messageCount).toBe(1); + }); +}); diff --git a/packages/daemon/tests/unit/4-space-storage/storage/sdk-message-repository.test.ts b/packages/daemon/tests/unit/4-space-storage/storage/sdk-message-repository.test.ts index 07f764ce89..9541e2ea5a 100644 --- a/packages/daemon/tests/unit/4-space-storage/storage/sdk-message-repository.test.ts +++ b/packages/daemon/tests/unit/4-space-storage/storage/sdk-message-repository.test.ts @@ -11,6 +11,7 @@ import { type SendStatus, } from '../../../../src/storage/repositories/sdk-message-repository'; import type { SDKMessage } from '@hyperneo/shared/sdk'; +import type { HyperNeoActionMessage } from '@hyperneo/shared'; describe('SDKMessageRepository', () => { let db: Database; @@ -1263,6 +1264,260 @@ describe('SDKMessageRepository', () => { }); }); + describe('visible_message_count maintenance', () => { + // These tests use a sessions table that carries `visible_message_count`, + // unlike the suite-wide setup (which has no sessions table at all and so + // exercises the no-op guard). + let badgeDb: Database; + let badgeRepo: SDKMessageRepository; + const SID = 'sess-badge'; + + function createSession(id: string): void { + badgeDb + .prepare( + `INSERT INTO sessions (id, title, created_at, last_active_at, status, config, metadata) + VALUES (?, '', '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 'active', '{}', '{}')` + ) + .run(id); + } + + function badgeCount(sessionId: string = SID): number { + const row = badgeDb + .prepare(`SELECT visible_message_count AS n FROM sessions WHERE id = ?`) + .get(sessionId) as { n: number }; + return row.n; + } + + /** Same predicate the former spaceSessions.bySpace subquery used — drift check. */ + function freshBadgeCount(sessionId: string = SID): number { + const excluded = [ + 'session_state_changed', + 'commands_changed', + 'task_started', + 'task_progress', + 'task_updated', + 'mirror_error', + 'elicitation_complete', + 'thinking_tokens', + ] + .map((s) => `'${s}'`) + .join(','); + const row = badgeDb + .prepare( + `SELECT COUNT(*) AS n FROM sdk_messages + WHERE session_id = ? + AND parent_tool_use_id IS NULL + AND (message_type != 'user' + OR COALESCE(send_status, 'consumed') IN ('consumed', 'failed')) + AND COALESCE(message_subtype, '') NOT IN (${excluded})` + ) + .get(sessionId) as { n: number }; + return row.n; + } + + function createActionMessage(uuid: string = crypto.randomUUID()): HyperNeoActionMessage { + return { + type: 'hyperneo_action', + uuid, + session_id: SID, + action: 'sdk_resume_choice', + resolved: false, + timestamp: Date.now(), + }; + } + + beforeEach(() => { + badgeDb = new Database(':memory:'); + badgeDb.exec(` + CREATE TABLE sessions ( + id TEXT PRIMARY KEY, + title TEXT NOT NULL, + created_at TEXT NOT NULL, + last_active_at TEXT NOT NULL, + status TEXT NOT NULL, + config TEXT NOT NULL, + metadata TEXT NOT NULL, + session_context TEXT, + type TEXT DEFAULT 'worker', + visible_message_count INTEGER NOT NULL DEFAULT 0 + ); + CREATE TABLE sdk_messages ( + id TEXT PRIMARY KEY, + session_id TEXT NOT NULL, + message_type TEXT NOT NULL, + message_subtype TEXT, + sdk_message TEXT NOT NULL, + timestamp TEXT NOT NULL, + send_status TEXT, + origin TEXT DEFAULT NULL CHECK(origin IS NULL OR origin IN ('human', 'system')), + is_renderable INTEGER NOT NULL DEFAULT 1, + is_terminal INTEGER NOT NULL DEFAULT 0, + parent_tool_use_id TEXT, + task_id TEXT, + sdk_uuid TEXT, + replacement_metadata_normalized INTEGER NOT NULL DEFAULT 0 + ); + CREATE TABLE sdk_message_replacements ( + source_message_id TEXT NOT NULL, + session_id TEXT NOT NULL, + task_id TEXT, + target_uuid TEXT NOT NULL, + kind TEXT NOT NULL CHECK(kind IN ('superseded', 'retracted')), + PRIMARY KEY (source_message_id, target_uuid, kind), + FOREIGN KEY (source_message_id) REFERENCES sdk_messages(id) ON DELETE CASCADE + ); + CREATE INDEX idx_sdk_messages_session ON sdk_messages(session_id); + `); + badgeRepo = new SDKMessageRepository(badgeDb as any); + createSession(SID); + }); + + afterEach(() => { + badgeDb.close(); + }); + + it('increments on a visible top-level SDK message', () => { + badgeRepo.saveSDKMessage(SID, createAssistantMessage('hello')); + expect(badgeCount()).toBe(1); + expect(badgeCount()).toBe(freshBadgeCount()); + }); + + it('does not increment for subagent (parent_tool_use_id) rows', () => { + badgeRepo.saveSDKMessage(SID, createAssistantMessage('top', 'toolu_1')); + badgeRepo.saveSDKMessage(SID, createSubagentMessage('sub', 'toolu_1')); + expect(badgeCount()).toBe(1); + expect(badgeCount()).toBe(freshBadgeCount()); + }); + + it('does not increment for hidden system subtypes or thinking_tokens', () => { + badgeRepo.saveSDKMessage(SID, { + type: 'system', + subtype: 'session_state_changed', + uuid: 'u1', + } as unknown as SDKMessage); + badgeRepo.saveSDKMessage(SID, { + type: 'system', + subtype: 'thinking_tokens', + uuid: 'u2', + } as unknown as SDKMessage); + badgeRepo.saveSDKMessage(SID, createAssistantMessage('visible')); + expect(badgeCount()).toBe(1); + expect(badgeCount()).toBe(freshBadgeCount()); + }); + + it('counts consumed/failed user messages but not deferred/enqueued', () => { + badgeRepo.saveUserMessage(SID, createUserMessage('sent'), 'consumed'); + badgeRepo.saveUserMessage(SID, createUserMessage('failed'), 'failed'); + badgeRepo.saveUserMessage(SID, createUserMessage('deferred'), 'deferred'); + badgeRepo.saveUserMessage(SID, createUserMessage('enqueued'), 'enqueued'); + expect(badgeCount()).toBe(2); + expect(badgeCount()).toBe(freshBadgeCount()); + }); + + it('recounts when send_status flips into or out of visibility', () => { + const id = badgeRepo.saveUserMessage(SID, createUserMessage('queued'), 'deferred'); + expect(badgeCount()).toBe(0); + badgeRepo.updateMessageStatus([id], 'consumed'); + expect(badgeCount()).toBe(1); + // Back to a non-visible status removes it again. + badgeRepo.updateMessageStatus([id], 'enqueued'); + expect(badgeCount()).toBe(0); + expect(badgeCount()).toBe(freshBadgeCount()); + }); + + it('increments for hyperneo_action messages', () => { + badgeRepo.saveHyperNeoActionMessage(SID, createActionMessage()); + expect(badgeCount()).toBe(1); + expect(badgeCount()).toBe(freshBadgeCount()); + }); + + it('recomputes after rewind deletes messages', () => { + badgeRepo.saveSDKMessage(SID, createAssistantMessage('a')); + badgeRepo.saveSDKMessage(SID, createAssistantMessage('b')); + badgeRepo.saveSDKMessage(SID, createAssistantMessage('c')); + expect(badgeCount()).toBe(3); + const earliest = badgeDb + .prepare(`SELECT MIN(timestamp) AS t FROM sdk_messages WHERE session_id = ?`) + .get(SID) as { t: string }; + badgeRepo.deleteMessagesAtAndAfter(SID, Date.parse(earliest.t)); + expect(badgeCount()).toBe(0); + expect(badgeCount()).toBe(freshBadgeCount()); + }); + + it('recomputeVisibleMessageCount repairs drift after a bypass insert', () => { + // Simulates scripts/recover-messages.ts: a raw INSERT into sdk_messages + // that skips the maintained counter, then a repair via the shared public + // recompute entry point (so the script reuses the predicate, not a copy). + const insertRaw = badgeDb.prepare( + `INSERT INTO sdk_messages + (id, session_id, message_type, message_subtype, sdk_message, timestamp, send_status, parent_tool_use_id) + VALUES (?, ?, ?, ?, ?, ?, 'consumed', NULL)` + ); + insertRaw.run('raw-1', SID, 'assistant', null, '{}', '2026-01-01T00:00:00Z'); + insertRaw.run('raw-2', SID, 'user', null, '{}', '2026-01-01T00:00:01Z'); + // The bypass insert left the counter stale at 0. + expect(badgeCount()).toBe(0); + // Returns true because the value actually changed; counter now matches. + expect(badgeRepo.recomputeVisibleMessageCount(SID)).toBe(true); + expect(badgeCount()).toBe(2); + expect(badgeCount()).toBe(freshBadgeCount()); + // A second recompute is a no-op (returns false, value unchanged). + expect(badgeRepo.recomputeVisibleMessageCount(SID)).toBe(false); + }); + + it('stays consistent with a fresh COUNT(*) across a mixed sequence', () => { + // Anti-drift: a representative mix of inserts, a status flip, a hidden + // subtype, and a partial rewind — the maintained counter must equal a + // freshly computed badge count throughout. + badgeRepo.saveSDKMessage(SID, createAssistantMessage('a')); + badgeRepo.saveSDKMessage(SID, createSubagentMessage('sub', 'tu1')); + const pending = badgeRepo.saveUserMessage(SID, createUserMessage('p'), 'enqueued'); + badgeRepo.saveUserMessage(SID, createUserMessage('q'), 'consumed'); + badgeRepo.updateMessageStatus([pending], 'consumed'); + badgeRepo.saveSDKMessage(SID, { + type: 'system', + subtype: 'task_progress', + uuid: 'hidden', + } as unknown as SDKMessage); + expect(badgeCount()).toBe(freshBadgeCount()); + // Rewind from the 3rd-oldest row onward. + const mid = badgeDb + .prepare( + `SELECT timestamp FROM sdk_messages WHERE session_id = ? ORDER BY timestamp ASC LIMIT 1 OFFSET 2` + ) + .get(SID) as { timestamp: string } | undefined; + if (mid) badgeRepo.deleteMessagesAtAndAfter(SID, Date.parse(mid.timestamp)); + expect(badgeCount()).toBe(freshBadgeCount()); + }); + + it('is a no-op without a sessions table (schema subset)', () => { + const subsetDb = new Database(':memory:'); + subsetDb.exec(` + CREATE TABLE sdk_messages ( + id TEXT PRIMARY KEY, session_id TEXT NOT NULL, message_type TEXT NOT NULL, + message_subtype TEXT, sdk_message TEXT NOT NULL, timestamp TEXT NOT NULL, + send_status TEXT, origin TEXT, is_renderable INTEGER NOT NULL DEFAULT 1, + is_terminal INTEGER NOT NULL DEFAULT 0, parent_tool_use_id TEXT, task_id TEXT, + sdk_uuid TEXT, replacement_metadata_normalized INTEGER NOT NULL DEFAULT 0 + ); + CREATE TABLE sdk_message_replacements ( + source_message_id TEXT NOT NULL, session_id TEXT NOT NULL, task_id TEXT, + target_uuid TEXT NOT NULL, kind TEXT NOT NULL CHECK(kind IN ('superseded','retracted')), + PRIMARY KEY (source_message_id, target_uuid, kind) + ); + `); + const subsetRepo = new SDKMessageRepository(subsetDb as never); + // Must not throw and must not try to UPDATE a non-existent sessions table. + expect(() => + subsetRepo.saveSDKMessage('no-sessions', createAssistantMessage('x')) + ).not.toThrow(); + expect(() => + subsetRepo.saveUserMessage('no-sessions', createUserMessage('y'), 'consumed') + ).not.toThrow(); + subsetDb.close(); + }); + }); + describe('saveUserMessage', () => { it('should save user message with consumed status by default', () => { const message = createUserMessage('Test message'); diff --git a/packages/daemon/tests/unit/helpers/space-test-db.ts b/packages/daemon/tests/unit/helpers/space-test-db.ts index 4988d38397..b4521c330c 100644 --- a/packages/daemon/tests/unit/helpers/space-test-db.ts +++ b/packages/daemon/tests/unit/helpers/space-test-db.ts @@ -508,7 +508,8 @@ export function createSpaceTables(db: BunDatabase): void { archived_at TEXT, parent_id TEXT, type TEXT DEFAULT 'worker' CHECK(type IN ('worker', 'room_chat', 'planner', 'coder', 'leader', 'general', 'lobby', 'spaces_global', 'space_task_agent', 'space_chat')), - session_context TEXT + session_context TEXT, + visible_message_count INTEGER NOT NULL DEFAULT 0 ) `); db.exec(`CREATE INDEX IF NOT EXISTS idx_sessions_space_agent_provenance