diff --git a/dependencies/aimee-repositories.lock.json b/dependencies/aimee-repositories.lock.json index afca55501..845e13aef 100644 --- a/dependencies/aimee-repositories.lock.json +++ b/dependencies/aimee-repositories.lock.json @@ -378,7 +378,7 @@ "placements": [ "kb" ], - "source_sha256": "66d64ce9e3efb425d4b0bf7aaa6481f74d6d9735fed7f89b915b28b0f22551d2", + "source_sha256": "8f3a706ef4e2aeef3250b7cc33c0c2bbd3f29250ed24bdaeff548ae3f2cf8878", "runtime": "go", "principal_class": 1, "principal_ref": 22, diff --git a/src/cli_v1_routes_b.c b/src/cli_v1_routes_b.c index 27ccf84e1..92cba18de 100644 --- a/src/cli_v1_routes_b.c +++ b/src/cli_v1_routes_b.c @@ -1611,6 +1611,19 @@ void print_server_health(cJSON *resp) cJSON_ArrayForEach(b, blockers) if (cJSON_IsString(b)) printf(" BLOCKED: %s\n", b->valuestring); } + /* Advisory findings. They do not move the verdict, which is exactly why they + * need printing: a kb that is genuinely "ok" can still be accumulating work + * nothing will process, and a status line that only ever renders blockers + * reports that as a clean bill of health. Measured: a typed-fact backlog of + * 4 jobs, unclaimable for 11.5 hours, with `aimee status` showing "aimee-kb: + * ok" and nothing else. */ + cJSON *kbwarn = cJSON_GetObjectItemCaseSensitive(kb, "warnings"); + if (cJSON_IsArray(kbwarn) && cJSON_GetArraySize(kbwarn) > 0) + { + cJSON *w; + cJSON_ArrayForEach(w, kbwarn) if (cJSON_IsString(w)) + printf(" note: %s\n", w->valuestring); + } /* An open transport breaker refuses every call locally, so the kb can be * "ok" here while nothing works. Print it before the detail lines: this is * the line that explains an index that answers "unavailable" on a server diff --git a/src/db2/kb_payload.c b/src/db2/kb_payload.c index 15cf8f9e3..837b9cd2f 100644 --- a/src/db2/kb_payload.c +++ b/src/db2/kb_payload.c @@ -1131,6 +1131,27 @@ int db2_kb_pdf_search_chunks(const char *project, const char *query, int max, return n; } +int db2_kb_async_count_kind_pending(const char *kind) +{ + if (!kind || !*kind) + return -1; + void *conn = db2_conn(); + if (!conn) + return -1; + char err[KBP_ERRBUF] = ""; + aimee_pg_stmt_t *st = aimee_pg_prepare( + conn, "SELECT COUNT(*) FROM kb_async_jobs WHERE kind = ?1 AND status = 'pending'", err, + sizeof(err)); + if (!st) + return -1; + aimee_pg_bind_text(st, "?1", kind); + int n = -1; + if (aimee_pg_step(st, err, sizeof(err)) == AIMEE_PG_ROW) + n = aimee_pg_column_int(st, 0); + aimee_pg_finalize(st); + return n; +} + int db2_kb_async_count_kind(const char *kind) { if (!kind || !*kind) diff --git a/src/db2/kb_payload.h b/src/db2/kb_payload.h index d7cace71c..f1bb02071 100644 --- a/src/db2/kb_payload.h +++ b/src/db2/kb_payload.h @@ -240,6 +240,12 @@ extern "C" * helper. Returns the count (>=0) or -1 on error. */ int db2_kb_async_count_kind(const char *kind); + /* Count only the PENDING rows for a kind. Separate from the total above because a + * backlog is the interesting quantity: total conflates work still waiting with work + * long finished, and cannot distinguish a queue that is draining from one nothing + * will ever claim. Returns the count (>=0) or -1 on error. */ + int db2_kb_async_count_kind_pending(const char *kind); + /* structured-pdf Phase B: a recognised table cell. Stored ONLY in kb_table_cells (never * in the shared typed_facts table — see schema.sql). */ typedef struct diff --git a/src/kb/kb_service_kb.c b/src/kb/kb_service_kb.c index ee544e36d..6fa4858e0 100644 --- a/src/kb/kb_service_kb.c +++ b/src/kb/kb_service_kb.c @@ -5,6 +5,7 @@ #include "aimee.h" #include "cJSON.h" #include "config.h" +#include "config_database.h" /* config_synth_chat_endpoint_current */ #include "kb_curator_queue.h" #include "kb_curator_provider.h" #include "json_fluent.h" @@ -489,6 +490,44 @@ static cJSON *kb_service_health_object(void) snprintf(msg, sizeof(msg), "curator: %d job(s) failing", n); cJSON_AddItemToArray(warnings, cJSON_CreateString(msg)); } + /* TWO GATES THAT DO NOT AGREE. + * + * memory.store enqueues a "memory_facts" job whenever typed_facts_enabled is on, + * which is the DEFAULT. The only consumer of those jobs is kb_memory_facts_drain, + * which runs on the curator LLM lane -- and that lane deliberately does not start + * without a synthesis endpoint ("NO SYNTHESIS PROVIDER => NO LLM LANE"). Both + * decisions are individually right. Together they mean an install with no synth + * provider, which is a supported configuration, enqueues one row per stored + * memory that nothing will ever claim. + * + * Measured on the e2e VM: 4 memory_facts jobs pending for 11.5 hours with + * attempts=0, while /v1/health reported status ok and an empty warnings array, + * and `aimee kb status` printed "4 pending" with no hint that pending here means + * forever. It grows without bound for the life of the install. + * + * NOT a blocker. Running without a synthesis provider is explicitly supported, + * and memory store and search work perfectly; only typed-fact enrichment is + * deferred. Degrading the verdict for a supported configuration is exactly the + * dilution this file's status derivation exists to avoid. It is a warning, and + * it says the backlog is not lost -- configuring a provider drains it. */ + if (config_typed_facts_enabled()) + { + int facts_pending = db2_kb_async_count_kind_pending("memory_facts"); + char synth_endpoint[512]; + if (facts_pending > 0 && + !config_synth_chat_endpoint_current(synth_endpoint, sizeof(synth_endpoint))) + { + char msg[320]; + snprintf(msg, sizeof(msg), + "typed-fact extraction: %d job(s) queued with nothing to drain them — no " + "synthesis endpoint is configured, so the curator LLM lane is not running. " + "Memories are stored and searchable but contribute no typed facts. The " + "backlog is not lost: configuring a synthesis provider drains it.", + facts_pending); + cJSON_AddItemToArray(warnings, cJSON_CreateString(msg)); + } + } + if (freshness_days > 30) cJSON_AddItemToArray(warnings, cJSON_CreateString("KB not ingested in over 30 days")); else if (freshness_days > 7) diff --git a/src/server/server_api_status.c b/src/server/server_api_status.c index c1f2114a6..ce4673662 100644 --- a/src/server/server_api_status.c +++ b/src/server/server_api_status.c @@ -95,6 +95,31 @@ static int api_bearer_extra_count(void); * storage tier's vocabulary — build-integrity greps the client binary for exactly * that and fails the build. The client reports whether the kb's store and vector * index are healthy without knowing what implements them. */ +/* Split a newline-joined kb field back into a JSON array under `key`, omitting the + * key entirely when there is nothing to say. Extracted when warnings joined + * blockers: two copies of the same pointer arithmetic was how the warnings array + * came to be dropped in the first place. */ +static void kb_health_add_lines(cJSON *kbo, const char *key, const char *joined) +{ + if (!kbo || !key || !joined || !joined[0]) + return; + cJSON *arr = cJSON_AddArrayToObject(kbo, key); + if (!arr) + return; + for (const char *p = joined; p && *p;) + { + const char *nl = strchr(p, '\n'); + size_t len = nl ? (size_t)(nl - p) : strlen(p); + char line[384]; + if (len >= sizeof(line)) + len = sizeof(line) - 1; + memcpy(line, p, len); + line[len] = '\0'; + cJSON_AddItemToArray(arr, cJSON_CreateString(line)); + p = nl ? nl + 1 : NULL; + } +} + void server_health_add_kb(cJSON *resp) { if (!resp) @@ -150,23 +175,17 @@ void server_health_add_kb(cJSON *resp) return; /* Pass the reasons through verbatim. The kb composed them next to the evidence * and named the remedy; re-deriving them here from the booleans below would be - * a second place for the verdict to drift out of step with the facts. */ - if (kb.blockers[0]) - { - cJSON *arr = cJSON_AddArrayToObject(kbo, "blockers"); - for (const char *p = kb.blockers; p && *p;) - { - const char *nl = strchr(p, '\n'); - size_t len = nl ? (size_t)(nl - p) : strlen(p); - char line[320]; - if (len >= sizeof(line)) - len = sizeof(line) - 1; - memcpy(line, p, len); - line[len] = '\0'; - cJSON_AddItemToArray(arr, cJSON_CreateString(line)); - p = nl ? nl + 1 : NULL; - } - } + * a second place for the verdict to drift out of step with the facts. + * + * WARNINGS TRAVEL TOO, not just blockers. The kb has always assembled a + * warnings array and this block dropped it on the floor, so findings that do + * not move the verdict reached no operator at all: a typed-fact backlog that + * nothing will ever drain sat in /v1/health for hours while `aimee status` + * showed a clean kb and `aimee kb status` printed a bare "4 pending". Publishing + * a finding into a field no surface renders is the same defect as not computing + * it -- the evidence exists and the summary does not carry it. */ + kb_health_add_lines(kbo, "blockers", kb.blockers); + kb_health_add_lines(kbo, "warnings", kb.warnings); cJSON_AddBoolToObject(kbo, "store_ok", kb.db2_ok ? 1 : 0); cJSON_AddBoolToObject(kbo, "vectors_ok", kb.pgvec_ok ? 1 : 0); cJSON_AddBoolToObject(kbo, "embed_configured", kb.embed_ok ? 1 : 0); diff --git a/src/tests/test_curator_queue.c b/src/tests/test_curator_queue.c index 09e77f9ec..dfe40c24d 100644 --- a/src/tests/test_curator_queue.c +++ b/src/tests/test_curator_queue.c @@ -127,6 +127,43 @@ static void test_polymorphic_async_subject(sqlite3 *db) printf(" PASS: polymorphic async subject accepts a memory id without a document row\n"); } +/* A backlog nothing will drain has to be countable, and countable SEPARATELY from + * work already finished. + * + * memory.store enqueues a memory_facts job whenever typed_facts is on (the + * default); the only consumer runs on the curator LLM lane, which does not start + * without a synthesis endpoint. On an install with no synth provider -- a + * supported configuration -- that queue grows by one row per stored memory and is + * never claimed. Observed on the e2e VM: 4 pending for 11.5 hours, attempts=0, + * while health reported ok with an empty warnings array. + * + * The health surface reports it, and to do that it needs PENDING, not total: + * total cannot tell a queue that is draining from one that never will. */ +static void test_pending_count_excludes_finished(sqlite3 *db) +{ + seed(db, "DELETE FROM kb_async_jobs WHERE kind='memory_facts'"); + assert(db2_kb_async_count_kind_pending("memory_facts") == 0); + + assert(db2_kb_async_enqueue("memory_facts", 881001, "memory") == 0); + assert(db2_kb_async_enqueue("memory_facts", 881002, "memory") == 0); + assert(db2_kb_async_count_kind_pending("memory_facts") == 2); + assert(db2_kb_async_count_kind("memory_facts") == 2); + + /* One finishes. The backlog is 1, even though two rows exist -- reporting 2 + * here would keep warning about work that already completed. */ + seed(db, "UPDATE kb_async_jobs SET status='done' WHERE kind='memory_facts'" + " AND document_id=881001"); + assert(db2_kb_async_count_kind_pending("memory_facts") == 1); + assert(db2_kb_async_count_kind("memory_facts") == 2); + + /* Other kinds are not counted into this backlog: kb_async_jobs is shared. */ + assert(db2_kb_async_enqueue("extract_doc", 881003, "p") == 0); + assert(db2_kb_async_count_kind_pending("memory_facts") == 1); + + seed(db, "DELETE FROM kb_async_jobs WHERE document_id IN (881001,881002,881003)"); + printf(" PASS: pending count is the backlog, not the row total\n"); +} + static const char *job_status(sqlite3 *db, int64_t id) { static char buf[64]; @@ -359,6 +396,7 @@ int main(void) " VALUES ('p','/p','2026-01-01 00:00:00')"); test_polymorphic_async_subject(db); + test_pending_count_excludes_finished(db); /* Contract: every non-pdf doc gets one extract_doc job; PDFs are excluded; * re-running enqueues nothing. Guard for "docs present but zero jobs". */ diff --git a/src/tests/test_server_dispatch.c b/src/tests/test_server_dispatch.c index 3c25bcd55..3dc66639d 100644 --- a/src/tests/test_server_dispatch.c +++ b/src/tests/test_server_dispatch.c @@ -2282,6 +2282,29 @@ static void test_health_kb_verdict_states(void) assert(strcmp(cJSON_GetArrayItem(blockers, 1)->valuestring, "vector table missing") == 0); cJSON_Delete(resp); + /* Warnings travel even though they do not move the verdict. This block used to + * drop the kb's warnings array entirely, so an advisory finding reached no + * operator: a typed-fact backlog nothing could drain sat unreported for hours + * behind a status of "ok". Publishing a finding into a field no surface renders + * is the same defect as never computing it. */ + kb_health_stub_reset(); + g_kb_health_rc = 0; + g_kb_health.process_ok = 1; + snprintf(g_kb_health.status, sizeof(g_kb_health.status), "ok"); + snprintf(g_kb_health.warnings, sizeof(g_kb_health.warnings), + "typed-fact extraction: 4 job(s) queued with nothing to drain them\nKB not ingested " + "in over 7 days"); + resp = cJSON_CreateObject(); + server_health_add_kb(resp); + assert(strcmp(kb_status_of(resp), "ok") == 0); /* advisory: verdict unchanged */ + kb = cJSON_GetObjectItemCaseSensitive(resp, "kb"); + cJSON *warns = cJSON_GetObjectItemCaseSensitive(kb, "warnings"); + assert(cJSON_IsArray(warns) && cJSON_GetArraySize(warns) == 2); + assert(strstr(cJSON_GetArrayItem(warns, 0)->valuestring, "typed-fact extraction") != NULL); + /* No blockers key at all when there are none, rather than an empty array. */ + assert(cJSON_GetObjectItemCaseSensitive(kb, "blockers") == NULL); + cJSON_Delete(resp); + /* 4. An open transport breaker refuses every call locally, so a kb that * considers itself perfectly healthy still cannot be queried. The breaker is * part of the verdict rather than a flag beside it. */