From 10eee4712354562cba6691c0c5b3c9ed640b2e16 Mon Sep 17 00:00:00 2001 From: JBailes Date: Tue, 4 Aug 2026 09:57:52 +0000 Subject: [PATCH 1/3] fix: report the typed-fact backlog nothing will drain Two gates that do not agree. memory.store enqueues a "memory_facts" job whenever typed_facts_enabled is on, which is the DEFAULT. The only consumer of those jobs is kb_memory_facts_drain, which runs on the curator LLM lane -- and that lane deliberately does not start without a synthesis endpoint ("NO SYNTHESIS PROVIDER => NO LLM LANE"). Both decisions are individually correct. Together they mean an install with no synthesis provider, which is a supported configuration, enqueues one row per stored memory that nothing will ever claim. Found by exploratory testing of :testing, not by a failing check. On the e2e VM: 4 memory_facts jobs pending for 11.5 hours with attempts=0, while /v1/health reported status ok with an empty warnings array and `aimee kb status` printed "4 pending" with nothing to say that pending here means forever. It grows without bound for the life of the install. Reported as a WARNING, not a blocker. Running without a synthesis provider is explicitly supported and memory store and search work perfectly; only typed-fact enrichment is deferred. Degrading the verdict for a supported configuration is exactly the dilution the status derivation exists to avoid. The message says the backlog is not lost: configuring a provider drains it. Counting needs PENDING specifically, so add db2_kb_async_count_kind_pending alongside the existing total. Total conflates work still waiting with work long finished and cannot tell a queue that is draining from one that never will -- which is the entire distinction being reported. Test red-verified by dropping the status filter. --- dependencies/aimee-repositories.lock.json | 2 +- src/db2/kb_payload.c | 21 +++++++++++++ src/db2/kb_payload.h | 6 ++++ src/kb/kb_service_kb.c | 38 +++++++++++++++++++++++ src/tests/test_curator_queue.c | 38 +++++++++++++++++++++++ 5 files changed, 104 insertions(+), 1 deletion(-) diff --git a/dependencies/aimee-repositories.lock.json b/dependencies/aimee-repositories.lock.json index afca55501..845e13aef 100644 --- a/dependencies/aimee-repositories.lock.json +++ b/dependencies/aimee-repositories.lock.json @@ -378,7 +378,7 @@ "placements": [ "kb" ], - "source_sha256": "66d64ce9e3efb425d4b0bf7aaa6481f74d6d9735fed7f89b915b28b0f22551d2", + "source_sha256": "8f3a706ef4e2aeef3250b7cc33c0c2bbd3f29250ed24bdaeff548ae3f2cf8878", "runtime": "go", "principal_class": 1, "principal_ref": 22, diff --git a/src/db2/kb_payload.c b/src/db2/kb_payload.c index 15cf8f9e3..837b9cd2f 100644 --- a/src/db2/kb_payload.c +++ b/src/db2/kb_payload.c @@ -1131,6 +1131,27 @@ int db2_kb_pdf_search_chunks(const char *project, const char *query, int max, return n; } +int db2_kb_async_count_kind_pending(const char *kind) +{ + if (!kind || !*kind) + return -1; + void *conn = db2_conn(); + if (!conn) + return -1; + char err[KBP_ERRBUF] = ""; + aimee_pg_stmt_t *st = aimee_pg_prepare( + conn, "SELECT COUNT(*) FROM kb_async_jobs WHERE kind = ?1 AND status = 'pending'", err, + sizeof(err)); + if (!st) + return -1; + aimee_pg_bind_text(st, "?1", kind); + int n = -1; + if (aimee_pg_step(st, err, sizeof(err)) == AIMEE_PG_ROW) + n = aimee_pg_column_int(st, 0); + aimee_pg_finalize(st); + return n; +} + int db2_kb_async_count_kind(const char *kind) { if (!kind || !*kind) diff --git a/src/db2/kb_payload.h b/src/db2/kb_payload.h index d7cace71c..f1bb02071 100644 --- a/src/db2/kb_payload.h +++ b/src/db2/kb_payload.h @@ -240,6 +240,12 @@ extern "C" * helper. Returns the count (>=0) or -1 on error. */ int db2_kb_async_count_kind(const char *kind); + /* Count only the PENDING rows for a kind. Separate from the total above because a + * backlog is the interesting quantity: total conflates work still waiting with work + * long finished, and cannot distinguish a queue that is draining from one nothing + * will ever claim. Returns the count (>=0) or -1 on error. */ + int db2_kb_async_count_kind_pending(const char *kind); + /* structured-pdf Phase B: a recognised table cell. Stored ONLY in kb_table_cells (never * in the shared typed_facts table — see schema.sql). */ typedef struct diff --git a/src/kb/kb_service_kb.c b/src/kb/kb_service_kb.c index ee544e36d..cfcfde1cd 100644 --- a/src/kb/kb_service_kb.c +++ b/src/kb/kb_service_kb.c @@ -489,6 +489,44 @@ static cJSON *kb_service_health_object(void) snprintf(msg, sizeof(msg), "curator: %d job(s) failing", n); cJSON_AddItemToArray(warnings, cJSON_CreateString(msg)); } + /* TWO GATES THAT DO NOT AGREE. + * + * memory.store enqueues a "memory_facts" job whenever typed_facts_enabled is on, + * which is the DEFAULT. The only consumer of those jobs is kb_memory_facts_drain, + * which runs on the curator LLM lane -- and that lane deliberately does not start + * without a synthesis endpoint ("NO SYNTHESIS PROVIDER => NO LLM LANE"). Both + * decisions are individually right. Together they mean an install with no synth + * provider, which is a supported configuration, enqueues one row per stored + * memory that nothing will ever claim. + * + * Measured on the e2e VM: 4 memory_facts jobs pending for 11.5 hours with + * attempts=0, while /v1/health reported status ok and an empty warnings array, + * and `aimee kb status` printed "4 pending" with no hint that pending here means + * forever. It grows without bound for the life of the install. + * + * NOT a blocker. Running without a synthesis provider is explicitly supported, + * and memory store and search work perfectly; only typed-fact enrichment is + * deferred. Degrading the verdict for a supported configuration is exactly the + * dilution this file's status derivation exists to avoid. It is a warning, and + * it says the backlog is not lost -- configuring a provider drains it. */ + if (config_typed_facts_enabled()) + { + int facts_pending = db2_kb_async_count_kind_pending("memory_facts"); + char synth_endpoint[512]; + if (facts_pending > 0 && + !config_synth_chat_endpoint_current(synth_endpoint, sizeof(synth_endpoint))) + { + char msg[320]; + snprintf(msg, sizeof(msg), + "typed-fact extraction: %d job(s) queued with nothing to drain them — no " + "synthesis endpoint is configured, so the curator LLM lane is not running. " + "Memories are stored and searchable but contribute no typed facts. The " + "backlog is not lost: configuring a synthesis provider drains it.", + facts_pending); + cJSON_AddItemToArray(warnings, cJSON_CreateString(msg)); + } + } + if (freshness_days > 30) cJSON_AddItemToArray(warnings, cJSON_CreateString("KB not ingested in over 30 days")); else if (freshness_days > 7) diff --git a/src/tests/test_curator_queue.c b/src/tests/test_curator_queue.c index 09e77f9ec..dfe40c24d 100644 --- a/src/tests/test_curator_queue.c +++ b/src/tests/test_curator_queue.c @@ -127,6 +127,43 @@ static void test_polymorphic_async_subject(sqlite3 *db) printf(" PASS: polymorphic async subject accepts a memory id without a document row\n"); } +/* A backlog nothing will drain has to be countable, and countable SEPARATELY from + * work already finished. + * + * memory.store enqueues a memory_facts job whenever typed_facts is on (the + * default); the only consumer runs on the curator LLM lane, which does not start + * without a synthesis endpoint. On an install with no synth provider -- a + * supported configuration -- that queue grows by one row per stored memory and is + * never claimed. Observed on the e2e VM: 4 pending for 11.5 hours, attempts=0, + * while health reported ok with an empty warnings array. + * + * The health surface reports it, and to do that it needs PENDING, not total: + * total cannot tell a queue that is draining from one that never will. */ +static void test_pending_count_excludes_finished(sqlite3 *db) +{ + seed(db, "DELETE FROM kb_async_jobs WHERE kind='memory_facts'"); + assert(db2_kb_async_count_kind_pending("memory_facts") == 0); + + assert(db2_kb_async_enqueue("memory_facts", 881001, "memory") == 0); + assert(db2_kb_async_enqueue("memory_facts", 881002, "memory") == 0); + assert(db2_kb_async_count_kind_pending("memory_facts") == 2); + assert(db2_kb_async_count_kind("memory_facts") == 2); + + /* One finishes. The backlog is 1, even though two rows exist -- reporting 2 + * here would keep warning about work that already completed. */ + seed(db, "UPDATE kb_async_jobs SET status='done' WHERE kind='memory_facts'" + " AND document_id=881001"); + assert(db2_kb_async_count_kind_pending("memory_facts") == 1); + assert(db2_kb_async_count_kind("memory_facts") == 2); + + /* Other kinds are not counted into this backlog: kb_async_jobs is shared. */ + assert(db2_kb_async_enqueue("extract_doc", 881003, "p") == 0); + assert(db2_kb_async_count_kind_pending("memory_facts") == 1); + + seed(db, "DELETE FROM kb_async_jobs WHERE document_id IN (881001,881002,881003)"); + printf(" PASS: pending count is the backlog, not the row total\n"); +} + static const char *job_status(sqlite3 *db, int64_t id) { static char buf[64]; @@ -359,6 +396,7 @@ int main(void) " VALUES ('p','/p','2026-01-01 00:00:00')"); test_polymorphic_async_subject(db); + test_pending_count_excludes_finished(db); /* Contract: every non-pdf doc gets one extract_doc job; PDFs are excluded; * re-running enqueues nothing. Guard for "docs present but zero jobs". */ From c72cca682ae7c963e54dd1abad4f0adbf139ac44 Mon Sep 17 00:00:00 2001 From: JBailes Date: Tue, 4 Aug 2026 10:08:21 +0000 Subject: [PATCH 2/3] fix: include config_database.h for the synth endpoint accessor The kb target compiles kb_service_kb.c with a different include set than the server target, so config_synth_chat_endpoint_current was an implicit declaration there and `make ../aimee-kb` failed under -Werror. The container build is that target, so the image build broke while the default target still passed. --- src/kb/kb_service_kb.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/kb/kb_service_kb.c b/src/kb/kb_service_kb.c index cfcfde1cd..6fa4858e0 100644 --- a/src/kb/kb_service_kb.c +++ b/src/kb/kb_service_kb.c @@ -5,6 +5,7 @@ #include "aimee.h" #include "cJSON.h" #include "config.h" +#include "config_database.h" /* config_synth_chat_endpoint_current */ #include "kb_curator_queue.h" #include "kb_curator_provider.h" #include "json_fluent.h" From 939626031647737c94d4d71682af7920a35c379b Mon Sep 17 00:00:00 2001 From: JBailes Date: Tue, 4 Aug 2026 10:15:02 +0000 Subject: [PATCH 3/3] fix: carry kb warnings to the operator, not just blockers The backlog warning added in the previous commit reached /v1/health and stopped there. server_health_add_kb passed blockers through and dropped the kb's warnings array on the floor, and the status renderer only ever printed blockers. Verified live before this commit: the kb reported the 4-job typed-fact backlog in its warnings array, while `aimee status` printed "aimee-kb: ok" with four clean detail lines and `aimee kb status` printed a bare "4 pending". Publishing a finding into a field no surface renders is the same defect as never computing it -- the evidence exists and the summary does not carry it, which is the whole shape this series is about. Warnings are advisory by construction: they do not move the verdict, and that is exactly why they need printing. A kb that is genuinely ok can still be accumulating work nothing will process, and a status line that renders only blockers reports that as a clean bill of health. Rendered as "note:" to keep it visibly distinct from "BLOCKED:", which means cannot work. Extracted kb_health_add_lines: two copies of the same newline-splitting arithmetic is how the warnings array came to be dropped to begin with. Test asserts warnings survive with the verdict unchanged, and that the blockers key is absent rather than empty when there are none. Red verified by removing the pass-through. --- src/cli_v1_routes_b.c | 13 ++++++++ src/server/server_api_status.c | 53 ++++++++++++++++++++++---------- src/tests/test_server_dispatch.c | 23 ++++++++++++++ 3 files changed, 72 insertions(+), 17 deletions(-) diff --git a/src/cli_v1_routes_b.c b/src/cli_v1_routes_b.c index 27ccf84e1..92cba18de 100644 --- a/src/cli_v1_routes_b.c +++ b/src/cli_v1_routes_b.c @@ -1611,6 +1611,19 @@ void print_server_health(cJSON *resp) cJSON_ArrayForEach(b, blockers) if (cJSON_IsString(b)) printf(" BLOCKED: %s\n", b->valuestring); } + /* Advisory findings. They do not move the verdict, which is exactly why they + * need printing: a kb that is genuinely "ok" can still be accumulating work + * nothing will process, and a status line that only ever renders blockers + * reports that as a clean bill of health. Measured: a typed-fact backlog of + * 4 jobs, unclaimable for 11.5 hours, with `aimee status` showing "aimee-kb: + * ok" and nothing else. */ + cJSON *kbwarn = cJSON_GetObjectItemCaseSensitive(kb, "warnings"); + if (cJSON_IsArray(kbwarn) && cJSON_GetArraySize(kbwarn) > 0) + { + cJSON *w; + cJSON_ArrayForEach(w, kbwarn) if (cJSON_IsString(w)) + printf(" note: %s\n", w->valuestring); + } /* An open transport breaker refuses every call locally, so the kb can be * "ok" here while nothing works. Print it before the detail lines: this is * the line that explains an index that answers "unavailable" on a server diff --git a/src/server/server_api_status.c b/src/server/server_api_status.c index c1f2114a6..ce4673662 100644 --- a/src/server/server_api_status.c +++ b/src/server/server_api_status.c @@ -95,6 +95,31 @@ static int api_bearer_extra_count(void); * storage tier's vocabulary — build-integrity greps the client binary for exactly * that and fails the build. The client reports whether the kb's store and vector * index are healthy without knowing what implements them. */ +/* Split a newline-joined kb field back into a JSON array under `key`, omitting the + * key entirely when there is nothing to say. Extracted when warnings joined + * blockers: two copies of the same pointer arithmetic was how the warnings array + * came to be dropped in the first place. */ +static void kb_health_add_lines(cJSON *kbo, const char *key, const char *joined) +{ + if (!kbo || !key || !joined || !joined[0]) + return; + cJSON *arr = cJSON_AddArrayToObject(kbo, key); + if (!arr) + return; + for (const char *p = joined; p && *p;) + { + const char *nl = strchr(p, '\n'); + size_t len = nl ? (size_t)(nl - p) : strlen(p); + char line[384]; + if (len >= sizeof(line)) + len = sizeof(line) - 1; + memcpy(line, p, len); + line[len] = '\0'; + cJSON_AddItemToArray(arr, cJSON_CreateString(line)); + p = nl ? nl + 1 : NULL; + } +} + void server_health_add_kb(cJSON *resp) { if (!resp) @@ -150,23 +175,17 @@ void server_health_add_kb(cJSON *resp) return; /* Pass the reasons through verbatim. The kb composed them next to the evidence * and named the remedy; re-deriving them here from the booleans below would be - * a second place for the verdict to drift out of step with the facts. */ - if (kb.blockers[0]) - { - cJSON *arr = cJSON_AddArrayToObject(kbo, "blockers"); - for (const char *p = kb.blockers; p && *p;) - { - const char *nl = strchr(p, '\n'); - size_t len = nl ? (size_t)(nl - p) : strlen(p); - char line[320]; - if (len >= sizeof(line)) - len = sizeof(line) - 1; - memcpy(line, p, len); - line[len] = '\0'; - cJSON_AddItemToArray(arr, cJSON_CreateString(line)); - p = nl ? nl + 1 : NULL; - } - } + * a second place for the verdict to drift out of step with the facts. + * + * WARNINGS TRAVEL TOO, not just blockers. The kb has always assembled a + * warnings array and this block dropped it on the floor, so findings that do + * not move the verdict reached no operator at all: a typed-fact backlog that + * nothing will ever drain sat in /v1/health for hours while `aimee status` + * showed a clean kb and `aimee kb status` printed a bare "4 pending". Publishing + * a finding into a field no surface renders is the same defect as not computing + * it -- the evidence exists and the summary does not carry it. */ + kb_health_add_lines(kbo, "blockers", kb.blockers); + kb_health_add_lines(kbo, "warnings", kb.warnings); cJSON_AddBoolToObject(kbo, "store_ok", kb.db2_ok ? 1 : 0); cJSON_AddBoolToObject(kbo, "vectors_ok", kb.pgvec_ok ? 1 : 0); cJSON_AddBoolToObject(kbo, "embed_configured", kb.embed_ok ? 1 : 0); diff --git a/src/tests/test_server_dispatch.c b/src/tests/test_server_dispatch.c index 3c25bcd55..3dc66639d 100644 --- a/src/tests/test_server_dispatch.c +++ b/src/tests/test_server_dispatch.c @@ -2282,6 +2282,29 @@ static void test_health_kb_verdict_states(void) assert(strcmp(cJSON_GetArrayItem(blockers, 1)->valuestring, "vector table missing") == 0); cJSON_Delete(resp); + /* Warnings travel even though they do not move the verdict. This block used to + * drop the kb's warnings array entirely, so an advisory finding reached no + * operator: a typed-fact backlog nothing could drain sat unreported for hours + * behind a status of "ok". Publishing a finding into a field no surface renders + * is the same defect as never computing it. */ + kb_health_stub_reset(); + g_kb_health_rc = 0; + g_kb_health.process_ok = 1; + snprintf(g_kb_health.status, sizeof(g_kb_health.status), "ok"); + snprintf(g_kb_health.warnings, sizeof(g_kb_health.warnings), + "typed-fact extraction: 4 job(s) queued with nothing to drain them\nKB not ingested " + "in over 7 days"); + resp = cJSON_CreateObject(); + server_health_add_kb(resp); + assert(strcmp(kb_status_of(resp), "ok") == 0); /* advisory: verdict unchanged */ + kb = cJSON_GetObjectItemCaseSensitive(resp, "kb"); + cJSON *warns = cJSON_GetObjectItemCaseSensitive(kb, "warnings"); + assert(cJSON_IsArray(warns) && cJSON_GetArraySize(warns) == 2); + assert(strstr(cJSON_GetArrayItem(warns, 0)->valuestring, "typed-fact extraction") != NULL); + /* No blockers key at all when there are none, rather than an empty array. */ + assert(cJSON_GetObjectItemCaseSensitive(kb, "blockers") == NULL); + cJSON_Delete(resp); + /* 4. An open transport breaker refuses every call locally, so a kb that * considers itself perfectly healthy still cannot be queried. The breaker is * part of the verdict rather than a flag beside it. */