From d742b9f3d2aa772bf31bdf97029f8a8f1fb25479 Mon Sep 17 00:00:00 2001 From: Jack Arturo Date: Thu, 11 Jun 2026 00:20:43 +0200 Subject: [PATCH 1/3] feat(recall): cap tag-score denominator to fix query-length bias MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tag-overlap score divided token hits by the full query length, so a 1-token query earned full tag credit from a single hit while a 5-token query needed all 5 tokens to match — biasing short queries up and long queries down. Introduce SEARCH_TAG_SCORE_TOKEN_CAP (default 3): the denominator becomes min(len(query_tokens), cap), clipped to 1.0 on the way out. Setting the cap to 0 restores the legacy full-query-length denominator; negative values fall back to the default (safer than treating a typo'd negative as an intentional legacy opt-out), and unparseable values raise like the neighboring int()/float() env parses. This intentionally changes default scoring for queries longer than 3 tokens. The default-change ships behind the section-4 eval gates per the release plan; the legacy escape hatch (cap=0) is covered by tests. Co-Authored-By: Claude Fable 5 --- automem/config.py | 22 +++++++++ automem/utils/scoring.py | 14 +++++- docs/ENVIRONMENT_VARIABLES.md | 1 + tests/test_api_endpoints.py | 89 ++++++++++++++++++++++++++++++++++- 4 files changed, 124 insertions(+), 2 deletions(-) diff --git a/automem/config.py b/automem/config.py index 8b3cea8..0aad92c 100644 --- a/automem/config.py +++ b/automem/config.py @@ -484,6 +484,28 @@ def _positive_or_default(raw: str, default: float) -> float: _RECENCY_CURVE_RAW = os.getenv("SEARCH_RECENCY_CURVE", "linear").strip().lower() SEARCH_RECENCY_CURVE = _RECENCY_CURVE_RAW if _RECENCY_CURVE_RAW in {"linear", "exp"} else "linear" + +def _non_negative_int_or_default(raw: str, default: int) -> int: + """Parse an int env value, falling back to ``default`` when negative. + + Unparseable values raise ValueError, matching the neighboring int()/float() + parses. 0 is a valid sentinel here (it selects legacy behavior), so only + negative values fall back — mirroring ``_positive_or_default``'s + fail-safe-to-default spirit rather than silently meaning "legacy". + """ + value = int(raw) + return value if value >= 0 else default + + +# Tag-score query-length normalization: the tag-overlap score divides token +# hits by min(len(query_tokens), cap) so long queries aren't penalized +# relative to short ones. 0 disables the cap (legacy: denominator = full +# query length). Negative values fall back to the default of 3 — falling back +# is safer than treating a typo'd negative as an intentional legacy opt-out. +SEARCH_TAG_SCORE_TOKEN_CAP = _non_negative_int_or_default( + os.getenv("SEARCH_TAG_SCORE_TOKEN_CAP", "3"), 3 +) + # API tokens API_TOKEN = os.getenv("AUTOMEM_API_TOKEN") ADMIN_TOKEN = os.getenv("ADMIN_API_TOKEN") diff --git a/automem/utils/scoring.py b/automem/utils/scoring.py index eba7648..56664df 100644 --- a/automem/utils/scoring.py +++ b/automem/utils/scoring.py @@ -7,6 +7,7 @@ from automem.config import ( SEARCH_RECENCY_CURVE, SEARCH_RECENCY_WINDOW_DAYS, + SEARCH_TAG_SCORE_TOKEN_CAP, SEARCH_WEIGHT_CONFIDENCE, SEARCH_WEIGHT_EXACT, SEARCH_WEIGHT_IMPORTANCE, @@ -157,7 +158,18 @@ def _compute_metadata_score( recency_score = _compute_recency_score(memory.get("timestamp")) - tag_score = token_hits / max(len(tokens), 1) if tokens else 0.0 + if tokens: + if SEARCH_TAG_SCORE_TOKEN_CAP > 0: + # Cap the denominator so long queries aren't penalized relative to + # short ones: a query with more tokens than the cap only needs + # `cap` tag/metadata hits for full credit. + denominator = max(min(len(tokens), SEARCH_TAG_SCORE_TOKEN_CAP), 1) + else: + # Legacy behavior (cap == 0): denominator is the full query length. + denominator = max(len(tokens), 1) + tag_score = min(1.0, token_hits / denominator) + else: + tag_score = 0.0 vector_component = ( result.get("match_score", 0.0) if result.get("match_type") == "vector" else 0.0 diff --git a/docs/ENVIRONMENT_VARIABLES.md b/docs/ENVIRONMENT_VARIABLES.md index 00c0f34..64ffa3a 100644 --- a/docs/ENVIRONMENT_VARIABLES.md +++ b/docs/ENVIRONMENT_VARIABLES.md @@ -316,6 +316,7 @@ Controls how different factors are weighted in memory recall scoring. | `SEARCH_WEIGHT_METADATA` | Metadata sidecar match | `0.35` | Candidates admitted via the metadata sidecar channel (see `RECALL_METADATA_SEARCH_ENABLED`) | | `SEARCH_WEIGHT_RELATION` | Graph relationship boost | `0.25` | Memories connected via edges | | `SEARCH_WEIGHT_TAG` | Tag matching | `0.20` | Tag overlap scoring | +| `SEARCH_TAG_SCORE_TOKEN_CAP` | Tag-score denominator cap | `3` | Tag score divides token hits by `min(query tokens, cap)` so long queries aren't penalized; `0` restores the legacy full-query-length denominator. Not a weight | | `SEARCH_WEIGHT_EXACT` | Exact phrase match | `0.20` | Full query in metadata | | `SEARCH_WEIGHT_IMPORTANCE` | Memory importance | `0.10` | User/system defined | | `SEARCH_WEIGHT_RECENCY` | Recent memories | `0.10` | Decay shaped by `SEARCH_RECENCY_WINDOW_DAYS` and `SEARCH_RECENCY_CURVE` | diff --git a/tests/test_api_endpoints.py b/tests/test_api_endpoints.py index 47309b7..3886214 100644 --- a/tests/test_api_endpoints.py +++ b/tests/test_api_endpoints.py @@ -402,7 +402,11 @@ def test_compute_metadata_score_preserves_keyword_match_score_for_trending_resul assert components["keyword"] == 0.33 -def test_compute_metadata_score_ignores_generated_entities_for_generic_tag_score(): +def test_compute_metadata_score_ignores_generated_entities_for_generic_tag_score(monkeypatch): + # Pin the cap (config.py runs load_dotenv() at import, so a tuned .env + # could otherwise leak in); 1 hit / min(3 tokens, cap 3) == 1/3 either way. + monkeypatch.setattr(scoring, "SEARCH_TAG_SCORE_TOKEN_CAP", 3) + _score, components = _compute_metadata_score( { "match_type": "vector", @@ -426,6 +430,89 @@ def test_compute_metadata_score_ignores_generated_entities_for_generic_tag_score assert components["tag"] == 1 / 3 +def _tag_score_result(tags: list) -> dict: + return { + "match_type": "vector", + "match_score": 0.7, + "memory": {"content": "Benchmark notes", "tags": tags}, + } + + +def test_compute_metadata_score_tag_score_single_token_full_credit(monkeypatch): + monkeypatch.setattr(scoring, "SEARCH_TAG_SCORE_TOKEN_CAP", 3) + + _score, components = _compute_metadata_score( + _tag_score_result(["automem"]), + "automem", + ["automem"], + ) + + assert components["tag"] == 1.0 + + +def test_compute_metadata_score_tag_score_caps_denominator_for_long_queries(monkeypatch): + monkeypatch.setattr(scoring, "SEARCH_TAG_SCORE_TOKEN_CAP", 3) + + _score, components = _compute_metadata_score( + _tag_score_result(["alpha", "bravo"]), + "alpha bravo charlie delta echo", + ["alpha", "bravo", "charlie", "delta", "echo"], + ) + + # 2 hits over min(5, cap=3) instead of the legacy 2/5 + assert components["tag"] == pytest.approx(2 / 3, abs=1e-9) + + +def test_compute_metadata_score_tag_score_clips_at_one_when_hits_exceed_cap(monkeypatch): + monkeypatch.setattr(scoring, "SEARCH_TAG_SCORE_TOKEN_CAP", 3) + + _score, components = _compute_metadata_score( + _tag_score_result(["alpha", "bravo", "charlie", "delta"]), + "alpha bravo charlie delta echo", + ["alpha", "bravo", "charlie", "delta", "echo"], + ) + + # 4 hits over a capped denominator of 3 would exceed 1.0; clip it. + assert components["tag"] == 1.0 + + +def test_compute_metadata_score_tag_score_cap_zero_restores_legacy_denominator(monkeypatch): + monkeypatch.setattr(scoring, "SEARCH_TAG_SCORE_TOKEN_CAP", 0) + + _score, components = _compute_metadata_score( + _tag_score_result(["alpha", "bravo"]), + "alpha bravo charlie delta echo", + ["alpha", "bravo", "charlie", "delta", "echo"], + ) + + # Legacy behavior: denominator is the full query length (2/5) + assert components["tag"] == pytest.approx(0.4, abs=1e-9) + + +def test_compute_metadata_score_tag_score_short_query_below_cap_uses_query_length(monkeypatch): + monkeypatch.setattr(scoring, "SEARCH_TAG_SCORE_TOKEN_CAP", 3) + + _score, components = _compute_metadata_score( + _tag_score_result(["alpha"]), + "alpha zulu", + ["alpha", "zulu"], + ) + + # Below the cap, the denominator stays min(len(tokens), cap) == 2 + assert components["tag"] == pytest.approx(0.5, abs=1e-9) + + +def test_tag_score_token_cap_config_falls_back_on_negative_values(): + # 0 is a valid sentinel (legacy full-length denominator), so only negative + # values fall back to the default; unparseable values raise like the + # neighboring int()/float() env parses. + assert config._non_negative_int_or_default("-1", 3) == 3 + assert config._non_negative_int_or_default("0", 3) == 0 + assert config._non_negative_int_or_default("5", 3) == 5 + with pytest.raises(ValueError): + config._non_negative_int_or_default("not-an-int", 3) + + def _timestamp_days_ago(days: float) -> str: return (datetime.now(timezone.utc) - timedelta(days=days)).isoformat() From a227afe8fa4223c7f1783a562cd1a0327fd22f64 Mon Sep 17 00:00:00 2001 From: Jack Arturo Date: Thu, 11 Jun 2026 04:40:17 +0200 Subject: [PATCH 2/3] fix(scoring)!: default SEARCH_TAG_SCORE_TOKEN_CAP to 0 (opt-in) after production-corpus regression evidence Production-corpus A/B testing (2026-06-11, 200 queries against a 10,107-memory production clone, 3-run baseline R@5 0.650-0.655 / MRR 0.429-0.433) showed every tested cap value regresses recall on ungated free-text queries: - cap=2: R@5 0.510 (-14.2pp, paired p<0.0001), MRR 0.324 - cap=3 (previous default): R@5 0.580 (-7.2pp, p=0.0002), MRR 0.361 - cap=4: R@5 0.615 (-3.7pp, p=0.0103), MRR 0.390 - cap=0 (legacy) = baseline Mechanism: on ungated free-text queries the capped denominator inflates tag scores (1 matching tag on a 12-token query: 0.083 -> 0.33), amplifying tag noise over vector/keyword evidence. A two-stack probe A/B also showed cap=3 raising top-1 scores on known-garbage negative probes by +0.04. The cap remains available as an opt-in for tag-scoped retrieval experiments; all cap-behavior tests already pin the value explicitly via monkeypatch, so no test expectations changed. Co-Authored-By: Claude Fable 5 --- automem/config.py | 10 +++++++--- docs/ENVIRONMENT_VARIABLES.md | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/automem/config.py b/automem/config.py index 0aad92c..c6a843f 100644 --- a/automem/config.py +++ b/automem/config.py @@ -500,10 +500,14 @@ def _non_negative_int_or_default(raw: str, default: int) -> int: # Tag-score query-length normalization: the tag-overlap score divides token # hits by min(len(query_tokens), cap) so long queries aren't penalized # relative to short ones. 0 disables the cap (legacy: denominator = full -# query length). Negative values fall back to the default of 3 — falling back -# is safer than treating a typo'd negative as an intentional legacy opt-out. +# query length). Default is 0 (opt-in): a production-corpus A/B (2026-06-11, +# 200 queries, 10k-memory clone) showed cap values 2/3/4 regress Recall@5 by +# 14/7/4pp on ungated queries — the capped denominator inflates tag scores +# and amplifies tag noise over vector/keyword evidence. Negative values fall +# back to the default — falling back is safer than treating a typo'd +# negative as intentional. SEARCH_TAG_SCORE_TOKEN_CAP = _non_negative_int_or_default( - os.getenv("SEARCH_TAG_SCORE_TOKEN_CAP", "3"), 3 + os.getenv("SEARCH_TAG_SCORE_TOKEN_CAP", "0"), 0 ) # API tokens diff --git a/docs/ENVIRONMENT_VARIABLES.md b/docs/ENVIRONMENT_VARIABLES.md index 64ffa3a..1519fc2 100644 --- a/docs/ENVIRONMENT_VARIABLES.md +++ b/docs/ENVIRONMENT_VARIABLES.md @@ -316,7 +316,7 @@ Controls how different factors are weighted in memory recall scoring. | `SEARCH_WEIGHT_METADATA` | Metadata sidecar match | `0.35` | Candidates admitted via the metadata sidecar channel (see `RECALL_METADATA_SEARCH_ENABLED`) | | `SEARCH_WEIGHT_RELATION` | Graph relationship boost | `0.25` | Memories connected via edges | | `SEARCH_WEIGHT_TAG` | Tag matching | `0.20` | Tag overlap scoring | -| `SEARCH_TAG_SCORE_TOKEN_CAP` | Tag-score denominator cap | `3` | Tag score divides token hits by `min(query tokens, cap)` so long queries aren't penalized; `0` restores the legacy full-query-length denominator. Not a weight | +| `SEARCH_TAG_SCORE_TOKEN_CAP` | Tag-score denominator cap | `0` (opt-in) | When > 0, tag score divides token hits by `min(query tokens, cap)`; `0` (default) keeps the legacy full-query-length denominator. Production-corpus A/B (2026-06-11, 200 queries, 10k-memory clone) showed cap values 2/3/4 regress Recall@5 by 14/7/4pp on ungated queries; enable only for tag-scoped retrieval experiments. Not a weight | | `SEARCH_WEIGHT_EXACT` | Exact phrase match | `0.20` | Full query in metadata | | `SEARCH_WEIGHT_IMPORTANCE` | Memory importance | `0.10` | User/system defined | | `SEARCH_WEIGHT_RECENCY` | Recent memories | `0.10` | Decay shaped by `SEARCH_RECENCY_WINDOW_DAYS` and `SEARCH_RECENCY_CURVE` | From f6e3099f3544d67fc56ecabb856ea4ea30560fdd Mon Sep 17 00:00:00 2001 From: Jack Arturo Date: Thu, 11 Jun 2026 18:05:07 +0200 Subject: [PATCH 3/3] test(recall): tighten tag score helper typing --- tests/test_api_endpoints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_api_endpoints.py b/tests/test_api_endpoints.py index 3886214..ee948cf 100644 --- a/tests/test_api_endpoints.py +++ b/tests/test_api_endpoints.py @@ -430,7 +430,7 @@ def test_compute_metadata_score_ignores_generated_entities_for_generic_tag_score assert components["tag"] == 1 / 3 -def _tag_score_result(tags: list) -> dict: +def _tag_score_result(tags: list[str]) -> dict[str, Any]: return { "match_type": "vector", "match_score": 0.7,