diff --git a/automem/config.py b/automem/config.py index 8b3cea82..c6a843f8 100644 --- a/automem/config.py +++ b/automem/config.py @@ -484,6 +484,32 @@ def _positive_or_default(raw: str, default: float) -> float: _RECENCY_CURVE_RAW = os.getenv("SEARCH_RECENCY_CURVE", "linear").strip().lower() SEARCH_RECENCY_CURVE = _RECENCY_CURVE_RAW if _RECENCY_CURVE_RAW in {"linear", "exp"} else "linear" + +def _non_negative_int_or_default(raw: str, default: int) -> int: + """Parse an int env value, falling back to ``default`` when negative. + + Unparseable values raise ValueError, matching the neighboring int()/float() + parses. 0 is a valid sentinel here (it selects legacy behavior), so only + negative values fall back — mirroring ``_positive_or_default``'s + fail-safe-to-default spirit rather than silently meaning "legacy". + """ + value = int(raw) + return value if value >= 0 else default + + +# Tag-score query-length normalization: the tag-overlap score divides token +# hits by min(len(query_tokens), cap) so long queries aren't penalized +# relative to short ones. 0 disables the cap (legacy: denominator = full +# query length). Default is 0 (opt-in): a production-corpus A/B (2026-06-11, +# 200 queries, 10k-memory clone) showed cap values 2/3/4 regress Recall@5 by +# 14/7/4pp on ungated queries — the capped denominator inflates tag scores +# and amplifies tag noise over vector/keyword evidence. Negative values fall +# back to the default — falling back is safer than treating a typo'd +# negative as intentional. +SEARCH_TAG_SCORE_TOKEN_CAP = _non_negative_int_or_default( + os.getenv("SEARCH_TAG_SCORE_TOKEN_CAP", "0"), 0 +) + # API tokens API_TOKEN = os.getenv("AUTOMEM_API_TOKEN") ADMIN_TOKEN = os.getenv("ADMIN_API_TOKEN") diff --git a/automem/utils/scoring.py b/automem/utils/scoring.py index eba7648a..56664dfe 100644 --- a/automem/utils/scoring.py +++ b/automem/utils/scoring.py @@ -7,6 +7,7 @@ from automem.config import ( SEARCH_RECENCY_CURVE, SEARCH_RECENCY_WINDOW_DAYS, + SEARCH_TAG_SCORE_TOKEN_CAP, SEARCH_WEIGHT_CONFIDENCE, SEARCH_WEIGHT_EXACT, SEARCH_WEIGHT_IMPORTANCE, @@ -157,7 +158,18 @@ def _compute_metadata_score( recency_score = _compute_recency_score(memory.get("timestamp")) - tag_score = token_hits / max(len(tokens), 1) if tokens else 0.0 + if tokens: + if SEARCH_TAG_SCORE_TOKEN_CAP > 0: + # Cap the denominator so long queries aren't penalized relative to + # short ones: a query with more tokens than the cap only needs + # `cap` tag/metadata hits for full credit. + denominator = max(min(len(tokens), SEARCH_TAG_SCORE_TOKEN_CAP), 1) + else: + # Legacy behavior (cap == 0): denominator is the full query length. + denominator = max(len(tokens), 1) + tag_score = min(1.0, token_hits / denominator) + else: + tag_score = 0.0 vector_component = ( result.get("match_score", 0.0) if result.get("match_type") == "vector" else 0.0 diff --git a/docs/ENVIRONMENT_VARIABLES.md b/docs/ENVIRONMENT_VARIABLES.md index 00c0f34e..1519fc2a 100644 --- a/docs/ENVIRONMENT_VARIABLES.md +++ b/docs/ENVIRONMENT_VARIABLES.md @@ -316,6 +316,7 @@ Controls how different factors are weighted in memory recall scoring. | `SEARCH_WEIGHT_METADATA` | Metadata sidecar match | `0.35` | Candidates admitted via the metadata sidecar channel (see `RECALL_METADATA_SEARCH_ENABLED`) | | `SEARCH_WEIGHT_RELATION` | Graph relationship boost | `0.25` | Memories connected via edges | | `SEARCH_WEIGHT_TAG` | Tag matching | `0.20` | Tag overlap scoring | +| `SEARCH_TAG_SCORE_TOKEN_CAP` | Tag-score denominator cap | `0` (opt-in) | When > 0, tag score divides token hits by `min(query tokens, cap)`; `0` (default) keeps the legacy full-query-length denominator. Production-corpus A/B (2026-06-11, 200 queries, 10k-memory clone) showed cap values 2/3/4 regress Recall@5 by 14/7/4pp on ungated queries; enable only for tag-scoped retrieval experiments. Not a weight | | `SEARCH_WEIGHT_EXACT` | Exact phrase match | `0.20` | Full query in metadata | | `SEARCH_WEIGHT_IMPORTANCE` | Memory importance | `0.10` | User/system defined | | `SEARCH_WEIGHT_RECENCY` | Recent memories | `0.10` | Decay shaped by `SEARCH_RECENCY_WINDOW_DAYS` and `SEARCH_RECENCY_CURVE` | diff --git a/tests/test_api_endpoints.py b/tests/test_api_endpoints.py index 47309b74..ee948cff 100644 --- a/tests/test_api_endpoints.py +++ b/tests/test_api_endpoints.py @@ -402,7 +402,11 @@ def test_compute_metadata_score_preserves_keyword_match_score_for_trending_resul assert components["keyword"] == 0.33 -def test_compute_metadata_score_ignores_generated_entities_for_generic_tag_score(): +def test_compute_metadata_score_ignores_generated_entities_for_generic_tag_score(monkeypatch): + # Pin the cap (config.py runs load_dotenv() at import, so a tuned .env + # could otherwise leak in); 1 hit / min(3 tokens, cap 3) == 1/3 either way. + monkeypatch.setattr(scoring, "SEARCH_TAG_SCORE_TOKEN_CAP", 3) + _score, components = _compute_metadata_score( { "match_type": "vector", @@ -426,6 +430,89 @@ def test_compute_metadata_score_ignores_generated_entities_for_generic_tag_score assert components["tag"] == 1 / 3 +def _tag_score_result(tags: list[str]) -> dict[str, Any]: + return { + "match_type": "vector", + "match_score": 0.7, + "memory": {"content": "Benchmark notes", "tags": tags}, + } + + +def test_compute_metadata_score_tag_score_single_token_full_credit(monkeypatch): + monkeypatch.setattr(scoring, "SEARCH_TAG_SCORE_TOKEN_CAP", 3) + + _score, components = _compute_metadata_score( + _tag_score_result(["automem"]), + "automem", + ["automem"], + ) + + assert components["tag"] == 1.0 + + +def test_compute_metadata_score_tag_score_caps_denominator_for_long_queries(monkeypatch): + monkeypatch.setattr(scoring, "SEARCH_TAG_SCORE_TOKEN_CAP", 3) + + _score, components = _compute_metadata_score( + _tag_score_result(["alpha", "bravo"]), + "alpha bravo charlie delta echo", + ["alpha", "bravo", "charlie", "delta", "echo"], + ) + + # 2 hits over min(5, cap=3) instead of the legacy 2/5 + assert components["tag"] == pytest.approx(2 / 3, abs=1e-9) + + +def test_compute_metadata_score_tag_score_clips_at_one_when_hits_exceed_cap(monkeypatch): + monkeypatch.setattr(scoring, "SEARCH_TAG_SCORE_TOKEN_CAP", 3) + + _score, components = _compute_metadata_score( + _tag_score_result(["alpha", "bravo", "charlie", "delta"]), + "alpha bravo charlie delta echo", + ["alpha", "bravo", "charlie", "delta", "echo"], + ) + + # 4 hits over a capped denominator of 3 would exceed 1.0; clip it. + assert components["tag"] == 1.0 + + +def test_compute_metadata_score_tag_score_cap_zero_restores_legacy_denominator(monkeypatch): + monkeypatch.setattr(scoring, "SEARCH_TAG_SCORE_TOKEN_CAP", 0) + + _score, components = _compute_metadata_score( + _tag_score_result(["alpha", "bravo"]), + "alpha bravo charlie delta echo", + ["alpha", "bravo", "charlie", "delta", "echo"], + ) + + # Legacy behavior: denominator is the full query length (2/5) + assert components["tag"] == pytest.approx(0.4, abs=1e-9) + + +def test_compute_metadata_score_tag_score_short_query_below_cap_uses_query_length(monkeypatch): + monkeypatch.setattr(scoring, "SEARCH_TAG_SCORE_TOKEN_CAP", 3) + + _score, components = _compute_metadata_score( + _tag_score_result(["alpha"]), + "alpha zulu", + ["alpha", "zulu"], + ) + + # Below the cap, the denominator stays min(len(tokens), cap) == 2 + assert components["tag"] == pytest.approx(0.5, abs=1e-9) + + +def test_tag_score_token_cap_config_falls_back_on_negative_values(): + # 0 is a valid sentinel (legacy full-length denominator), so only negative + # values fall back to the default; unparseable values raise like the + # neighboring int()/float() env parses. + assert config._non_negative_int_or_default("-1", 3) == 3 + assert config._non_negative_int_or_default("0", 3) == 0 + assert config._non_negative_int_or_default("5", 3) == 5 + with pytest.raises(ValueError): + config._non_negative_int_or_default("not-an-int", 3) + + def _timestamp_days_ago(days: float) -> str: return (datetime.now(timezone.utc) - timedelta(days=days)).isoformat()