diff --git a/benchmark/codex_recall/README.md b/benchmark/codex_recall/README.md new file mode 100644 index 0000000000..fa4f964f98 --- /dev/null +++ b/benchmark/codex_recall/README.md @@ -0,0 +1,51 @@ +# Codex auto-recall regression benchmark + +This benchmark measures the user-visible Codex `UserPromptSubmit` hook rather +than retrieval in isolation. It records end-to-end hook latency, irrelevant +memory injection, known-positive recall, and injected-context tokens. + +Input is JSONL with `id`, `query`, and `expected` (`accept` or `abstain`). A +positive case may provide `gold_uri` or `gold_uris`; the run counts a hit only +when the injected context cites at least one expected URI. Keep +deployment-specific positives in an untracked file. The checked-in fixture is +synthetic and contains no user memory or profile data. + +Run a controlled low-latency profile against a local server: + +```bash +python benchmark/codex_recall/run.py \ + --input /path/to/private-cases.jsonl \ + --output /tmp/codex-recall-main.json \ + --label main \ + --repeat 5 \ + --max-p50-ms 700 \ + --max-p95-ms 1500 \ + --max-false-injection-rate 0.02 \ + --min-positive-recall-rate 0.90 \ + --max-injection-p95-tokens 900 +``` + +The default profile disables compression and query expansion, caps assembly at +800 tokens, disables cross-turn deduplication, and uses a unique session per +sample. Override the corresponding CLI options to measure another profile. +Credentials are passed to the child hook but never included in output. +Candidate-only tuning can be passed with repeatable +`--variant-env OPENVIKING_NAME=VALUE`. Only recall-related tuning names are +accepted, and their non-secret values are recorded for reproducibility. + +Run the same fixture and profile from a candidate checkout, then reject a +reverse optimization with: + +```bash +python benchmark/codex_recall/compare.py \ + --baseline /tmp/codex-recall-main.json \ + --candidate /tmp/codex-recall-candidate.json +``` + +Both runs must use the same fixture, repetitions, timeout, and tokenizer. Their +strategy settings may differ intentionally. Quality and injected-token +regressions are strict. Latency allows 5% plus 25ms by default to absorb local +scheduling jitter; both tolerances are configurable. +The reports contain case IDs, decisions, aggregate metrics, and a fixture hash, +but omit query text, memory content, gold URIs, server URLs, credentials, and +hook paths. diff --git a/benchmark/codex_recall/__init__.py b/benchmark/codex_recall/__init__.py new file mode 100644 index 0000000000..d9a4b7e5f5 --- /dev/null +++ b/benchmark/codex_recall/__init__.py @@ -0,0 +1 @@ +"""Codex auto-recall regression benchmark.""" diff --git a/benchmark/codex_recall/compare.py b/benchmark/codex_recall/compare.py new file mode 100644 index 0000000000..e713a7d392 --- /dev/null +++ b/benchmark/codex_recall/compare.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. +# SPDX-License-Identifier: AGPL-3.0 +"""Compare two privacy-safe Codex auto-recall benchmark reports.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +LOWER_IS_BETTER = ( + "false_injection_rate", + "latency_ms_p50", + "latency_ms_p95", + "injection_tokens_p95", +) +HIGHER_IS_BETTER = ("positive_recall_rate",) + + +def load_report(path: Path) -> dict[str, Any]: + try: + report = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ValueError(f"invalid report: {path}") from exc + if report.get("schema_version") != 1 or not isinstance(report.get("summary"), dict): + raise ValueError(f"unsupported report schema: {path}") + return report + + +def compare_reports( + baseline: dict[str, Any], + candidate: dict[str, Any], + *, + latency_ratio: float, + latency_jitter_ms: float, +) -> dict[str, Any]: + if baseline.get("fixture_sha256") != candidate.get("fixture_sha256"): + raise ValueError("baseline and candidate used different fixtures") + if baseline.get("protocol") != candidate.get("protocol"): + raise ValueError("baseline and candidate used different benchmark protocols") + if baseline["summary"].get("tokenizer") != candidate["summary"].get("tokenizer"): + raise ValueError("baseline and candidate used different tokenizers") + + base = baseline["summary"] + new = candidate["summary"] + failures: list[str] = [] + deltas: dict[str, float] = {} + for metric in LOWER_IS_BETTER + HIGHER_IS_BETTER: + deltas[metric] = round(float(new[metric]) - float(base[metric]), 4) + + if new["false_injection_rate"] > base["false_injection_rate"]: + failures.append("false_injection_rate") + if new["positive_recall_rate"] < base["positive_recall_rate"]: + failures.append("positive_recall_rate") + if new["injection_tokens_p95"] > base["injection_tokens_p95"]: + failures.append("injection_tokens_p95") + for metric in ("latency_ms_p50", "latency_ms_p95"): + allowed = float(base[metric]) * (1 + latency_ratio) + latency_jitter_ms + if float(new[metric]) > allowed: + failures.append(metric) + + return { + "baseline_label": baseline.get("label", "baseline"), + "candidate_label": candidate.get("label", "candidate"), + "deltas": deltas, + "non_regression_passed": not failures, + "failures": failures, + } + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--baseline", type=Path, required=True) + parser.add_argument("--candidate", type=Path, required=True) + parser.add_argument("--max-latency-regression-ratio", type=float, default=0.05) + parser.add_argument("--latency-jitter-ms", type=float, default=25.0) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + if args.max_latency_regression_ratio < 0 or args.latency_jitter_ms < 0: + raise ValueError("latency tolerances must be non-negative") + result = compare_reports( + load_report(args.baseline), + load_report(args.candidate), + latency_ratio=args.max_latency_regression_ratio, + latency_jitter_ms=args.latency_jitter_ms, + ) + print(json.dumps(result, ensure_ascii=False, sort_keys=True)) + return 0 if result["non_regression_passed"] else 1 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except ValueError as exc: + print(f"Codex recall comparison failed: {exc}", file=sys.stderr) + raise SystemExit(2) from exc diff --git a/benchmark/codex_recall/fixtures/public_negative_queries.jsonl b/benchmark/codex_recall/fixtures/public_negative_queries.jsonl new file mode 100644 index 0000000000..f39fdd5cb0 --- /dev/null +++ b/benchmark/codex_recall/fixtures/public_negative_queries.jsonl @@ -0,0 +1,4 @@ +{"id":"negative-random-token","query":"zxqv-1847 unrelated synthetic lookup","expected":"abstain"} +{"id":"negative-out-of-domain","query":"Explain the taxonomy of deep-sea bioluminescent organisms","expected":"abstain"} +{"id":"negative-control-continue","query":"continue","expected":"abstain"} +{"id":"negative-control-confirm","query":"确认","expected":"abstain"} diff --git a/benchmark/codex_recall/run.py b/benchmark/codex_recall/run.py new file mode 100644 index 0000000000..c4e0f58e2b --- /dev/null +++ b/benchmark/codex_recall/run.py @@ -0,0 +1,379 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. +# SPDX-License-Identifier: AGPL-3.0 +"""Measure Codex auto-recall latency, precision, recall, and injection size.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +import subprocess +import sys +import tempfile +import time +from pathlib import Path +from typing import Any + +SCHEMA_VERSION = 1 +DEFAULT_HOOK = ( + Path(__file__).resolve().parents[2] + / "examples" + / "codex-memory-plugin" + / "scripts" + / "auto-recall.mjs" +) +_TOKEN_ENCODER: Any = None +_TOKENIZER_NAME = "conservative-cjk" + + +def load_cases(path: Path) -> list[dict[str, Any]]: + cases: list[dict[str, Any]] = [] + seen_ids: set[str] = set() + for line_number, raw_line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + if not raw_line.strip(): + continue + try: + value = json.loads(raw_line) + except json.JSONDecodeError as exc: + raise ValueError(f"line {line_number}: invalid JSON") from exc + case_id = str(value.get("id", "")).strip() + query = str(value.get("query", "")).strip() + expected = str(value.get("expected", "")).strip().lower() + if not case_id or not query or expected not in {"accept", "abstain"}: + raise ValueError( + f"line {line_number}: id, query, and expected=accept|abstain are required" + ) + if case_id in seen_ids: + raise ValueError(f"line {line_number}: duplicate id: {case_id}") + seen_ids.add(case_id) + raw_gold = value.get("gold_uris", value.get("gold_uri", [])) + if isinstance(raw_gold, str): + raw_gold = [raw_gold] + if not isinstance(raw_gold, list) or not all(isinstance(uri, str) for uri in raw_gold): + raise ValueError(f"line {line_number}: gold_uri(s) must be strings") + gold_uris = [uri.strip() for uri in raw_gold if uri.strip()] + cases.append( + { + "id": case_id, + "query": query, + "expected": expected, + "gold_uris": gold_uris, + } + ) + if not cases: + raise ValueError("input contains no cases") + return cases + + +def percentile(values: list[float], fraction: float) -> float: + if not values: + return 0.0 + ordered = sorted(values) + index = min(len(ordered) - 1, math.ceil(len(ordered) * fraction) - 1) + return round(ordered[max(0, index)], 2) + + +def count_tokens(text: str) -> tuple[int, str]: + global _TOKEN_ENCODER, _TOKENIZER_NAME + if _TOKEN_ENCODER is None: + try: + import tiktoken + + _TOKEN_ENCODER = tiktoken.get_encoding("o200k_base") + _TOKENIZER_NAME = "o200k_base" + except (ImportError, ValueError): + _TOKEN_ENCODER = False + if _TOKEN_ENCODER: + return len(_TOKEN_ENCODER.encode(text)), _TOKENIZER_NAME + + tokens = 0 + latin_run = 0 + for char in text: + if ord(char) > 255: + if latin_run: + tokens += math.ceil(latin_run / 4) + latin_run = 0 + tokens += 1 + else: + latin_run += 1 + if latin_run: + tokens += math.ceil(latin_run / 4) + return tokens, _TOKENIZER_NAME + + +def extract_context(stdout: str) -> str: + try: + payload = json.loads(stdout.strip()) + except json.JSONDecodeError as exc: + raise ValueError("hook returned invalid JSON") from exc + context = payload.get("hookSpecificOutput", {}).get("additionalContext", "") + return context if isinstance(context, str) else "" + + +def parse_variant_env(value: str) -> tuple[str, str]: + if "=" not in value: + raise argparse.ArgumentTypeError("expected OPENVIKING_NAME=VALUE") + name, raw_value = value.split("=", 1) + name = name.strip() + allowed = ( + name.startswith("OPENVIKING_RECALL_"), + name.startswith("OPENVIKING_FAST_RECALL_"), + name in {"OPENVIKING_SCORE_THRESHOLD", "OPENVIKING_MIN_QUERY_LENGTH"}, + ) + if not any(allowed) or not name.replace("_", "").isalnum(): + raise argparse.ArgumentTypeError("variant env must be a recall tuning setting") + return name, raw_value + + +def build_environment(args: argparse.Namespace, config_dir: Path) -> dict[str, str]: + env = dict(os.environ) + env.update( + { + "OPENVIKING_AUTO_RECALL": "1", + "OPENVIKING_CREDENTIAL_SOURCE": "env", + "OPENVIKING_URL": args.url, + "OPENVIKING_CONFIG_FILE": str(config_dir / "missing-ov.conf"), + "OPENVIKING_CLI_CONFIG_FILE": str(config_dir / "missing-ovcli.conf"), + "OPENVIKING_RECALL_COMPRESS": "1" if args.compression == "on" else "0", + "OPENVIKING_RECALL_QUERY_EXPANSION": args.query_expansion, + "OPENVIKING_RECALL_MAX_TOKENS": str(args.max_tokens), + "OPENVIKING_RECALL_DEDUP_TURNS": "0", + "OPENVIKING_RECALL_TIMEOUT_MS": str(round(args.timeout * 1000)), + "OPENVIKING_TIMEOUT_MS": str(round(args.timeout * 1000)), + "OPENVIKING_WORKSPACE_PEER": "0", + "OPENVIKING_DEBUG": "0", + } + ) + optional = { + "OPENVIKING_API_KEY": args.api_key, + "OPENVIKING_ACCOUNT": args.account, + "OPENVIKING_USER": args.user, + "OPENVIKING_PEER_ID": args.actor_peer, + } + for name, value in optional.items(): + if value: + env[name] = value + else: + env.pop(name, None) + env.update(dict(args.variant_env)) + return env + + +def run_hook( + *, + node: str, + hook: Path, + query: str, + session_id: str, + env: dict[str, str], + timeout: float, +) -> tuple[str, float]: + payload = json.dumps({"prompt": query, "session_id": session_id}, ensure_ascii=False) + started = time.perf_counter() + try: + completed = subprocess.run( + [node, str(hook)], + input=payload, + text=True, + capture_output=True, + env=env, + timeout=timeout + 2.0, + check=False, + ) + except subprocess.TimeoutExpired as exc: + raise ValueError("hook process timed out") from exc + elapsed_ms = round((time.perf_counter() - started) * 1000, 2) + if completed.returncode != 0: + raise ValueError(f"hook process exited with status {completed.returncode}") + return extract_context(completed.stdout), elapsed_ms + + +def summarize(records: list[dict[str, Any]], tokenizer: str) -> dict[str, Any]: + negative = [record for record in records if record["expected"] == "abstain"] + positive = [record for record in records if record["expected"] == "accept"] + latencies = [float(record["latency_ms"]) for record in records] + injection_tokens = [int(record["injection_tokens"]) for record in records] + return { + "runs": len(records), + "negative_runs": len(negative), + "positive_runs": len(positive), + "false_injection_rate": round( + sum(record["injected"] for record in negative) / len(negative), 4 + ) + if negative + else 0.0, + "positive_recall_rate": round(sum(record["hit"] for record in positive) / len(positive), 4) + if positive + else 0.0, + "latency_ms_p50": percentile(latencies, 0.5), + "latency_ms_p95": percentile(latencies, 0.95), + "injection_tokens_p50": percentile(injection_tokens, 0.5), + "injection_tokens_p95": percentile(injection_tokens, 0.95), + "tokenizer": tokenizer, + } + + +def threshold_failures(summary: dict[str, Any], args: argparse.Namespace) -> list[str]: + checks = ( + ("latency_ms_p50", args.max_p50_ms, lambda actual, limit: actual <= limit), + ("latency_ms_p95", args.max_p95_ms, lambda actual, limit: actual <= limit), + ( + "false_injection_rate", + args.max_false_injection_rate, + lambda actual, limit: actual <= limit, + ), + ( + "positive_recall_rate", + args.min_positive_recall_rate, + lambda actual, limit: actual >= limit, + ), + ( + "injection_tokens_p95", + args.max_injection_p95_tokens, + lambda actual, limit: actual <= limit, + ), + ) + return [ + name + for name, limit, check in checks + if limit is not None and not check(summary[name], limit) + ] + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--input", type=Path, required=True, help="JSONL benchmark cases") + parser.add_argument("--output", type=Path, help="write a privacy-safe JSON report") + parser.add_argument("--label", default="candidate", help="non-sensitive variant label") + parser.add_argument("--hook", type=Path, default=DEFAULT_HOOK) + parser.add_argument("--node", default="node") + parser.add_argument("--url", default="http://127.0.0.1:1933") + parser.add_argument("--api-key", default="", help="optional API key; never printed") + parser.add_argument("--account", default="") + parser.add_argument("--user", default="") + parser.add_argument("--actor-peer", default="") + parser.add_argument("--timeout", type=float, default=10.0) + parser.add_argument("--repeat", type=int, default=1) + parser.add_argument("--warmup", type=int, default=0) + parser.add_argument("--compression", choices=("on", "off"), default="off") + parser.add_argument("--query-expansion", choices=("auto", "off"), default="off") + parser.add_argument("--max-tokens", type=int, default=800) + parser.add_argument( + "--variant-env", + action="append", + default=[], + type=parse_variant_env, + metavar="OPENVIKING_NAME=VALUE", + help="non-secret recall tuning recorded in the report", + ) + parser.add_argument("--max-p50-ms", type=float) + parser.add_argument("--max-p95-ms", type=float) + parser.add_argument("--max-false-injection-rate", type=float) + parser.add_argument("--min-positive-recall-rate", type=float) + parser.add_argument("--max-injection-p95-tokens", type=float) + parser.add_argument("--fail-on-case-mismatch", action="store_true") + return parser.parse_args() + + +def validate_args(args: argparse.Namespace) -> None: + if args.timeout <= 0: + raise ValueError("--timeout must be positive") + if args.repeat <= 0 or args.warmup < 0: + raise ValueError("--repeat must be positive and --warmup must be non-negative") + if args.max_tokens <= 0: + raise ValueError("--max-tokens must be positive") + for name in ("max_false_injection_rate", "min_positive_recall_rate"): + value = getattr(args, name) + if value is not None and not 0 <= value <= 1: + raise ValueError(f"--{name.replace('_', '-')} must be between 0 and 1") + if not args.hook.is_file(): + raise ValueError(f"hook does not exist: {args.hook}") + + +def main() -> int: + args = parse_args() + validate_args(args) + cases = load_cases(args.input) + fixture_sha256 = hashlib.sha256(args.input.read_bytes()).hexdigest() + records: list[dict[str, Any]] = [] + tokenizer = "conservative-cjk" + + with tempfile.TemporaryDirectory(prefix="ov-codex-recall-bench-") as directory: + env = build_environment(args, Path(directory)) + sequence = 0 + for case_index, case in enumerate(cases): + for iteration in range(args.warmup + args.repeat): + session_id = f"codex-benchmark-{fixture_sha256[:12]}-{case_index}-{iteration}" + context, latency_ms = run_hook( + node=args.node, + hook=args.hook, + query=case["query"], + session_id=session_id, + env=env, + timeout=args.timeout, + ) + if iteration < args.warmup: + continue + injection_tokens, tokenizer = count_tokens(context) + injected = bool(context.strip()) + hit = injected + if case["expected"] == "accept" and case["gold_uris"]: + hit = any(uri in context for uri in case["gold_uris"]) + record = { + "sequence": sequence, + "case_id": case["id"], + "expected": case["expected"], + "injected": injected, + "hit": hit, + "passed": (not injected) if case["expected"] == "abstain" else hit, + "latency_ms": latency_ms, + "injection_tokens": injection_tokens, + } + sequence += 1 + records.append(record) + print(json.dumps(record, ensure_ascii=False, sort_keys=True)) + + summary = summarize(records, tokenizer) + failures = threshold_failures(summary, args) + report = { + "schema_version": SCHEMA_VERSION, + "label": args.label, + "fixture_sha256": fixture_sha256, + "protocol": { + "timeout_ms": round(args.timeout * 1000), + "repeat": args.repeat, + "warmup": args.warmup, + }, + "variant": { + "compression": args.compression, + "query_expansion": args.query_expansion, + "max_tokens": args.max_tokens, + "extra_env": dict(sorted(args.variant_env)), + }, + "records": records, + "summary": summary, + "threshold_failures": failures, + } + print(json.dumps({"summary": summary, "threshold_failures": failures}, sort_keys=True)) + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text( + json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + if failures: + return 1 + if args.fail_on_case_mismatch and not all(record["passed"] for record in records): + return 1 + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except (OSError, ValueError) as exc: + print(f"Codex recall benchmark failed: {exc}", file=sys.stderr) + raise SystemExit(2) from exc diff --git a/tests/benchmark/test_codex_recall_benchmark.py b/tests/benchmark/test_codex_recall_benchmark.py new file mode 100644 index 0000000000..f49883edd3 --- /dev/null +++ b/tests/benchmark/test_codex_recall_benchmark.py @@ -0,0 +1,131 @@ +# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. +# SPDX-License-Identifier: AGPL-3.0 + +import argparse +import json +import sys +from types import SimpleNamespace + +import pytest + +from benchmark.codex_recall import compare, run + + +def test_runner_omits_private_queries_and_gold_uris(tmp_path, monkeypatch, capsys): + private_query = "private deployment incident query" + private_uri = "viking://user/private/memories/experiences/incident.md" + fixture = tmp_path / "private.jsonl" + fixture.write_text( + json.dumps( + { + "id": "positive-1", + "query": private_query, + "expected": "accept", + "gold_uri": private_uri, + } + ), + encoding="utf-8", + ) + hook = tmp_path / "auto-recall.mjs" + hook.write_text("// fixture", encoding="utf-8") + report_path = tmp_path / "report.json" + + def fake_subprocess_run(*_args, **_kwargs): + payload = { + "hookSpecificOutput": { + "additionalContext": f"matched memory ({private_uri})", + } + } + return SimpleNamespace(returncode=0, stdout=json.dumps(payload), stderr="") + + monkeypatch.setattr(run.subprocess, "run", fake_subprocess_run) + monkeypatch.setattr( + sys, + "argv", + [ + "run.py", + "--input", + str(fixture), + "--output", + str(report_path), + "--hook", + str(hook), + ], + ) + + assert run.main() == 0 + output = capsys.readouterr().out + report = report_path.read_text(encoding="utf-8") + assert private_query not in output + report + assert private_uri not in output + report + assert json.loads(report)["summary"]["positive_recall_rate"] == 1.0 + + +def test_load_cases_rejects_duplicate_ids(tmp_path): + fixture = tmp_path / "duplicates.jsonl" + fixture.write_text( + "\n".join( + [ + '{"id":"same","query":"one","expected":"accept"}', + '{"id":"same","query":"two","expected":"abstain"}', + ] + ), + encoding="utf-8", + ) + with pytest.raises(ValueError, match="duplicate id"): + run.load_cases(fixture) + + +def _report(label, *, false_rate, recall_rate, p50, p95, tokens): + return { + "schema_version": 1, + "label": label, + "fixture_sha256": "fixture", + "protocol": {"repeat": 5, "timeout_ms": 10000}, + "variant": {"max_tokens": 800}, + "summary": { + "false_injection_rate": false_rate, + "positive_recall_rate": recall_rate, + "latency_ms_p50": p50, + "latency_ms_p95": p95, + "injection_tokens_p95": tokens, + "tokenizer": "o200k_base", + }, + } + + +def test_compare_accepts_improvement_and_rejects_reverse_optimization(): + baseline = _report("main", false_rate=0.1, recall_rate=0.9, p50=600, p95=1000, tokens=800) + improved = _report("candidate", false_rate=0.0, recall_rate=0.95, p50=500, p95=900, tokens=700) + regressed = _report("candidate", false_rate=0.0, recall_rate=0.85, p50=500, p95=900, tokens=700) + + accepted = compare.compare_reports(baseline, improved, latency_ratio=0.05, latency_jitter_ms=25) + rejected = compare.compare_reports( + baseline, regressed, latency_ratio=0.05, latency_jitter_ms=25 + ) + + assert accepted["non_regression_passed"] is True + assert rejected["non_regression_passed"] is False + assert rejected["failures"] == ["positive_recall_rate"] + + +def test_compare_requires_identical_fixture_and_protocol(): + baseline = _report("main", false_rate=0, recall_rate=1, p50=1, p95=2, tokens=3) + different = dict(baseline, fixture_sha256="other") + with pytest.raises(ValueError, match="different fixtures"): + compare.compare_reports(baseline, different, latency_ratio=0.05, latency_jitter_ms=25) + + different_protocol = dict(baseline, protocol={"repeat": 1, "timeout_ms": 10000}) + with pytest.raises(ValueError, match="different benchmark protocols"): + compare.compare_reports( + baseline, different_protocol, latency_ratio=0.05, latency_jitter_ms=25 + ) + + +def test_variant_env_accepts_only_recall_tuning(): + assert run.parse_variant_env("OPENVIKING_RECALL_STRATEGY=fast") == ( + "OPENVIKING_RECALL_STRATEGY", + "fast", + ) + with pytest.raises(argparse.ArgumentTypeError, match="recall tuning"): + run.parse_variant_env("OPENVIKING_API_KEY=private")