From a9aae4f6a8722b0f1052a18b2d60a52d1a7a709d Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 2 Sep 2026 12:09:08 -0700 Subject: [PATCH] update cicd --- CICD_EVAL.md | 53 ++-- EVALUATIONS.md | 91 ++++--- README.md | 30 ++- TRACING.md | 188 ++++++++++---- agentx/integrations/anthropic.py | 2 +- agentx/integrations/crewai.py | 2 +- setup.py | 3 +- tests/test_selfhost_compat.py | 424 +++++++++++++++++++++++++++++++ 8 files changed, 664 insertions(+), 129 deletions(-) create mode 100644 tests/test_selfhost_compat.py diff --git a/CICD_EVAL.md b/CICD_EVAL.md index 61d72c7..fd9779c 100644 --- a/CICD_EVAL.md +++ b/CICD_EVAL.md @@ -6,7 +6,12 @@ The AgentX CI/CD evaluation SDK lets you **gate agent releases** in any CI pipel If the gate fails, the SDK raises `CIGateFailure` (or returns the result so you can `sys.exit(1)`), blocking the pipeline. -> **Status:** This feature is in the design phase. The API and SDK interfaces described here reflect the planned implementation. +> **Hosted platform only.** `tracer.run_eval()` and the low-level methods below call the hosted +> API's `/ingest/ci-runs` routes. The self-host engine does not serve them - against a self-host +> base URL they return 404, which the SDK surfaces as `DatasetNotFound`. On self-host, gate CI +> with the run + gate flow instead: `client.evaluations.run(...).execute(...).finalize().gate(...)` +> (or `pytest` + `assert_evaluation`, below) - see +> [CI gate (self-host)](EVALUATIONS.md#ci-gate-self-host). --- @@ -72,7 +77,7 @@ result = tracer.run_eval( | `dataset_id` | `str` | - | ID of the evaluation dataset (must have `ci.enabled: true`) | | `agent_fn` | `Callable[[str], str]` | - | Function that takes a query string and returns the agent's response | | `agent_name` | `str` | `None` | Label for this agent (used to create/reuse a reference agent in AgentX) | -| `pass_rate_threshold` | `float` | `None` | Override the dataset's `passRateThreshold` for this run (0.0–1.0) | +| `pass_rate_threshold` | `float` | `None` | Override the dataset's `passRateThreshold` for this run (0.0-1.0) | | `git_context` | `dict` | `None` | Git metadata (see [Git context](#git-context)) | | `concurrency` | `int` | `1` | Number of questions to run in parallel (use with caution for rate-limited agents) | | `fail_on_gate` | `bool` | `False` | Raise `CIGateFailure` if gate is `"fail"` instead of returning the result | @@ -85,23 +90,24 @@ result = tracer.run_eval( class CIRunResult: run_id: str gate: Literal["pass", "fail"] - pass_rate: float # 0.0–1.0 + pass_rate: float # 0.0-1.0 total_questions: int passed_questions: int scores: list[CIQuestionScore] violations: list[ThresholdViolation] git_context: dict | None - finalized_at: str # ISO 8601 + finalized_at: str | None # ISO 8601 ``` ```python @dataclass class CIQuestionScore: question_index: int - rating: int # 1–5 + rating: int # 0-10 justification: str passed: bool - input: Any | None + gate_fired: bool # True if failFast triggered early finalization + input: Any | None # populated on finalize; None on submit_result() output: Any | None ``` @@ -137,7 +143,7 @@ print(f"{'='*50}\n") for score in result.scores: icon = "✓" if score.passed else "✗" - print(f" [{icon}] Q{score.question_index} rating={score.rating}/5") + print(f" [{icon}] Q{score.question_index} rating={score.rating}/10") print(f" {score.justification[:100]}") if result.violations: @@ -176,7 +182,7 @@ class CIRun: run_id: str dataset_id: str total_questions: int - test_cases: list[CITestCase] # empty if ci.exposeTestInputs is false + test_cases: list[CITestCase] expires_at: str # ISO 8601 ``` @@ -213,22 +219,13 @@ score = tracer.submit_result( question_index: int, output: Any, *, - input: Any | None = None, # Actual input sent to agent (defaults to test case query) + input: Any | None = None, # the actual input sent to the agent, recorded with the result latency_ms: int | None = None, ) ``` -**Returns: `CIQuestionScore`** - -```python -@dataclass -class CIQuestionScore: - question_index: int - rating: int - justification: str - passed: bool - gate_fired: bool # True if failFast triggered early finalization -``` +**Returns: `CIQuestionScore`** (see [above](#return-type-cirunresult); `gate_fired=True` means +failFast finalized the run early). **Example:** @@ -241,7 +238,7 @@ for case in run.test_cases: output=output, input=case.query, ) - print(f"Q{case.index}: {score.rating}/5 - {score.justification[:60]}") + print(f"Q{case.index}: {score.rating}/10 - {score.justification[:60]}") if score.gate_fired: print("Gate fired (failFast) - stopping early") break @@ -449,7 +446,8 @@ sys.exit(0 if result.gate == "pass" else 1) `agentx.testing` turns a run into a plain pytest failure, so quality checks live in the same suite as everything else. Both helpers raise `AssertionError` subclasses - no plugin, no -registration, works in any runner. +registration, works in any runner. Unlike the CI-runs API above, these ride the run + gate flow, +so they **work on self-host** as well as hosted. ### `assert_evaluation` - does it clear the bar @@ -641,7 +639,7 @@ print(f"\nAgentX Eval Gate: {result.gate.upper()}") print(f"Pass rate: {result.pass_rate:.0%} ({result.passed_questions}/{result.total_questions})\n") for score in result.scores: icon = "✓" if score.passed else "✗" - print(f" [{icon}] Q{score.question_index}: {score.rating}/5 {score.justification[:80]}") + print(f" [{icon}] Q{score.question_index}: {score.rating}/10 {score.justification[:80]}") sys.exit(0 if result.gate == "pass" else 1) ``` @@ -667,7 +665,7 @@ test_case = run.test_cases[2] # pick question index 2 output = my_agent(test_case.query) score = tracer.submit_result(run.run_id, test_case.index, output) -print(f"Q{test_case.index}: {score.rating}/5 - {score.justification}") +print(f"Q{test_case.index}: {score.rating}/10 - {score.justification}") # No need to finalize - just abandon the run (it expires automatically) ``` @@ -679,8 +677,9 @@ print(f"Q{test_case.index}: {score.rating}/5 - {score.justification}") ```python client = AgentX( api_key="ax_live_xxxxxxxxxxxxxxxx", # or set AGENTX_API_KEY - workspace_id="...", # optional workspace override - base_url="https://api.agentx.so", # optional for self-hosted - timeout=30, # HTTP timeout per request (seconds) + workspace_id="...", # optional - or set AGENTX_WORKSPACE_ID + base_url="...", # optional - or set AGENTX_API_BASE_URL; + # defaults to the hosted API, which is where + # the CI-runs routes on this page live ) ``` diff --git a/EVALUATIONS.md b/EVALUATIONS.md index 3afc8c6..c59bf4e 100644 --- a/EVALUATIONS.md +++ b/EVALUATIONS.md @@ -73,15 +73,16 @@ print(f"Average rating: {report.average_rating:.2f}") # Optional similarity metrics - present only when enabled on the dataset. if report.cosine_similarity is not None: - print(f"Cosine similarity: {report.cosine_similarity:.3f}") # 0–1 + print(f"Cosine similarity: {report.cosine_similarity:.3f}") # 0-1 if report.jaccard_similarity is not None: - print(f"Jaccard similarity:{report.jaccard_similarity:.3f}") # 0–1 + print(f"Jaccard similarity:{report.jaccard_similarity:.3f}") # 0-1 if report.bleu_score is not None: - print(f"BLEU score: {report.bleu_score:.3f}") # 0–1 + print(f"BLEU score: {report.bleu_score:.3f}") # 0-1 if report.rouge_score is not None: - print(f"ROUGE-L score: {report.rouge_score:.3f}") # 0–1 + print(f"ROUGE-L score: {report.rouge_score:.3f}") # 0-1 -print(f"Dashboard: {report.dashboard_url}") +if report.dashboard_url: # hosted runs; self-host doesn't send one + print(f"Dashboard: {report.dashboard_url}") ``` ### Environment variables @@ -89,13 +90,15 @@ print(f"Dashboard: {report.dashboard_url}") | Variable | Description | |---|---| | `AGENTX_API_KEY` | Required. Your AgentX API key. | -| `AGENTX_API_BASE_URL` | Optional. Override the API base URL (useful for local dev). | +| `AGENTX_API_BASE_URL` | Optional. Override the API base URL, e.g. `http://localhost:4700/api/v1` for self-host. | +| `AGENTX_WORKSPACE_ID` | Optional. Explicit workspace instead of the API key's default. | +| `AGENTX_EVAL_QUIET` | Optional. `1` silences the interactive progress UI (spinners, per-case lines) for CI logs; results, gate verdicts, and errors still print. | -You can also pass `base_url` directly to the constructor: +You can also pass `base_url` directly to the constructor (the `/custom-agent-evaluations` suffix is appended automatically, so `/api/v1` is enough): ```python -# Point at your local dev server -client = AgentX(api_key="your-key", base_url="http://localhost:3000/api/v1/custom-agent-evaluations") +# Point at a local self-host engine +client = AgentX(api_key="your-key", base_url="http://localhost:4700/api/v1") ``` --- @@ -129,6 +132,8 @@ print(dataset.id) # use this id in .run() #### Import from CSV +`from_csv()` returns a builder (add more cases if you like), so finish with `.publish()`: + ```python dataset = client.evaluations.datasets.from_csv( path="cases.csv", @@ -136,7 +141,7 @@ dataset = client.evaluations.datasets.from_csv( number_of_requests=2, acceptance_criteria="...", rejection_criteria="...", -) +).publish() ``` CSV format: @@ -146,7 +151,7 @@ query,expected_results "What is your refund policy?","Describe refund terms." ``` -Optional `case_id` column for stable idempotency keys across re-runs. +`query` is the only required column. Optional columns: `expected_results`, plus semicolon-separated `expected_capabilities`, `expected_knowledge_base`, and `expected_delegations`. Rows with an empty query are skipped with a logged warning. `from_dataframe(df, name, ...)` does the same from a pandas DataFrame. --- @@ -168,7 +173,7 @@ dataset = ( ) ``` -`smoke_test_guidance` is optional free text steering *what kind* of variants get generated (tone, adversarial phrasing, different languages, ...); leave it out for natural rewording only. Both the paraphrase text and the count are decided entirely server-side, reusing the same generation the AgentX dashboard's native runs use, `.execute()` just asks the extra variants and submits them for you, nothing to configure on the SDK side beyond these two kwargs. Ignored on `follow_up_questions`, only a case's opening question can be smoke-tested. +`smoke_test_count` accepts 1-10 extra variants per case. `smoke_test_guidance` is optional free text steering *what kind* of variants get generated (tone, adversarial phrasing, different languages, ...); leave it out for natural rewording only. Both the paraphrase text and the count are decided entirely server-side, reusing the same generation the AgentX dashboard's native runs use, `.execute()` just asks the extra variants and submits them for you, nothing to configure on the SDK side beyond these two kwargs. Ignored on `follow_up_questions`, only a case's opening question can be smoke-tested. Variants show up as extra entries in `EvaluationCase`/`EvaluationResult`: @@ -428,6 +433,7 @@ Score strictly: any missing policy detail is a failing response.""", - `judge_prompt` is a raw template. `{input}`, `{output}`, and `{expected}` are substituted in; everything else (chain of thought, capabilities/references, criteria, per-question `judge_guideline`, delegation notes) is appended automatically after it, so a custom prompt can restructure the grading philosophy without ever losing that context. Omit it to keep the default rubric. - `judge_model` accepts any OpenAI or Anthropic model id (`client.evaluations.list_models(provider="Anthropic")` to discover valid ones). Omit it to keep the default (`gpt-5.5`). +- `list_models()` is **hosted platform only**: it calls the hosted API's `/custom-agent-evaluations/models` registry, which the self-host engine does not serve (404, surfaced as `AgentXEvaluationsError`). On self-host, pass any model id your engine's judge keys can reach. --- @@ -471,13 +477,15 @@ with client.tracer.trace("support-agent", metadata={"promptName": prompt.name}) ... # your agent's own call ``` -From the self-host dashboard: Governance → Improve → **Prompt Management** → a prompt's row menu → **Suggest +From the self-host dashboard: Governance > Manage > **Prompts** > a prompt's row menu > **Suggest improvement**. It merges both kinds of evidence - deliberate eval runs (defaulting to the *current published version only*, auto-widening to every version if there isn't enough recent evidence yet) and worst-scoring Online Evaluator ratings from a recent time window - feeds the worst-rated examples to a judge, and shows a full rewrite plus reasoning. **Nothing is saved until a human -clicks Publish as new version** - there is no `publish()` on this client; a rewrite only ever -reaches your agent through that one explicit, dashboard-only write. Your agent's next +approves it as a new version.** The same propose loop is scriptable: `prompts.examples(prompt.id)` +returns the evidence, `prompts.propose(prompt.id)` asks the judge for a rewrite (returns +`revisedText`/`reasoning` without saving anything), and `prompts.publish_version(prompt.id, +text=...)` is the explicit human-approval write. Your agent's next `client.evaluations.prompts.get(name)` call picks up the new version immediately. Tagging `metadata.version` as `@v` (shown above) means the dataset's **Compare versions** dialog also tells you whether the published rewrite actually scored better, no separate comparison @@ -737,12 +745,14 @@ report = ( Expected endpoint contract: ``` -POST /your-endpoint +POST /your-endpoint (method="..." on the adapter overrides the verb) Content-Type: application/json -Body: { "query": "..." } -Response: { "output": "..." } +Body: { "query": "...", "case_id": "...", "question_index": 0, "run_number": 1 } +Response: { "output": "..." } (or "text"; optional "metadata" and "trace") ``` +A non-2xx response or a timeout is recorded as that case's error instead of aborting the run. + Full example: [`examples/evaluations/http_endpoint_eval.py`](examples/evaluations/http_endpoint_eval.py) --- @@ -755,11 +765,19 @@ Submit outputs you already have without running any agent during evaluation: from agentx.evaluations.adapters.precomputed import PrecomputedAdapter outputs = { - 0: "To reset your password, go to Login → Forgot Password.", + 0: "To reset your password, go to Login > Forgot Password.", 1: "We accept Visa, Mastercard, PayPal, and bank transfers.", } adapter = PrecomputedAdapter(outputs) +``` + +Keys are matched against the case's `case_id` first, then its question index; a plain list works +too (one entry per question, in order). Values can be strings or `{"output": ..., "metadata": +...}` dicts: + +```python +adapter = PrecomputedAdapter(["First answer.", "Second answer."]) report = ( client.evaluations @@ -778,12 +796,15 @@ Full example: [`examples/evaluations/csv_import_eval.py`](examples/evaluations/c | Field | Values | Description | |---|---|---| -| `kind` | `custom_agent` | Always `custom_agent` for external agents | +| `kind` | `custom_agent` (default), `agentx_agent`, `agentx_team` | `custom_agent` for external agents | | `displayName` | any string | Human-readable name shown in the dashboard | | `framework` | `raw_python`, `openai`, `anthropic`, `google`, `langchain`, `llamaindex`, `crewai`, `autogen`, `n8n`, `flowise`, `other` | Framework used | -| `runtime` | `local`, `ci`, `customer_hosted`, `low_code` | Where the agent runs | -| `version` | any string | Optional version tag for the agent | -| `endpoint` | URL | Optional, for HTTP-based agents | +| `frameworkVersion` | any string | Optional framework version tag | +| `runtime` | `local` (default), `ci`, `customer_hosted`, `low_code` | Where the agent runs | +| `agentInstructions` | any string | The agent's own system instructions - what the report's instruction-adherence section grades against | +| `metadata` | `dict[str, str \| int \| bool]` | Free-form tags, e.g. `{"promptName": ..., "version": ...}` (see [Prompt registry](#prompt-registry)) | + +Unknown fields are silently dropped, so a typo here fails quietly - stick to the fields above. ### Similarity metrics (optional) @@ -792,7 +813,7 @@ Each scored result can be enriched with reference-based similarity scores compar | Metric | What it measures | Cost | |---|---|---| | **Cosine** (vector similarity) | Cosine of OpenAI embeddings of `expected_results` vs the actual response. Captures semantic similarity. | One embedding API call per case. | -| **Jaccard** | Token-set overlap `|A ∩ B| / |A ∪ B|` over lowercased word tokens. Pure lexical match. | Free - no API calls. | +| **Jaccard** | Token-set overlap `\|A ∩ B\| / \|A ∪ B\|` over lowercased word tokens. Pure lexical match. | Free - no API calls. | | **BLEU** | Sentence-level BLEU-4 (n-gram precision, up to 4-grams, with brevity penalty). Standard machine-translation-style metric - rewards responses that reuse the expected result's exact phrasing. | Free - no API calls. | | **ROUGE-L** | F1 over the longest common (in-order) subsequence of tokens. Standard summarization-style metric - more tolerant of reordering/insertions than BLEU. | Free - no API calls. | @@ -828,10 +849,10 @@ report = client.evaluations.run(...).execute(my_agent).finalize().analyze() # Top-level convenience accessors - return None when the metric was not # enabled on the dataset, or no case has a value yet. report.average_rating # float | None - same as report.statistics.average_rating -report.cosine_similarity # float | None - averaged across cases (0–1) -report.jaccard_similarity # float | None - averaged across cases (0–1) -report.bleu_score # float | None - averaged across cases (0–1) -report.rouge_score # float | None - averaged across cases (0–1), ROUGE-L F1 +report.cosine_similarity # float | None - averaged across cases (0-1) +report.jaccard_similarity # float | None - averaged across cases (0-1) +report.bleu_score # float | None - averaged across cases (0-1) +report.rouge_score # float | None - averaged across cases (0-1), ROUGE-L F1 # Same values are also available nested under the statistics block: report.statistics.cosine_similarity @@ -869,7 +890,9 @@ sys.exit(gate.exit_code) # 0 = merge, 1 = block | `tolerance` | `float` | `0.5` | Slack for `no_regression` - judge scores are noisy, and an exact comparison would flake builds on variance rather than regressions | | `caller` | `str` | `"sdk"` | Free label shown in the dashboard's CI Gates history ("github-actions", ...) | -At least one of `fail_under` / `no_regression` is required. `GateResult` exposes `.passed`, `.exit_code` (0/1), `.average_rating`, `.baseline_average`, `.baseline_run_id`, and `.checks` (the per-check verdict list). Every `gate()` call is recorded into the dashboard's CI Gates tab by default; use the lower-level `client.evaluations.gate_run(run_id, ..., record=False)` for an unrecorded check. See [self-host's CI docs](https://docs.agentx.so/integrations/self-host-ci) for the GitHub Actions recipe. +At least one of `fail_under` / `no_regression` is required. `GateResult` exposes `.passed`, `.exit_code` (0/1), `.average_rating`, `.baseline_average`, `.baseline_run_id`, and `.checks` (the per-check verdict list). Every `gate()` call is recorded into the dashboard's CI Gates tab by default; use the lower-level `client.evaluations.gate_run(run_id, ..., record=False)` for an unrecorded check, or to gate a run created elsewhere by id. Set `AGENTX_EVAL_QUIET=1` in CI to silence the interactive progress UI while keeping results and gate verdicts. See [self-host's CI docs](https://docs.agentx.so/integrations/self-host-ci) for the GitHub Actions recipe. + +This run + gate flow is the **self-host CI path**. The separate CI-runs API in [CICD_EVAL.md](CICD_EVAL.md) (`tracer.run_eval()` and friends) targets the hosted platform only. ### AI analysis report @@ -877,8 +900,8 @@ At least one of `fail_under` / `no_regression` is required. `GateResult` exposes ```python report = client.evaluations.run(...).execute(my_agent).finalize().analyze( - mode="auto", # "auto" (default) | "sync" | "batch" - quality_mode="quality_first", # "quality_first" (default) | "balanced" + mode="auto", # "auto" | "sync" | "batch" + quality_mode="quality_first", # "quality_first" | "balanced" judges=["gpt-5.5", "claude-opus-4-8"], # 1-3 model ids; omit for a single gpt-5.5 judge ) @@ -914,6 +937,8 @@ This is separate from, and available even without, the numeric `average_rating`/ | `poll_interval` | seconds, default `5.0` | How often to check job status while waiting | | `timeout` | seconds, default `1800.0` | Give up waiting after this long (the job keeps running server-side; call `get_report()` later to check on it) | +`mode` and `quality_mode` default server-side (omitting them keeps the server's behavior). On self-host, the analysis runs synchronously on the engine: `judges` is honored, `mode`/`quality_mode` are accepted but ignored, and the polling loop sees a terminal status on its first check. + ```python # Check on a long-running analysis without calling .analyze() again, even from a # separate script execution: @@ -928,8 +953,8 @@ Your callable can return any of: | Return type | Behavior | |---|---| | `str` | Used directly as the output text | -| `dict` with `"output"` key | Output text from `output`, rest stored as metadata | -| `EvaluationResult` | Full control - pass rating, justification, trace, timings | +| `dict` | Recognized keys: `output` (or `text`/`response`), `metadata` (a dict), `trace_id`, `input_tokens`/`output_tokens` (also read from metadata), `retrieval_context`, `trace`, `error`. Unrecognized keys are dropped - put extra data under `metadata` | +| `EvaluationResult` | Full control - pass error, trace, timings, metadata, retrieval context | ### What gets uploaded diff --git a/README.md b/README.md index 3abe811..64ae21f 100644 --- a/README.md +++ b/README.md @@ -45,12 +45,14 @@ pip install --upgrade agentx-python Requires Python 3.9 or newer. -#### Run self host eval framework locally +#### Run the self-host governance suite locally -``` -agentx-trace-eval --dev --update +```bash +agentx-trace-eval --dev ``` +(See [Self-host](#self-host) below for what this downloads and how to point the SDK at it.) + --- ## Quick start @@ -73,7 +75,7 @@ report = ( .analyze() ) -print(report.average_rating) # LLM-graded score, 0–10 +print(report.average_rating) # LLM-graded score, 0-10 print(report.summary) # AI-generated narrative from .analyze() ``` @@ -94,9 +96,9 @@ report = ( .analyze() ) -print(report.average_rating) # LLM-graded score, 0–10 -print(report.cosine_similarity) # embedding cosine, 0–1 (None if not enabled) -print(report.jaccard_similarity) # token-set overlap, 0–1 (None if not enabled) +print(report.average_rating) # LLM-graded score, 0-10 +print(report.cosine_similarity) # embedding cosine, 0-1 (None if not enabled) +print(report.jaccard_similarity) # token-set overlap, 0-1 (None if not enabled) print(report.summary) # AI-generated narrative from .analyze() print(report.recommendations) # list of prioritized, actionable fixes @@ -120,7 +122,7 @@ client.evaluations.run( See [Prompt registry](EVALUATIONS.md#prompt-registry) in the full guide, or [self-host's docs](https://docs.agentx.so/improve/prompt-management) for the "Suggest improvement" dashboard flow (self-host only - no hosted-SaaS equivalent yet). -On self-host, a finalized run can also **gate a CI job**: `run.gate(fail_under=7, no_regression=True)` (on the run context `.execute()` returns) checks the run's average rating against an absolute floor and/or the dataset's previous run, prints per-check verdicts into the CI log, and returns an exit code - `sys.exit(gate.exit_code)` blocks the merge on regression. Recorded gates appear in the dashboard's CI Gates tab. See [self-host's CI docs](https://docs.agentx.so/integrations/self-host-ci) for the GitHub Actions recipe. +On self-host, a finalized run can also **gate a CI job**: `run.gate(fail_under=7, no_regression=True)` (on the run context `.execute()` returns) checks the run's average rating against an absolute floor and/or the dataset's previous run, prints per-check verdicts into the CI log, and returns a `GateResult` - `sys.exit(gate.exit_code)` blocks the merge on regression. Recorded gates appear in the dashboard's CI Gates tab. See [self-host's CI docs](https://docs.agentx.so/integrations/self-host-ci) for the GitHub Actions recipe. See **[EVALUATIONS.md](EVALUATIONS.md)** for the full guide - dataset builder, framework adapters, similarity metrics, smoke testing, judge configuration, prompt registry, and the complete API reference. @@ -188,6 +190,10 @@ extra: | LlamaIndex | `pip install "agentx-python[llamaindex]"` | `AgentXLlamaIndexHandler` | | AutoGen | `pip install "agentx-python[autogen]"` | `AgentXAutoGenObserver` | +Two more platforms are covered by **pull importers** rather than in-process hooks, each with its +own CLI: `agentx-moveworks` (Moveworks Data API sync, no extra needed) and `agentx-databricks` +(`pip install "agentx-python[databricks]"`, MLflow/Databricks trace sync). + Or plain Python - wrap any function with `@tracer.trace(...)` and it just works, no framework required. Tracing is **platform agnostic**: each integration stamps its platform label automatically, a plain trace auto-detects the one orchestration framework imported in the @@ -228,12 +234,14 @@ for signal in client.monitor.signals.list(severity="high"): print(signal.summary, signal.occurrence_count) ``` -Per-agent coverage/threshold settings (sample rate, retention, and threshold overrides like the built-in "Latency regression" pattern's threshold) are `client.monitor.profile.get()`/`.update()`: +Per-agent monitoring settings (enable/disable, detection categories, notification channels) are `client.monitor.profile.get()`/`.update()`: ```python -client.monitor.profile.update("agent_123", threshold_overrides={"latencyMs": 15000}) +client.monitor.profile.update("agent_123", info_detection_enabled=False) ``` +On self-host, coverage mode, sample rate, retention, and the built-in latency threshold are project-level defaults set in the dashboard's Platform Settings; `update()` still accepts them for wire compatibility, but only the per-agent fields above take effect there. + Self-host also has **online evaluators**: a real LLM judge scoring a sample of live traffic continuously, distinct from a pattern's rule-matching. A score below `alert_threshold` raises a signal the same way a failing pattern does, deduped and triage-ready in `client.monitor.signals`. ```python @@ -307,7 +315,7 @@ export AGENTX_API_BASE_URL=http://localhost:4700/api/v1 export AGENTX_API_KEY= ``` -`agentx-trace-eval` isn't this SDK's own code - the engine itself is a separate, compiled binary, downloaded on demand rather than bundled into this package, so installing `agentx-python` doesn't get any heavier for the (much more common) case of just talking to the hosted AgentX API. See that repo's README for what's included, and `AGENTX_INSTALL_DIR`/`AGENTX_TRACE_EVAL_VERSION`/`AGENTX_TRACE_EVAL_SKIP_WEB` env vars to control where/what it installs. +`agentx-trace-eval` isn't this SDK's own code - the engine itself is a separate, compiled binary, downloaded on demand rather than bundled into this package, so installing `agentx-python` doesn't get any heavier for the (much more common) case of just talking to the hosted AgentX API. Each SDK release pins the engine release it was tested against and converges the install to that pin, so upgrading the SDK upgrades the engine too. Two flags to know: `--update` (consumed by this launcher) force-reinstalls the resolved engine release, while `--upgrade` passes through to `agentx-server` and re-downloads the dashboard bundle before serving. See that repo's README for what's included, and the `AGENTX_INSTALL_DIR`/`AGENTX_TRACE_EVAL_VERSION`/`AGENTX_TRACE_EVAL_SKIP_WEB` env vars to control where/what it installs. --- diff --git a/TRACING.md b/TRACING.md index 1ab986d..0f37af2 100644 --- a/TRACING.md +++ b/TRACING.md @@ -41,16 +41,20 @@ response = handle_query("How do I reset my password?") # Explicit API key client = AgentX(api_key="ax_live_xxxxxxxxxxxxxxxx") -# From environment variable (AGENTX_API_KEY) +# From environment variables client = AgentX.from_env() ``` -Set the environment variable: +`from_env()` reads `AGENTX_API_KEY`, plus - for the base URL - the first of `AGENTX_API_BASE_URL` / `AGENTX_SELFHOST_BASE_URL` / `BASE_URL` that is set: ```bash export AGENTX_API_KEY=ax_live_xxxxxxxxxxxxxxxx +# self-host only: +export AGENTX_API_BASE_URL=http://localhost:4700/api/v1 ``` +The tracer is fire-and-forget: a wrong key or URL surfaces only as a one-time log warning while traces silently go nowhere. For a long-running service, call `client.ping()` once at startup - it raises immediately (`AgentXConnectionError` / `AgentXAuthError`) on a bad URL or key. + --- ## The `Tracer` @@ -67,7 +71,7 @@ All tracing methods are on the `tracer` object. ## `tracer.trace()` - decorator / context manager -The primary tracing interface. Captures the wrapped function's arguments as `input`, return value as `output`, wall-clock time as `latencyMs`, and any exception as `error`. +The primary tracing interface. Captures the wrapped function's arguments as `input`, return value as `output`, wall-clock time as `latency_ms`, and any exception as `error`. ### As a decorator @@ -94,22 +98,79 @@ with tracer.trace("agent-name", framework="langchain") as span: ### Parameters +All parameters work in both decorator and context-manager form - the decorator forwards every one of them (including `sync`, `monitor`, `pattern_ids`, `agent_id`, and `span_kind`) onto the span it opens per call. + | Parameter | Type | Required | Description | |---|---|---|---| -| `name` | `str` | ✓ | Agent or operation label shown in the UI | +| `name` | `str` | ✓ | Agent or operation label shown in the UI. One stable agent is resolved per distinct name | +| `input` | any | - | Initial input value (context manager form; the decorator captures the function's arguments) | | `framework` | `str` | - | Platform label - any string, including custom platform names. Auto-filled when omitted: see [Platform detection](#platform-detection) | | `model` | `str` | - | LLM model used, e.g. `"gpt-4o"`, `"claude-sonnet-4-6"` | | `session_id` | `str` | - | Groups traces from the same user session or thread | -| `metadata` | `dict` | - | Arbitrary key-value metadata (not indexed, max 16 KB) | +| `metadata` | `dict` | - | Arbitrary key-value metadata | +| `sync` | `bool` | - | `True` sends the trace synchronously on exit so `span.trace_id` is populated. See [Getting the trace id back](#getting-the-trace-id-back-synctrue) | +| `monitor` | `bool` | - | `True` checks this trace against Monitor patterns immediately; `False` opts out of every ingest-time check. Default (`None`) leaves the server's standard behavior. See [Monitor](#monitor) | +| `pattern_ids` | `list[str]` | - | With `monitor=True`: restrict detection to exactly these pattern ids | +| `agent_id` | `str` | - | Pin this trace to a known agent id instead of resolving by `name` - a disambiguator for when the name alone isn't enough | +| `span_kind` | `str` | - | What kind of step this span is (`"agent"`, `"llm"`, `"tool"`, `"retrieval"`, ...), stated instead of left to the backend's classification fallback | -### `_TraceSpan` methods (context manager only) +### `_TraceSpan` methods and attributes (context manager form) | Method / Attribute | Description | |---|---| | `span.input = value` | Override the captured input | -| `span.output = value` | Set the output (required in context manager mode) | +| `span.output = value` | Set the output | | `span.add_tool_call(name, *, input, output, latency_ms)` | Record a tool call made during the span | | `span.set_error(message)` | Mark the span as failed with the given error message | +| `span.trace_id` | The ingested trace's id - populated after the `with` block exits, and only when the span was opened with `sync=True` (otherwise `None`) | +| `span.span_id` | This span's id, usable to parent further spans | +| `span.child_span(name, *, start_time, end_time, input, output, ...)` | Send one already-finished child-span row with explicit timing, parented to this span. Returns the child (its `.span_id` can parent grandchildren) | + +--- + +## Span trees and nesting + +Nested `with tracer.trace(...)` blocks link as real parent/child span rows sharing one session, so a multi-step run shows up as a tree in the trace dialog's span panel. Nesting is automatic: any span opened while another is active on the same thread becomes its child, and so does every auto-instrumented call made inside the block (a patched Anthropic/OpenAI/Google GenAI/LiteLLM client, or a framework integration like `AgentXCallbackHandler`): + +```python +with tracer.trace("orchestrator") as root: + with tracer.trace("plan") as plan: # child span of "orchestrator" + plan.output = make_plan(query) + reply = claude.messages.create(...) # patched client: its own child-span row + root.output = reply +``` + +The active-span stack is **thread-local**. Work submitted to a `ThreadPoolExecutor` (or any other thread) doesn't see a span opened on the calling thread - wrap the worker body in `tracer.use_span(span)` to attach it: + +```python +with tracer.trace("orchestrator") as span: + def worker(): + with tracer.use_span(span): + chain.invoke(..., config={"callbacks": [handler]}) + + with ThreadPoolExecutor(max_workers=2) as ex: + ex.submit(worker).result() +``` + +`use_span` is safe to use from multiple threads concurrently for the same span. + +--- + +## Getting the trace id back (`sync=True`) + +By default a trace is queued and delivered by a background thread - it never blocks the caller, but there is no way to learn the resulting trace id. Pass `sync=True` to send synchronously on block exit instead, so `span.trace_id` is populated: + +```python +with tracer.trace("support_agent_call", sync=True) as span: + resp = call_llm(query) + span.output = resp + +print(span.trace_id) # ready - e.g. to link this trace to an evaluation result +``` + +On a root span, `sync=True` covers the whole tree: child spans recorded inside the block are drained before the root is sent, so a read immediately afterwards sees every span. This is exactly what evaluation harnesses use to link a case to its trace: return `{"output": resp, "trace_id": span.trace_id}` from the agent function (see EVALUATIONS.md). + +`tracer.flush(timeout=5.0)` is the companion for the default async mode: it blocks until all queued traces are delivered (or the timeout elapses, returning `False` and leaving delivery running in the background). Call it before a short-lived process exits. --- @@ -147,9 +208,12 @@ works. The label resolves in priority order: unknown means no label - the trace still ingests fine and buckets as "Other / custom" in the dashboard, never mislabeled. -The label powers the Live Traces framework filter and Monitor's **Platforms** chart -(`GET /agent-monitoring/metrics` - `byFramework` buckets, `frameworks` totals, and a -`framework=` filter). +The label powers the Live Traces framework filter and Monitor's **Platforms** chart. From the +SDK, the same numbers come from `client.monitor.metrics()` (`GET /monitor/metrics`): bucketed +spans by kind, latency percentiles, tokens/cost, tool executions/failures, and platform +attribution (`frameworks` window totals plus per-bucket `byFramework`). It takes a `window` +(`"1h"` up to `"90d"`, default `"1d"`) and optional `agent`/`model`/`tool`/`framework`/`status` +filters matching the dashboard's filter chips - `framework="other"` selects unlabeled traffic. ## Framework examples @@ -288,7 +352,22 @@ with tracer.trace("support-agent") as span: span.output = answer ``` -An exception escaping the block records the call with `success=False` plus the error text, then propagates unchanged so your own error handling still runs. That `success: false` is what Monitor's built-in "Tool failure" check and the dashboard's Tool quality column read, so a flaky tool shows up in triage without any extra wiring. To set the outcome yourself instead (an API that returned a well-formed error payload, say), use `tracer.record_tool_call(name, input=..., output=..., success=False, error="...")`. +An exception escaping the block records the call with `success=False` plus the error text, then propagates unchanged so your own error handling still runs. That `success: false` is what Monitor's built-in "Tool failure" check and the dashboard's Tool quality column read, so a flaky tool shows up in triage without any extra wiring. To set the outcome yourself instead (an API that returned a well-formed error payload, say), use `tracer.record_tool_call(name, input=..., output=..., success=False, error="...")`. Both attach to the innermost active span; with no active span, the call is queued onto the next trace this tracer sends. + +### Recording retrievals + +The retrieval twins of the tool-call helpers mark a span as a knowledge-base / vector-store lookup, which feeds the engine's retrieval-context extraction (used by RAG judges) and the dashboard's references panel: + +```python +with tracer.trace("rag-agent") as span: + with tracer.trace_retrieval("kb_search", query=question) as r: + docs = retrieve(question) + r.doc_count = len(docs) + r.output = docs + span.output = answer_from(docs) +``` + +`tracer.record_retrieval(name, query=..., output=..., duration_ms=...)` is the after-the-fact form. Custom names like `"kb_search"` work - the span carries an explicit retrieval marker, not a name heuristic. --- @@ -348,39 +427,34 @@ with tracer.trace("my-agent") as span: ## `tracer.evaluate_trace()` -Evaluate a previously submitted trace against a dataset, without re-running the agent. Returns a score and justification. +Score a previously ingested trace against a dataset, without re-running the agent (`POST /ingest/traces/{trace_id}/evaluate`, synchronous - it blocks for one judge call). The trace's recorded input/output are used as-is. ```python +with tracer.trace("support-agent", sync=True) as span: + span.output = call_llm(query) + result = tracer.evaluate_trace( - trace_id="6876abc123def456789abc01", + trace_id=span.trace_id, dataset_id="6876ddd222bbb333ccc444ee", question_index=0, # optional - which question to score against ) -print(result.rating) # 1–5 -print(result.justification) # LLM explanation -print(result.run_id) # ID of the created eval run +print(result["rating"]) # 0-10 (None if the judge could not score) +print(result["justification"]) # LLM explanation +print(result["run_id"]) # id of the one-result eval run this created ``` ### Parameters | Parameter | Type | Required | Description | |---|---|---|---| -| `trace_id` | `str` | ✓ | Trace ID returned by `POST /ingest/traces` | +| `trace_id` | `str` | ✓ | Id of an ingested trace - from `span.trace_id` after a `sync=True` block | | `dataset_id` | `str` | ✓ | Evaluation dataset to score against | -| `question_index` | `int` | - | 0-based question index. Omit to score against general criteria only | +| `question_index` | `int` | - | 0-based question index; when supplied, that question's `expectedResults` is included in the scoring prompt. Omit to score against the dataset's general criteria only | -### Return type: `TraceEvalResult` +### Return value -```python -@dataclass -class TraceEvalResult: - run_id: str - trace_id: str - rating: int # 1–5 - justification: str - status: str # "completed" -``` +A plain `dict` with keys `run_id`, `trace_id`, `rating` (0-10, `None` when scoring failed), `justification`, and `status` (`"completed"`). Raises `requests.HTTPError` on a non-2xx response. --- @@ -418,7 +492,7 @@ Enable monitoring once per agent in the dashboard, and every subsequent trace fr ### What gets checked -Built-in detectors: empty response, trace/tool errors, latency regressions, and (native chat agents only) negative user feedback. Custom patterns: keyword, regex, or an LLM-judged semantic rubric, created via the dashboard or `client.monitor.patterns`. A match becomes a signal, deduped against repeat occurrences of the same issue. A trace that matches nothing counts toward the agent's health rate instead. +Built-in detectors: empty response, trace/tool errors, latency regressions, and negative user feedback (native chat votes; on self-host, votes forwarded via `client.feedback.report(...)` too). Custom patterns: keyword, regex, or an LLM-judged semantic rubric, created via the dashboard or `client.monitor.patterns`. A match becomes a signal, deduped against repeat occurrences of the same issue. A trace that matches nothing counts toward the agent's health rate instead. ### `client.monitor.patterns` @@ -450,7 +524,7 @@ client.monitor.patterns.list() # -> list[MonitorPattern] | `severity` | `str` | `"medium"` | `"low"`, `"medium"`, `"high"`, or `"critical"` | | `polarity` | `str` | `"failure"` | `"failure"` raises a signal to triage; `"proper"` logs a healthy tally instead | | `enabled` | `bool` | `True` | Whether the pattern is checked at all | -| `sample_rate` | `float` | `1.0` | Fraction of matching traces to actually check, `0.0`–`1.0` | +| `sample_rate` | `float` | `1.0` | Fraction of matching traces to actually check, `0.0`-`1.0` | | `scope_mode` / `agent_ids` | `str` / `list[str]` | `"all"` / `[]` | Restrict this pattern to specific agents instead of the whole workspace | `publish()` returns a `MonitorPattern` with `.id`, which you pass in `pattern_ids` at trace time. @@ -482,17 +556,17 @@ print(signal.recommended_actions) ### `client.monitor.profile` -Get/update one agent's Monitor coverage and detection settings, the same settings shown in the dashboard's per-agent monitoring settings dialog (Observe > Patterns > Agents view): coverage mode, sample rate, retention, redaction, approval policy, and `threshold_overrides` for built-in detectors that take a configurable threshold (e.g. the "Latency regression" pattern's threshold, which otherwise defaults to 20000ms). +Get/update one agent's Monitor settings, the same settings shown in the dashboard's per-agent monitoring settings dialog. ```python profile = client.monitor.profile.get("agent_123") -print(profile.coverage_mode if profile else "never configured, on defaults") +print(profile.enabled if profile else "never configured, on defaults") -# Override just the latency-regression threshold, e.g. 15s instead of the 20s default. -client.monitor.profile.update("agent_123", threshold_overrides={"latencyMs": 15000}) +# Opt this agent out of info (clean-run) signals. +client.monitor.profile.update("agent_123", info_detection_enabled=False) ``` -`get()` returns `None` when the agent has never been configured (still on platform defaults). `update()` upserts and only changes the fields you pass, everything else on an existing profile is left as is: +`get()` returns `None` when the agent has never been configured (still on platform defaults, e.g. a built-in latency threshold of 20000ms). `update()` upserts and only changes the fields you pass, everything else on an existing profile is left as is: | Parameter | Type | Description | |---|---|---| @@ -504,9 +578,10 @@ client.monitor.profile.update("agent_123", threshold_overrides={"latencyMs": 150 | `dataset_id` | `str` | Evaluation dataset this agent's signals feed into | | `threshold_overrides` | `dict` | Per-check threshold overrides, e.g. `{"latencyMs": 15000}` | | `retention_days` | `int` | How long monitored traces are kept | -| `redaction_mode` | `str` | `"none"`, `"standard"`, or `"strict"` | | `approval_policy` | `dict[str, str]` | Per-action approval mode for autotune actions | +**Self-host:** `coverage_mode`, `sample_rate`, `retention_days`, and `threshold_overrides["latencyMs"]` are project-level defaults there (set once for every agent in the dashboard's Platform Settings). `update()` still accepts them for wire compatibility, but the self-host engine doesn't read the stored per-agent values - `enabled`, the two detection toggles, and `channels` remain real per-agent settings everywhere. + ### `client.monitor.online_evaluators` (self-host only) > **Legacy view.** An online evaluator is the *online profile* of an **LLM Judge Scorer**, and this client is the half-view of it that predates the consolidation. It keeps working unchanged and its ids are the same ids, but it emits a `DeprecationWarning` on first use. Prefer [`client.monitor.judge_scorers`](EVALUATIONS.md#llm-judge-scorers---reusable-grading-configs), which manages the judge rubric, the offline (dataset-run) profile and this online profile as one entity - and note that route needs a self-host engine build that serves it, where this one works on every engine. @@ -555,19 +630,24 @@ A `scope="session"` evaluator is judged by the engine's background sweep rather ## Async support -All tracing methods work with both sync and async functions: +The decorator wraps both sync and async functions (it detects coroutine functions and awaits them): ```python @tracer.trace("async-agent", framework="openai-agents") async def handle_async(query: str) -> str: response = await async_llm_client.complete(query) return response +``` -# Or in async context manager: -async with tracer.trace("async-agent") as span: - span.input = query - result = await async_llm_client.complete(query) - span.output = result +The context-manager form is a regular (synchronous) context manager - use plain `with` inside async code, not `async with`: + +```python +async def handle(query: str) -> str: + with tracer.trace("async-agent") as span: + span.input = query + result = await async_llm_client.complete(query) + span.output = result + return result ``` --- @@ -578,22 +658,20 @@ async with tracer.trace("async-agent") as span: from agentx import AgentX client = AgentX( - api_key="ax_live_xxxxxxxxxxxxxxxx", # Required (or use AGENTX_API_KEY env var) - workspace_id="...", # Optional - explicit workspace override - base_url="https://api.agentx.so", # Optional - for self-hosted deployments - timeout=10, # HTTP timeout in seconds (default 10) + api_key="ax_live_xxxxxxxxxxxxxxxx", # or set AGENTX_API_KEY + workspace_id="...", # optional - or set AGENTX_WORKSPACE_ID + base_url="http://localhost:4700/api/v1", # optional - or set AGENTX_API_BASE_URL; + # defaults to the hosted API ) ``` ---- +Constructing the client makes no network call; `client.ping()` is the fail-fast startup check. -## Limits +--- -| Limit | Value | -|---|---| -| Traces per minute | 300 | -| Max tool calls per trace | 50 (excess silently truncated) | -| Max `input` / `output` size | 1 MB each | -| Max `metadata` size | 16 KB | +## Delivery behavior and limits -Traces that exceed size limits are submitted with the oversized field truncated and a warning logged to stderr. +- **Queueing** - traces are enqueued (up to 500 in flight) and drained by a background daemon thread. On overflow, or when retries are exhausted, the trace is dropped **with a logged warning** (first drop, then every 50th, with a cumulative count) - never silently. +- **Retries** - each queued trace is retried up to 3 times with backoff on connection errors, 429, and 5xx responses; a 429's `Retry-After` header is honored. `sync=True` sends block once with a 10s timeout and do not retry - a failed sync send just means `span.trace_id` stays `None`. +- **Payload truncation** - `input`, `output`, and `metadata` are serialized best-effort before sending: nesting deeper than 4 levels, dicts/lists beyond 30 entries, and unserializable objects are truncated/stringified (long fallback strings cut to 200 chars) to keep payloads bounded. +- **First failure warns** - the first delivery failure per client logs at WARNING with a hint (bad key vs. bad URL); repeats log at DEBUG. `client.ping()` at startup fails fast instead. diff --git a/agentx/integrations/anthropic.py b/agentx/integrations/anthropic.py index 716340c..2d54ea2 100644 --- a/agentx/integrations/anthropic.py +++ b/agentx/integrations/anthropic.py @@ -13,7 +13,7 @@ Works with both ``anthropic.Anthropic`` and ``anthropic.AsyncAnthropic`` clients. -Requires: ``pip install agentx[anthropic]`` +Requires: ``pip install "agentx-python[anthropic]"`` """ from __future__ import annotations diff --git a/agentx/integrations/crewai.py b/agentx/integrations/crewai.py index a951c60..7ca40b5 100644 --- a/agentx/integrations/crewai.py +++ b/agentx/integrations/crewai.py @@ -14,7 +14,7 @@ result = crew.kickoff(inputs={"topic": "AI"}) span.output = result.raw -Requires: ``pip install agentx[crewai]`` +Requires: ``pip install "agentx-python[crewai]"`` """ from __future__ import annotations diff --git a/setup.py b/setup.py index 7e805e5..3f1931f 100644 --- a/setup.py +++ b/setup.py @@ -70,6 +70,7 @@ def get_long_description(): "llama-index-core>=0.10.0", "autogen-agentchat>=0.4.0", "autogen-core>=0.4.0", + "mlflow>=3.6.0", ], }, author="Robin Wang and AgentX Team", @@ -83,5 +84,5 @@ def get_long_description(): "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", ], - python_requires=">=3.6", + python_requires=">=3.9", ) diff --git a/tests/test_selfhost_compat.py b/tests/test_selfhost_compat.py new file mode 100644 index 0000000..e959a01 --- /dev/null +++ b/tests/test_selfhost_compat.py @@ -0,0 +1,424 @@ +"""SDK-to-self-host compatibility matrix. + +Every public SDK surface is exercised against a LIVE self-host engine and must land in +exactly one of two tables: + +- SELF_HOST: the surface must answer without raising. Any exception fails the test with + the surface's name, so a "fictional" surface (one the engine never grew) can't ship. +- HOSTED_ONLY: the surface must KEEP failing against the engine AND carry a documented + reason (plus a docs-file banner that says so). If the engine grows the surface, the + test fails loudly telling us to promote the entry to SELF_HOST. + +Opt-in, like the engine's own backend suites: the whole module skips unless both env +vars below are set. + +How to run: + 1. Boot a scratch engine (any free port, throwaway home dir): + cd AgentX-trace-eval/engine + PORT=4799 AGENTX_HOME=$(mktemp -d) yarn dev + 2. Copy the "Default project API key: agtx_local_..." line from its startup log. + 3. Run the suite: + AGENTX_COMPAT_BASE_URL=http://localhost:4799/api/v1 \ + AGENTX_COMPAT_API_KEY=agtx_local_... \ + pytest tests/test_selfhost_compat.py -q + +The scratch engine usually has no judge/provider keys. That is fine and deliberate: +judge-dependent steps (eval-run scoring) then record results as skipped/unrated, and this +suite only asserts that every surface ANSWERS, never that the judge liked the answer. +""" + +from __future__ import annotations + +import os +import time +import uuid +from pathlib import Path + +import pytest + +BASE_URL = os.getenv("AGENTX_COMPAT_BASE_URL") +API_KEY = os.getenv("AGENTX_COMPAT_API_KEY") + +pytestmark = pytest.mark.skipif( + not (BASE_URL and API_KEY), + reason="self-host compat suite is opt-in: set AGENTX_COMPAT_BASE_URL and AGENTX_COMPAT_API_KEY", +) + +REPO_ROOT = Path(__file__).resolve().parent.parent + + +def _tag() -> str: + return uuid.uuid4().hex[:8] + + +# --------------------------------------------------------------------------- +# Shared state: one client, plus lazily created artifacts (a sync trace, one +# full eval run) reused across parametrized tests so the suite stays fast. +# --------------------------------------------------------------------------- + + +class CompatContext: + def __init__(self) -> None: + # Quiet the runner's interactive spinner/banner in test output. + os.environ.setdefault("AGENTX_EVAL_QUIET", "1") + from agentx import AgentX + + self.client = AgentX(api_key=API_KEY, base_url=BASE_URL) + self.session_id = f"compat-{_tag()}" + self._trace_id: str | None = None + self._eval: dict | None = None + self.created_dataset_ids: list[str] = [] + + # -- lazy shared artifacts ------------------------------------------------ + + def trace_id(self) -> str: + """One sync-ingested trace, created on first use (sync=True so the id exists).""" + if self._trace_id is None: + with self.client.tracer.trace( + "compat-check-agent", + input={"query": "compat ping"}, + session_id=self.session_id, + sync=True, + monitor=False, + ) as span: + span.output = "compat pong" + assert span.trace_id, "tracer.trace(sync=True) exited without a trace_id" + self._trace_id = span.trace_id + return self._trace_id + + def eval_artifacts(self) -> dict: + """One full evaluation lifecycle, run once: dataset publish -> init_run (via + client.evaluations.run) -> execute/submit one result -> finalize -> gate. + A keyless engine records the result unrated; every surface must still answer.""" + if self._eval is None: + ds = ( + self.client.evaluations.datasets.builder( + name=f"compat-ds-{_tag()}", + description="scratch dataset for the self-host compat matrix", + ) + .add_case("What is 2 + 2?", expected_results="4") + .publish() + ) + self.created_dataset_ids.append(ds.id) + run_ctx = self.client.evaluations.run( + ds.id, {"kind": "custom_agent", "displayName": "compat-check"} + ) + run_ctx.execute(lambda case: "4") + run_ctx.finalize() + gate = run_ctx.gate(fail_under=0.0) + self._eval = {"dataset_id": ds.id, "run_id": run_ctx.run_id, "gate": gate} + return self._eval + + # -- cleanup -------------------------------------------------------------- + + def cleanup(self) -> None: + for dataset_id in self.created_dataset_ids: + try: + self.client.evaluations.datasets.delete(dataset_id) + except Exception: + pass + + +@pytest.fixture(scope="module") +def compat(): + ctx = CompatContext() + yield ctx + ctx.cleanup() + + +# --------------------------------------------------------------------------- +# SELF_HOST checks - each must answer without raising +# --------------------------------------------------------------------------- + + +def _check_ping(ctx: CompatContext) -> None: + result = ctx.client.ping() + assert result.get("ok") is True + + +def _check_tracer_sync_trace_and_flush(ctx: CompatContext) -> None: + assert ctx.trace_id() + assert ctx.client.tracer.flush(timeout=10) is True + + +def _check_traces_get(ctx: CompatContext) -> None: + detail = ctx.client.traces.get(ctx.trace_id()) + assert isinstance(detail, dict) and detail + + +def _check_traces_list(ctx: CompatContext) -> None: + ctx.trace_id() # make sure at least one trace exists + page = ctx.client.traces.list(limit=5) + assert isinstance(page.get("traces"), list) and page["traces"] + + +def _check_monitor_kpis(ctx: CompatContext) -> None: + assert isinstance(ctx.client.monitor.kpis(), dict) + + +def _check_monitor_metrics(ctx: CompatContext) -> None: + assert isinstance(ctx.client.monitor.metrics(window="1h"), dict) + + +def _check_monitor_topics(ctx: CompatContext) -> None: + assert isinstance(ctx.client.monitor.topics(), dict) + + +def _check_monitor_list_agents(ctx: CompatContext) -> None: + ctx.trace_id() # tracing auto-creates the agent + agents = ctx.client.monitor.list_agents() + assert isinstance(agents, list) + + +def _check_monitor_patterns(ctx: CompatContext) -> None: + # builder + publish + get + list. The SDK exposes no pattern delete, so the + # published pattern stays behind on the scratch engine (throwaway by design). + pattern = ctx.client.monitor.patterns.builder( + name=f"compat-pattern-{_tag()}", + detector_kind="contains", + include_terms=["compat-term-that-never-matches"], + enabled=False, + ).publish() + assert pattern.id + assert ctx.client.monitor.patterns.get(pattern.id).id == pattern.id + assert any(p.id == pattern.id for p in ctx.client.monitor.patterns.list()) + + +def _check_monitor_judge_scorers_round_trip(ctx: CompatContext) -> None: + scorer = ctx.client.monitor.judge_scorers.builder( + name=f"compat-scorer-{_tag()}", + acceptance_criteria="The answer is correct.", + ).publish() + try: + assert ctx.client.monitor.judge_scorers.get(scorer.id).id == scorer.id + updated = ctx.client.monitor.judge_scorers.update( + scorer.id, + online={"enabled": False, "sampleRate": 0.1, "alertThreshold": 5}, + ) + assert updated.id == scorer.id + finally: + ctx.client.monitor.judge_scorers.delete(scorer.id) + + +def _check_monitor_scorers_list(ctx: CompatContext) -> None: + assert isinstance(ctx.client.monitor.scorers.list(), list) + + +def _check_monitor_review_queue_list(ctx: CompatContext) -> None: + assert isinstance(ctx.client.monitor.review_queue.list(status="all"), list) + + +def _check_monitor_rules_list(ctx: CompatContext) -> None: + assert isinstance(ctx.client.monitor.rules.list(), list) + + +def _check_monitor_sessions_spans(ctx: CompatContext) -> None: + ctx.trace_id() # ingests one span into ctx.session_id + spans = ctx.client.monitor.sessions.spans(ctx.session_id) + assert isinstance(spans, list) and spans + + +def _check_evaluations_dataset_round_trip(ctx: CompatContext) -> None: + ds = ( + ctx.client.evaluations.datasets.builder( + name=f"compat-ds-roundtrip-{_tag()}", + description="round-trip dataset (deleted by this test)", + ) + .add_case("Name a prime number.", expected_results="Any prime, e.g. 7") + .publish() + ) + assert ds.id + fetched = ctx.client.evaluations.datasets.get(ds.id) + assert fetched.id == ds.id and len(fetched.questions) == 1 + ctx.client.evaluations.datasets.delete(ds.id) + + +def _check_evaluations_run_lifecycle(ctx: CompatContext) -> None: + artifacts = ctx.eval_artifacts() + assert artifacts["run_id"] + # Keyless judge => unrated results => gate answers but may not pass. Both fine. + assert isinstance(artifacts["gate"].passed, bool) + + +def _check_evaluations_get_run(ctx: CompatContext) -> None: + run = ctx.client.evaluations.get_run(ctx.eval_artifacts()["run_id"]) + assert isinstance(run, dict) and run + + +def _check_evaluations_list_gates(ctx: CompatContext) -> None: + ctx.eval_artifacts() # records one gate verdict + assert isinstance(ctx.client.evaluations.list_gates(), list) + + +def _check_evaluations_prompts_registry(ctx: CompatContext) -> None: + name = f"compat-prompt-{_tag()}" + created = ctx.client.evaluations.prompts.create( + name, "You are a compat-check assistant.", description="compat matrix scratch prompt" + ) + assert created.version >= 1 + fetched = ctx.client.evaluations.prompts.get(name) + assert fetched.name == name and fetched.text + assert any(p.name == name for p in ctx.client.evaluations.prompts.list()) + + +def _check_feedback_report(ctx: CompatContext) -> None: + report = ctx.client.feedback.report( + trace_id=ctx.trace_id(), rating="up", end_user_id="compat-suite" + ) + assert isinstance(report, dict) + + +def _check_outcomes_report(ctx: CompatContext) -> None: + report = ctx.client.outcomes.report( + trace_id=ctx.trace_id(), + outcome="confirmed_good", + is_negative=False, + reported_by="compat-suite", + ) + assert isinstance(report, dict) + + +def _check_export_manifest_and_iter(ctx: CompatContext) -> None: + ctx.trace_id() # at least one exportable row + manifest = ctx.client.export.manifest() + assert isinstance(manifest, list) and manifest + rows = list(ctx.client.export.iter("traces")) + assert rows and all(isinstance(r, dict) for r in rows) + + +SELF_HOST = [ + ("client.ping", _check_ping), + ("tracer.trace(sync=True) + tracer.flush", _check_tracer_sync_trace_and_flush), + ("traces.get", _check_traces_get), + ("traces.list", _check_traces_list), + ("monitor.kpis", _check_monitor_kpis), + ("monitor.metrics", _check_monitor_metrics), + ("monitor.topics", _check_monitor_topics), + ("monitor.list_agents", _check_monitor_list_agents), + ("monitor.patterns builder/publish/get/list", _check_monitor_patterns), + ("monitor.judge_scorers create/get/update/delete", _check_monitor_judge_scorers_round_trip), + ("monitor.scorers.list", _check_monitor_scorers_list), + ("monitor.review_queue.list", _check_monitor_review_queue_list), + ("monitor.rules.list", _check_monitor_rules_list), + ("monitor.sessions.spans", _check_monitor_sessions_spans), + ("evaluations.datasets builder/publish/get/delete", _check_evaluations_dataset_round_trip), + ("evaluations run/execute/finalize/gate", _check_evaluations_run_lifecycle), + ("evaluations.get_run", _check_evaluations_get_run), + ("evaluations.list_gates", _check_evaluations_list_gates), + ("evaluations.prompts create/get/list", _check_evaluations_prompts_registry), + ("feedback.report", _check_feedback_report), + ("outcomes.report", _check_outcomes_report), + ("export.manifest + export.iter", _check_export_manifest_and_iter), +] + + +@pytest.mark.parametrize( + "surface,check", SELF_HOST, ids=[name for name, _ in SELF_HOST] +) +def test_self_host_surface(surface, check, compat): + try: + check(compat) + except AssertionError: + raise + except Exception as exc: + pytest.fail( + f"self-host surface {surface!r} raised {type(exc).__name__}: {exc} " + "(either the SDK or the engine drifted - this surface is supposed to work " + "against a self-host engine)" + ) + + +# --------------------------------------------------------------------------- +# HOSTED_ONLY checks - each must KEEP failing against the engine, and the docs +# must say so. If one starts working, promote it to SELF_HOST above. +# --------------------------------------------------------------------------- + +CI_BANNER_NEEDLE = "Hosted platform only." +CI_ROUTE_NEEDLE = "/ingest/ci-runs" + + +def _call_run_eval(ctx: CompatContext): + return ctx.client.tracer.run_eval("evds_compat_missing", lambda q: "answer") + + +def _call_create_ci_run(ctx: CompatContext): + return ctx.client.tracer.create_ci_run("evds_compat_missing") + + +def _call_get_ci_run(ctx: CompatContext): + return ctx.client.tracer.get_ci_run("cirun_compat_missing") + + +def _call_finalize_ci_run(ctx: CompatContext): + return ctx.client.tracer.finalize_ci_run("cirun_compat_missing") + + +def _call_list_models(ctx: CompatContext): + return ctx.client.evaluations.list_models() + + +HOSTED_ONLY = [ + ( + "tracer.run_eval", + _call_run_eval, + "Targets the hosted /ingest/ci-runs API; the self-host engine does not serve it. " + "Self-host CI gating is evaluations.run(...).execute(...).finalize().gate(...).", + "CICD_EVAL.md", + (CI_BANNER_NEEDLE, CI_ROUTE_NEEDLE), + ), + ( + "tracer.create_ci_run", + _call_create_ci_run, + "Low-level hosted /ingest/ci-runs call; 404 on self-host, surfaced as DatasetNotFound.", + "CICD_EVAL.md", + (CI_BANNER_NEEDLE, CI_ROUTE_NEEDLE), + ), + ( + "tracer.get_ci_run", + _call_get_ci_run, + "Low-level hosted /ingest/ci-runs call; 404 on self-host, surfaced as DatasetNotFound.", + "CICD_EVAL.md", + (CI_BANNER_NEEDLE, CI_ROUTE_NEEDLE), + ), + ( + "tracer.finalize_ci_run", + _call_finalize_ci_run, + "Low-level hosted /ingest/ci-runs call; 404 on self-host, surfaced as DatasetNotFound.", + "CICD_EVAL.md", + (CI_BANNER_NEEDLE, CI_ROUTE_NEEDLE), + ), + ( + "evaluations.list_models", + _call_list_models, + "Targets the hosted /custom-agent-evaluations/models registry; the engine explicitly " + "has not ported it (engine routes/evaluations.ts: 'Still not ported: list_models').", + "EVALUATIONS.md", + ("`list_models()` is **hosted platform only**",), + ), +] + + +@pytest.mark.parametrize( + "surface,call,reason,doc_file,doc_needles", + HOSTED_ONLY, + ids=[name for name, *_ in HOSTED_ONLY], +) +def test_hosted_only_surface(surface, call, reason, doc_file, doc_needles, compat): + assert reason and reason.strip(), f"hosted-only entry {surface!r} must document why" + + try: + call(compat) + except Exception: + pass # expected: the engine does not serve this surface + else: + pytest.fail( + f"hosted-only surface {surface!r} SUCCEEDED against the self-host engine. " + "The engine grew this surface: move the entry to SELF_HOST and update the docs." + ) + + doc_text = (REPO_ROOT / doc_file).read_text(encoding="utf-8") + for needle in doc_needles: + assert needle in doc_text, ( + f"hosted-only surface {surface!r}: expected {doc_file} to contain {needle!r} " + "so the limitation stays documented" + )