diff --git a/.github/workflows/claude-evaluation.yml b/.github/workflows/claude-evaluation.yml index c55d13feb..c4bccbd4a 100644 --- a/.github/workflows/claude-evaluation.yml +++ b/.github/workflows/claude-evaluation.yml @@ -24,6 +24,7 @@ on: - "bug-fix" - "test-generation" - "code-review" + - "data-query" - "extensibility-request-implement" - "extensibility-request-triage" test-run: @@ -41,6 +42,11 @@ on: required: false default: false type: boolean + skills: + description: "Enable agent skills" + required: false + default: false + type: boolean repeat: description: "Number of times to run sequentially (ignored for test runs)" required: false @@ -142,7 +148,8 @@ jobs: --repo-path "${{ steps.setup-env.outputs.repo_path }}" ` --output-dir "${{ env.EVALUATION_RESULTS_DIR }}" ` ${{ inputs.al-mcp && '--al-mcp' || '' }} ` - ${{ inputs.al-lsp && '--al-lsp' || '' }} + ${{ inputs.al-lsp && '--al-lsp' || '' }} ` + ${{ inputs.skills && '--skills' || '' }} - name: Upload evaluation results uses: actions/upload-artifact@v6 @@ -178,4 +185,4 @@ jobs: workflow-file: claude-evaluation.yml repeat: ${{ inputs.repeat }} workflow-inputs: | - {"model": "${{ inputs.model }}", "category": "${{ inputs.category }}", "test-run": "${{ inputs.test-run }}", "al-mcp": "${{ inputs.al-mcp }}", "al-lsp": "${{ inputs.al-lsp }}", "git-ref": "${{ inputs.git-ref || github.ref_name }}"} + {"model": "${{ inputs.model }}", "category": "${{ inputs.category }}", "test-run": "${{ inputs.test-run }}", "al-mcp": "${{ inputs.al-mcp }}", "al-lsp": "${{ inputs.al-lsp }}", "skills": "${{ inputs.skills }}", "git-ref": "${{ inputs.git-ref || github.ref_name }}"} diff --git a/.github/workflows/copilot-evaluation.yml b/.github/workflows/copilot-evaluation.yml index f8ae07197..186cd1271 100644 --- a/.github/workflows/copilot-evaluation.yml +++ b/.github/workflows/copilot-evaluation.yml @@ -29,6 +29,7 @@ on: - "bug-fix" - "test-generation" - "code-review" + - "data-query" - "extensibility-request-implement" - "extensibility-request-triage" test-run: @@ -46,6 +47,11 @@ on: required: false default: false type: boolean + skills: + description: "Enable agent skills" + required: false + default: false + type: boolean repeat: description: "Number of times to run sequentially (ignored for test runs)" required: false @@ -145,7 +151,8 @@ jobs: --repo-path "${{ steps.setup-env.outputs.repo_path }}" ` --output-dir "${{ env.EVALUATION_RESULTS_DIR }}" ` ${{ inputs.al-mcp && '--al-mcp' || '' }} ` - ${{ inputs.al-lsp && '--al-lsp' || '' }} + ${{ inputs.al-lsp && '--al-lsp' || '' }} ` + ${{ inputs.skills && '--skills' || '' }} - name: Upload evaluation results uses: actions/upload-artifact@v6 @@ -181,4 +188,4 @@ jobs: workflow-file: copilot-evaluation.yml repeat: ${{ inputs.repeat }} workflow-inputs: | - {"model": "${{ inputs.model }}", "category": "${{ inputs.category }}", "test-run": "${{ inputs.test-run }}", "al-mcp": "${{ inputs.al-mcp }}", "al-lsp": "${{ inputs.al-lsp }}", "git-ref": "${{ inputs.git-ref || github.ref_name }}"} + {"model": "${{ inputs.model }}", "category": "${{ inputs.category }}", "test-run": "${{ inputs.test-run }}", "al-mcp": "${{ inputs.al-mcp }}", "al-lsp": "${{ inputs.al-lsp }}", "skills": "${{ inputs.skills }}", "git-ref": "${{ inputs.git-ref || github.ref_name }}"} diff --git a/dataset/dataquery.jsonl b/dataset/dataquery.jsonl new file mode 100644 index 000000000..766c7db97 --- /dev/null +++ b/dataset/dataquery.jsonl @@ -0,0 +1,11 @@ +{"instance_id": "dataquery__outstanding-sales-value-by-customer-1", "environment_setup_version": "28.3", "created_at": "2026-07-10", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one open sales order line, return the customer's number, the customer's name, and their total outstanding amount. Use open sales order lines (Sales Line records whose document type is Order) and sum the line 'Outstanding Amount' field (which is net of VAT). Customers with no open sales order lines must not appear.", "ordered": false, "gold_query": "query 50100 OutstandingSalesByCustomer\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(SalesLine; \"Sales Line\")\n {\n DataItemLink = \"Sell-to Customer No.\" = Customer.\"No.\";\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(TotalOutstanding; \"Outstanding Amount\") { Method = Sum; }\n }\n }\n }\n}"} +{"instance_id": "dataquery__total-sold-quantity-by-item-1", "environment_setup_version": "28.3", "created_at": "2026-07-10", "metadata": {"area": "inventory"}, "nl_prompt": "Across all posted sales invoice lines whose type is Item, return each item's number together with the total quantity sold (the sum of the line Quantity). Produce one row per item number.", "ordered": false, "gold_query": "query 50100 SoldQuantityByItem\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n {\n DataItemTableFilter = Type = const(Item);\n column(ItemNo; \"No.\") { }\n column(TotalQuantity; Quantity) { Method = Sum; }\n }\n }\n}"} +{"instance_id": "dataquery__avg-invoice-amount-by-country-1", "environment_setup_version": "28.3", "created_at": "2026-07-10", "metadata": {"area": "finance"}, "nl_prompt": "For each country/region, return the country/region code and the average posted sales invoice line amount. Average the 'Amount' field (net of VAT) over all posted sales invoice lines, grouping the lines by their bill-to customer's Country/Region Code.", "ordered": false, "gold_query": "query 50100 AvgInvoiceAmountByCountry\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(CountryRegionCode; \"Country/Region Code\") { }\n dataitem(SalesInvoiceHeader; \"Sales Invoice Header\")\n {\n DataItemLink = \"Bill-to Customer No.\" = Customer.\"No.\";\n dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n {\n DataItemLink = \"Document No.\" = SalesInvoiceHeader.\"No.\";\n column(AvgAmount; Amount) { Method = Average; }\n }\n }\n }\n }\n}"} +{"instance_id": "dataquery__total-purchase-amount-by-vendor-1", "environment_setup_version": "28.3", "created_at": "2026-07-10", "metadata": {"area": "purchasing"}, "nl_prompt": "For each vendor that has at least one posted purchase invoice line, return the vendor's number, the vendor's name, and their total posted purchase amount. Sum the 'Amount' field (net of VAT) from posted purchase invoice lines. Vendors with no posted purchase invoice lines must not appear.", "ordered": false, "gold_query": "query 50100 PurchaseAmountByVendor\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Vendor; Vendor)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(PurchInvLine; \"Purch. Inv. Line\")\n {\n DataItemLink = \"Buy-from Vendor No.\" = Vendor.\"No.\";\n column(TotalAmount; Amount) { Method = Sum; }\n }\n }\n }\n}"} +{"instance_id": "dataquery__open-sales-order-count-by-customer-1", "environment_setup_version": "28.3", "created_at": "2026-07-13", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one open sales order, return the customer's number, the customer's name, and the number of open sales orders they have. An open sales order is a sales document whose document type is Order; count the order documents (headers), not the order lines. Customers with no open sales orders must not appear.", "ordered": false, "gold_query": "query 50100 OpenSalesOrdersByCustomer\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(SalesHeader; \"Sales Header\")\n {\n DataItemLink = \"Sell-to Customer No.\" = Customer.\"No.\";\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(OrderCount) { Method = Count; }\n }\n }\n }\n}"} +{"instance_id": "dataquery__opportunity-count-by-status-1", "environment_setup_version": "28.3", "created_at": "2026-07-10", "metadata": {"area": "crm"}, "nl_prompt": "Return the number of CRM opportunities in each status. Group the opportunities by their Status field and, for each status value that occurs, output the status and the count of opportunities with that status.", "ordered": false, "gold_query": "query 50100 OpportunityCountByStatus\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Opportunity; Opportunity)\n {\n column(Status; Status) { }\n column(OpportunityCount) { Method = Count; }\n }\n }\n}"} +{"instance_id": "dataquery__customer-count-by-country-1", "environment_setup_version": "28.3", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "Group all customers by their Country/Region Code and return, for each distinct code, the Country/Region Code and the number of customers that have it. Include customers whose Country/Region Code is blank as their own group. Count the Customer records (one row per distinct Country/Region Code).", "ordered": false, "gold_query": "query 50100 CustomerCountByCountry\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(CountryRegionCode; \"Country/Region Code\") { }\n column(CustomerCount) { Method = Count; }\n }\n }\n}"} +{"instance_id": "dataquery__outstanding-purchase-value-by-vendor-1", "environment_setup_version": "28.3", "created_at": "2026-07-28", "metadata": {"area": "purchase"}, "nl_prompt": "For each vendor that has at least one open purchase order line, return the vendor's number, the vendor's name, and their total outstanding amount. Use open purchase order lines (Purchase Line records whose Document Type is Order) and sum the line 'Outstanding Amount' field (net of VAT). Vendors with no open purchase order lines must not appear.", "ordered": false, "gold_query": "query 50100 OutstandingPurchaseByVendor\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Vendor; Vendor)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(PurchaseLine; \"Purchase Line\")\n {\n DataItemLink = \"Buy-from Vendor No.\" = Vendor.\"No.\";\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(TotalOutstanding; \"Outstanding Amount\") { Method = Sum; }\n }\n }\n }\n}"} +{"instance_id": "dataquery__total-posted-sales-amount-by-customer-1", "environment_setup_version": "28.3", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one posted sales invoice line, return the customer's number, the customer's name, and their total posted sales amount. Join posted sales invoice headers to their lines via Document No., group by the header's Bill-to Customer No., and sum the line 'Amount' field (net of VAT). Customers with no posted sales invoice lines must not appear.", "ordered": false, "gold_query": "query 50100 PostedSalesAmountByCustomer\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(SalesInvoiceHeader; \"Sales Invoice Header\")\n {\n DataItemLink = \"Bill-to Customer No.\" = Customer.\"No.\";\n dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n {\n DataItemLink = \"Document No.\" = SalesInvoiceHeader.\"No.\";\n column(TotalAmount; Amount) { Method = Sum; }\n }\n }\n }\n }\n}"} +{"instance_id": "dataquery__line-count-per-open-sales-order-1", "environment_setup_version": "28.3", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "For each open sales order, return the order's document number and the number of lines it has. Use Sales Line records whose Document Type is Order, group them by Document No., and count the lines per order (one row per order document number).", "ordered": false, "gold_query": "query 50100 LineCountPerOpenSalesOrder\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(SalesLine; \"Sales Line\")\n {\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(DocumentNo; \"Document No.\") { }\n column(LineCount) { Method = Count; }\n }\n }\n}"} +{"instance_id": "dataquery__total-purchased-quantity-by-item-1", "environment_setup_version": "28.3", "created_at": "2026-07-28", "metadata": {"area": "purchase"}, "nl_prompt": "Across all posted purchase invoice lines whose Type is Item, return each item's number together with the total purchased quantity (the sum of the line Quantity). Produce one row per item number.", "ordered": false, "gold_query": "query 50100 PurchasedQuantityByItem\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(PurchInvLine; \"Purch. Inv. Line\")\n {\n DataItemTableFilter = Type = const(Item);\n column(ItemNo; \"No.\") { }\n column(TotalQuantity; Quantity) { Method = Sum; }\n }\n }\n}"} diff --git a/docs/data-query.md b/docs/data-query.md new file mode 100644 index 000000000..dc8a19c3a --- /dev/null +++ b/docs/data-query.md @@ -0,0 +1,46 @@ +--- +layout: default +title: Data Query - BC-Bench +--- + +# Data Query + +This category benchmarks an agent's ability to **generate Business Central AL queries** from a natural-language data question — an offline query-generation benchmark. There is **no MCP server and no live server in the loop**: the agent writes an AL query, and the query is evaluated deterministically. + +Given a question, the agent authors a single AL `query` object and writes it to `query.al`. The harness then **compiles and runs both the generated query and a gold reference query** against a fixed dataset (the BC container's Contoso demo data) and compares the result sets. + +## How it is scored + +Data Query is **execution-based** (like bug-fix), with no LLM judge: + +- **build** — the generated query compiled and ran. +- **resolved** (the headline `ResolutionRate`) — the generated query's result set **matches the gold query's**. Rows are compared by value (numbers normalized, column names/order ignored); row order is ignored unless the entry marks the question as `ordered`. + +To execute a query, the harness wraps it as an API query (injecting `APIPublisher`/`APIGroup`/`APIVersion`/`EntitySetName`), publishes a throwaway app to the container, and reads the query's OData endpoint. This runs on the `GitHub-BCBench` self-hosted runner (`requires_container = True`). + +> This complements the AI Test Toolkit evals in the BC platform repo: those test the **MCP server** end-to-end, while BC-Bench benchmarks **models/agents** on query generation. + +## Dataset + +Each entry has an `nl_prompt` (the question) and a `gold_query` (the reference AL query whose result set defines "correct"), plus `environment_setup_version` (the BC artifact) and `ordered`. See `dataset/dataquery.jsonl`. + +## Running it (no local containers) + +Trigger it from the GitHub **Actions** tab — the self-hosted `GitHub-BCBench` runner provisions the BC container for you (a stock **sandbox artifact with Cronus/Contoso demo data** — no special build is needed, since the query is just compiled and run): + +1. Actions → **Evaluation with GitHub Copilot** (or **Evaluation with Claude Code**) → **Run workflow**. +2. Set **category** = `data-query`, pick a **model**, leave **test-run** = `true` for a quick 2-entry run. +3. The run: provisions the container → the agent writes `query.al` → the harness compiles + runs the generated and gold queries → compares result sets → `summarize-results` reports `ResolutionRate` / `BuildRate`. + +`data-query` sets `requires_container = True`; its container setup **skips the repo clone** (there is no repo — the agent generates from scratch) and just stands up the sandbox container. + +### Local (optional) + +```bash +uv run bcbench evaluate copilot dataquery__outstanding-sales-value-by-customer-1 \ + --category data-query --container-name --username admin --password +``` + +The optional `al-mcp` / `al-lsp` levers give the agent AL compiler/language-server feedback while it authors the query. + +[← Back to Home](index.md) diff --git a/docs/index.md b/docs/index.md index a34539ef0..d49dfc990 100644 --- a/docs/index.md +++ b/docs/index.md @@ -12,6 +12,7 @@ A benchmark for evaluating AI coding agents on real-world **Business Central (AL | [Bug Fixing](bug-fix.md) | Follows [SWE-Bench](https://www.swebench.com/) methodology to evaluate bug fixing in AL code | | [Test Generation](test-generation.md) | "Reverses" SWE-Bench: Generates reproduction tests (TDD) instead of fixes | | [Code Review](code-review.md) | Reviews AL pull requests; scored with Precision / Recall / F1 against gold findings | +| [Data Query](data-query.md) | Generates AL queries from natural-language data questions; scored deterministically by running the query and comparing its result set to a gold query | ## Diagnostics diff --git a/scripts/BCBenchUtils.psm1 b/scripts/BCBenchUtils.psm1 index e3baf0b2d..9b0f23436 100644 --- a/scripts/BCBenchUtils.psm1 +++ b/scripts/BCBenchUtils.psm1 @@ -490,7 +490,7 @@ function Get-BCBenchDatasetPath { param( [Parameter(Mandatory = $true)] # Category validation lives only here: every caller resolves the dataset path through this function, so there's no need to duplicate ValidateSet on each caller. - [ValidateSet("bug-fix", "test-generation", "code-review", "nl2al", "extensibility-request-implement", "extensibility-request-triage")] + [ValidateSet("bug-fix", "test-generation", "code-review", "nl2al", "data-query", "extensibility-request-implement", "extensibility-request-triage")] [string] $Category ) @@ -499,6 +499,7 @@ function Get-BCBenchDatasetPath { "test-generation" { $DatasetName = "bcbench.jsonl" } "code-review" { $DatasetName = "codereview.jsonl" } "nl2al" { $DatasetName = "nl2al.jsonl" } + "data-query" { $DatasetName = "dataquery.jsonl" } "extensibility-request-implement" { $DatasetName = "extensibility_request_implement.jsonl" } "extensibility-request-triage" { $DatasetName = "extensibility_request_triage.jsonl" } } diff --git a/scripts/Setup-ContainerAndRepository.ps1 b/scripts/Setup-ContainerAndRepository.ps1 index b6efd0179..efeffd1f3 100644 --- a/scripts/Setup-ContainerAndRepository.ps1 +++ b/scripts/Setup-ContainerAndRepository.ps1 @@ -72,7 +72,10 @@ if (-not $SkipRepo) { Invoke-GitCloneWithRetry -RepoUrl $cloneInfo.Url -Token $cloneInfo.Token -ClonePath $RepoPath -CommitSha $commitSha -SparseCheckoutPaths $cloneInfo.SparseCheckoutPaths } else { - Write-Log "Skipping repository clone (SkipRepo flag set)" -Level Info + # Categories that scaffold their own workspace still need the folder to exist: it is shared into + # the container below, and Compile-AppInBcContainer throws for any path not shared with it. + Write-Log "Skipping repository clone (SkipRepo flag set); creating empty workspace at $RepoPath" -Level Info + New-Item -ItemType Directory -Path $RepoPath -Force | Out-Null } if (-not $SkipContainer) { diff --git a/src/bcbench/agent/claude/agent.py b/src/bcbench/agent/claude/agent.py index 7c8d48671..f568f5593 100644 --- a/src/bcbench/agent/claude/agent.py +++ b/src/bcbench/agent/claude/agent.py @@ -26,6 +26,7 @@ def run_claude_code( output_dir: Path, al_mcp: bool = False, al_lsp: bool = False, + skills: bool = False, container_name: str = "bcbench", ) -> tuple[AgentMetrics | None, ExperimentConfiguration]: """Run Claude Code on a single dataset entry. @@ -46,7 +47,7 @@ def run_claude_code( mcp_config_json, mcp_server_names = build_mcp_config(claude_config, entry, repo_path, al_mcp=al_mcp, container_name=container_name) lsp_plugin_dir: Path | None = build_al_lsp_plugin(entry, category, repo_path, AgentHarness.CLAUDE, al_lsp=al_lsp, container_name=container_name) instructions_enabled: bool = setup_instructions_from_config(claude_config, entry, repo_path, harness=AgentHarness.CLAUDE) - skills_enabled: bool = setup_agent_skills(claude_config, entry, repo_path, harness=AgentHarness.CLAUDE) + skills_enabled: bool = setup_agent_skills(claude_config, entry, repo_path, harness=AgentHarness.CLAUDE, skills_enabled_override=skills) custom_agent: str | None = setup_custom_agent(claude_config, entry, repo_path, harness=AgentHarness.CLAUDE) tool_log_path: Path = setup_hooks(repo_path, AgentHarness.CLAUDE, output_dir) plugins: list[tuple[PluginConfig, Path]] = resolve_config_plugins(claude_config, allow_copilot_manifest=False) diff --git a/src/bcbench/agent/copilot/agent.py b/src/bcbench/agent/copilot/agent.py index 7f028f8bd..2a350f189 100644 --- a/src/bcbench/agent/copilot/agent.py +++ b/src/bcbench/agent/copilot/agent.py @@ -29,6 +29,7 @@ def run_copilot_agent( output_dir: Path, al_mcp: bool = False, al_lsp: bool = False, + skills: bool = False, container_name: str = "bcbench", ) -> tuple[AgentMetrics | None, ExperimentConfiguration]: """Run GitHub Copilot CLI agent on a single dataset entry. @@ -51,7 +52,7 @@ def run_copilot_agent( mcp_config_json, mcp_server_names = build_mcp_config(copilot_config, entry, repo_path, al_mcp=al_mcp, container_name=container_name) lsp_plugin_dir: Path | None = build_al_lsp_plugin(entry, category, repo_path, AgentHarness.COPILOT, al_lsp=al_lsp, container_name=container_name) instructions_enabled: bool = setup_instructions_from_config(copilot_config, entry, repo_path, harness=AgentHarness.COPILOT) - skills_enabled: bool = setup_agent_skills(copilot_config, entry, repo_path, harness=AgentHarness.COPILOT) + skills_enabled: bool = setup_agent_skills(copilot_config, entry, repo_path, harness=AgentHarness.COPILOT, skills_enabled_override=skills) custom_agent: str | None = setup_custom_agent(copilot_config, entry, repo_path, harness=AgentHarness.COPILOT) tool_log_path: Path = setup_hooks(repo_path, AgentHarness.COPILOT, output_dir) plugins: list[tuple[PluginConfig, Path]] = resolve_config_plugins(copilot_config, allow_copilot_manifest=True) diff --git a/src/bcbench/agent/shared/config.yaml b/src/bcbench/agent/shared/config.yaml index 54e3a0495..ae4b210e2 100644 --- a/src/bcbench/agent/shared/config.yaml +++ b/src/bcbench/agent/shared/config.yaml @@ -93,6 +93,17 @@ prompt: If there are no findings, write an empty array. Write only valid JSON to review.json, with no surrounding markdown or commentary. + data-query-template: | + Answer the following Business Central data question by writing a single AL `query` object + to a file named `query.al` in {{repo_path}}. Write only the query logic — do not add API + properties (QueryType, APIPublisher, APIGroup, APIVersion, EntityName, EntitySetName); the + evaluation harness adds those. Proceed without asking for confirmation. + + Question: + {{task}} + + You MUST write query.al before finishing; if you do not, there is no output to evaluate. + # controls: # 1. whether to copy custom instructions from `src/bcbench/agent/shared/instructions//` # - Copilot: copies to repo/.github/ and renames AGENTS.md to copilot-instructions.md diff --git a/src/bcbench/agent/shared/instructions/dataquery/skills/al-query-authoring/SKILL.md b/src/bcbench/agent/shared/instructions/dataquery/skills/al-query-authoring/SKILL.md new file mode 100644 index 000000000..ef771e4da --- /dev/null +++ b/src/bcbench/agent/shared/instructions/dataquery/skills/al-query-authoring/SKILL.md @@ -0,0 +1,53 @@ +--- +name: al-query-authoring +description: Guide for authoring Business Central AL query objects that answer data questions (joins, aggregates, filters, sorting). Use this when asked to write an AL query that returns Business Central data such as customers, vendors, items, sales, purchases, projects, or opportunities. +--- + +Write a single, compilable AL `query` object that returns exactly the data needed to answer +the question. Reference real Business Central tables and fields — a query that does not +compile, or returns the wrong data, fails. + +## Structure + +```al +query 50100 TopCustomersBySales +{ + QueryType = Normal; + + elements + { + dataitem(Customer; Customer) + { + column(No; "No.") { } + column(Name; Name) { } + dataitem(SalesLine; "Sales Line") + { + DataItemLink = "Sell-to Customer No." = Customer."No."; + DataItemTableFilter = "Document Type" = const(Order); + column(OutstandingAmount; "Outstanding Amount") { Method = Sum; } + } + } + } +} +``` + +## Rules of thumb + +- **Aggregate** a column with a method: `column(Total; "Amount (LCY)") { Method = Sum; }` + (also `Average`, `Min`, `Max`) — these take the field to aggregate. **`Count` takes no source + field** — write `column(RowCount) { Method = Count; }`, not `column(RowCount; "No.") { ... }`. + Non-aggregated columns become the GROUP BY. +- **Join** by nesting a `dataitem` and linking it: `DataItemLink = "" = Parent."";`. +- **Filter** rows with `DataItemTableFilter = "" = const();` (e.g. an Option + like `Document Type`) or a range/expression. +- **Order** with the `OrderBy` property: `OrderBy = descending();` (or `ascending`) when + the question asks for ranking or "top N" (combine with `TopNumberOfRows` where appropriate). +- **Quote** any field or table name that contains spaces or special characters: `"No."`, + `"Sales Line"`, `"Amount (LCY)"`. +- Prefer stored fields; FlowFields and Option fields are supported. + +## Common pitfalls + +- Don't invent table or field names — use the real Business Central schema. +- Return only the columns the question needs; extra or missing columns change the result set. +- One `query` object per file. diff --git a/src/bcbench/commands/evaluate.py b/src/bcbench/commands/evaluate.py index a8b59ad37..24a5eaa21 100644 --- a/src/bcbench/commands/evaluate.py +++ b/src/bcbench/commands/evaluate.py @@ -53,6 +53,7 @@ def evaluate_copilot( run_id: RunId = "copilot_test_run", al_mcp: Annotated[bool, typer.Option("--al-mcp", help="Enable AL MCP server")] = False, al_lsp: Annotated[bool, typer.Option("--al-lsp", help="Enable AL LSP server")] = False, + skills: Annotated[bool, typer.Option("--skills", help="Enable agent skills for the run")] = False, ) -> None: """ Evaluate GitHub Copilot CLI on single dataset entry. @@ -87,6 +88,7 @@ def evaluate_copilot( output_dir=ctx.result_dir, al_mcp=al_mcp if ctx.container else False, al_lsp=al_lsp, + skills=skills, container_name=ctx.get_container().name if ctx.container else "", ), ) @@ -108,6 +110,7 @@ def evaluate_claude_code( run_id: RunId = "claude_code_test_run", al_mcp: Annotated[bool, typer.Option("--al-mcp", help="Enable AL MCP server")] = False, al_lsp: Annotated[bool, typer.Option("--al-lsp", help="Enable AL LSP server")] = False, + skills: Annotated[bool, typer.Option("--skills", help="Enable agent skills for the run")] = False, ) -> None: """ Evaluate Claude Code on single dataset entry. @@ -142,6 +145,7 @@ def evaluate_claude_code( output_dir=ctx.result_dir, al_mcp=al_mcp if ctx.container else False, al_lsp=al_lsp, + skills=skills, container_name=ctx.get_container().name if ctx.container else "", ), ) @@ -307,7 +311,7 @@ def evaluate(self, context: EvaluationContext[BaseDatasetEntry]) -> None: logger.info("Mock pipeline: Generating random evaluation result") match context.category: - case EvaluationCategory.BUG_FIX | EvaluationCategory.TEST_GENERATION: + case EvaluationCategory.BUG_FIX | EvaluationCategory.TEST_GENERATION | EvaluationCategory.DATA_QUERY: scenarios = ["success", "build-fail"] case EvaluationCategory.CODE_REVIEW: scenarios = ["invalid", "valid"] diff --git a/src/bcbench/commands/run.py b/src/bcbench/commands/run.py index cb8e79acc..1bd924029 100644 --- a/src/bcbench/commands/run.py +++ b/src/bcbench/commands/run.py @@ -36,6 +36,7 @@ def run_copilot( output_dir: OutputDir = _config.paths.evaluation_results_path, al_mcp: Annotated[bool, typer.Option("--al-mcp", help="Enable AL MCP server")] = False, al_lsp: Annotated[bool, typer.Option("--al-lsp", help="Enable AL LSP server")] = False, + skills: Annotated[bool, typer.Option("--skills", help="Enable agent skills for the run")] = False, ) -> None: """ Run GitHub Copilot CLI on a single entry to generate a patch (without building/testing). @@ -56,6 +57,7 @@ def run_copilot( output_dir=output_dir, al_mcp=al_mcp if container_name else False, al_lsp=al_lsp, + skills=skills, container_name=container_name, ) @@ -70,6 +72,7 @@ def run_claude( output_dir: OutputDir = _config.paths.evaluation_results_path, al_mcp: Annotated[bool, typer.Option("--al-mcp", help="Enable AL MCP server")] = False, al_lsp: Annotated[bool, typer.Option("--al-lsp", help="Enable AL LSP server")] = False, + skills: Annotated[bool, typer.Option("--skills", help="Enable agent skills for the run")] = False, ) -> None: """ Run Claude Code on a single entry to generate a patch (without building/testing). @@ -90,6 +93,7 @@ def run_claude( output_dir=output_dir, al_mcp=al_mcp if container_name else False, al_lsp=al_lsp, + skills=skills, container_name=container_name, ) diff --git a/src/bcbench/dataset/__init__.py b/src/bcbench/dataset/__init__.py index f178a39b3..846d59131 100644 --- a/src/bcbench/dataset/__init__.py +++ b/src/bcbench/dataset/__init__.py @@ -1,13 +1,14 @@ """Dataset module for querying, validating and analyze dataset entries.""" from bcbench.dataset.codereview import CodeReviewEntry, ReviewComment, Severity -from bcbench.dataset.dataset_entry import BaseDatasetEntry, BugFixEntry, NL2ALEntry, RepoGroundedEntry, TestEntry, TestGenEntry +from bcbench.dataset.dataset_entry import BaseDatasetEntry, BugFixEntry, DataQueryEntry, NL2ALEntry, RepoGroundedEntry, TestEntry, TestGenEntry from bcbench.dataset.extensibility_request import ExtRequestImplementEntry, ExtRequestTriageEntry, ManagedLabel __all__ = [ "BaseDatasetEntry", "BugFixEntry", "CodeReviewEntry", + "DataQueryEntry", "ExtRequestImplementEntry", "ExtRequestTriageEntry", "ManagedLabel", diff --git a/src/bcbench/dataset/dataset_entry.py b/src/bcbench/dataset/dataset_entry.py index 3c87b6fe5..606d510fb 100644 --- a/src/bcbench/dataset/dataset_entry.py +++ b/src/bcbench/dataset/dataset_entry.py @@ -14,7 +14,7 @@ _config = get_config() -__all__ = ["BaseDatasetEntry", "BugFixEntry", "NL2ALEntry", "RepoGroundedEntry", "TestEntry", "TestGenEntry"] +__all__ = ["BaseDatasetEntry", "BugFixEntry", "DataQueryEntry", "NL2ALEntry", "RepoGroundedEntry", "TestEntry", "TestGenEntry"] class TestEntry(BaseModel): @@ -192,3 +192,28 @@ def get_task(self) -> str: def get_expected_output(self) -> Checklist: return {"assertions": self.expected} + + +class DataQueryEntry(BaseDatasetEntry): + """Dataset entry for the data-query category — generate an AL query that answers a data question. + + Execution-based: the agent authors an AL query; evaluation compiles + runs both the generated + query and the gold query against a fixed dataset (Contoso in a BC container) and compares the + result sets. The workspace is scaffolded by the pipeline, so there is no repo or commit. + """ + + nl_prompt: Annotated[str, Field(min_length=1, pattern=r"^[^\x00]*$")] + gold_query: Annotated[str, Field(min_length=1, pattern=r"^[^\x00]*$")] + # Whether row order is significant when comparing result sets (e.g. the question asks for a + # specific ranking). Defaults to False: result sets are compared order-insensitively. + ordered: bool = False + + @property + def customization_profile(self) -> str: + return "dataquery" + + def get_task(self) -> str: + return self.nl_prompt + + def get_expected_output(self) -> str: + return self.gold_query diff --git a/src/bcbench/evaluate/__init__.py b/src/bcbench/evaluate/__init__.py index 458f2cfe5..f4be463bb 100644 --- a/src/bcbench/evaluate/__init__.py +++ b/src/bcbench/evaluate/__init__.py @@ -3,9 +3,19 @@ from bcbench.evaluate.base import EvaluationPipeline from bcbench.evaluate.bugfix import BugFixPipeline from bcbench.evaluate.codereview import CodeReviewPipeline +from bcbench.evaluate.dataquery import DataQueryPipeline from bcbench.evaluate.ext_request_implement import ExtRequestImplementPipeline from bcbench.evaluate.ext_request_triage import ExtRequestTriagePipeline from bcbench.evaluate.nl2al import NL2ALPipeline from bcbench.evaluate.testgeneration import TestGenerationPipeline -__all__ = ["BugFixPipeline", "CodeReviewPipeline", "EvaluationPipeline", "ExtRequestImplementPipeline", "ExtRequestTriagePipeline", "NL2ALPipeline", "TestGenerationPipeline"] +__all__ = [ + "BugFixPipeline", + "CodeReviewPipeline", + "DataQueryPipeline", + "EvaluationPipeline", + "ExtRequestImplementPipeline", + "ExtRequestTriagePipeline", + "NL2ALPipeline", + "TestGenerationPipeline", +] diff --git a/src/bcbench/evaluate/dataquery.py b/src/bcbench/evaluate/dataquery.py new file mode 100644 index 000000000..8509c30d9 --- /dev/null +++ b/src/bcbench/evaluate/dataquery.py @@ -0,0 +1,107 @@ +from collections.abc import Callable, Mapping, Sequence +from decimal import Decimal, InvalidOperation +from pathlib import Path + +from bcbench.dataset import DataQueryEntry +from bcbench.evaluate.base import EvaluationPipeline +from bcbench.exceptions import BuildError, BuildTimeoutExpired +from bcbench.github_actions import github_log_group +from bcbench.logger import get_logger +from bcbench.operations import clear_directory +from bcbench.results.base import ExecutionBasedEvaluationResult +from bcbench.types import EvaluationContext + +logger = get_logger(__name__) + +__all__ = ["DataQueryPipeline", "result_sets_match"] + +GENERATED_QUERY_FILE = "query.al" + + +def _normalize_value(value: object) -> str: + if value is None: + return "" + if isinstance(value, bool): + return str(value).lower() + if isinstance(value, (int, float, Decimal)): + # Only values that arrived as numeric JSON types are canonicalized: scale/trailing-zero- + # insensitive (500 == 500.0) with full precision preserved (1.00001 != 1.00002) and no float + # rounding (Decimal built from the value's string form). Both gold and generated rows come + # through the same OData->JSON pipeline, so amounts are numbers on both sides. + try: + return str(Decimal(str(value)).normalize()) + except (InvalidOperation, ValueError): + return str(value) + # Strings (and anything else) are preserved verbatim apart from a whitespace trim. Business Central + # Code/No. fields are JSON strings even when digit-only, so "001" must NOT collapse to "1" — coercing + # them through Decimal would let a wrong result be scored as matching the gold. + return str(value).strip() + + +def _normalize_rows(rows: Sequence[Mapping[str, object]], ordered: bool) -> list[tuple[str, ...]]: + # Compare on values only: drop OData/system metadata keys ('@'-prefixed) and ignore column + # names/order so a correct query still matches the gold even if it names columns differently. + normalized = [tuple(sorted(_normalize_value(v) for k, v in row.items() if not k.startswith("@"))) for row in rows] + return normalized if ordered else sorted(normalized) + + +def result_sets_match(generated: Sequence[Mapping[str, object]], gold: Sequence[Mapping[str, object]], ordered: bool = False) -> bool: + """Compare two query result sets for equality. + + Values are compared (numbers normalized, column names/order ignored); row order is ignored + unless ``ordered`` is True (the question asks for a specific ranking). + """ + return _normalize_rows(generated, ordered) == _normalize_rows(gold, ordered) + + +class DataQueryPipeline(EvaluationPipeline[DataQueryEntry]): + """Pipeline for the data-query category — generate an AL query, evaluate deterministically. + + The agent writes an AL query to ``query.al``. Evaluation compiles + runs both the generated + query and the entry's gold query against the container's fixed (Contoso) dataset and compares + the result sets: build = the generated query compiled and ran; resolved = its result set + matches the gold query's. + """ + + def setup_workspace(self, entry: DataQueryEntry, repo_path: Path) -> None: + # The workspace is shared into the running container, so its contents are cleared in place. + clear_directory(repo_path) + + def setup(self, context: EvaluationContext[DataQueryEntry]) -> None: + self.setup_workspace(context.entry, context.repo_path) + + def run_agent(self, context: EvaluationContext[DataQueryEntry], agent_runner: Callable) -> None: + with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"): + context.metrics, context.experiment = agent_runner(context) + + def evaluate(self, context: EvaluationContext[DataQueryEntry]) -> None: + from bcbench.operations import execute_al_query + + query_file = context.repo_path / GENERATED_QUERY_FILE + generated_query = query_file.read_text(encoding="utf-8").strip() if query_file.exists() else "" + + if not generated_query: + logger.warning(f"Agent produced no {GENERATED_QUERY_FILE} for {context.entry.instance_id}") + self.save_result(context, ExecutionBasedEvaluationResult.create_build_failure(context, output="", error_message=f"No {GENERATED_QUERY_FILE} produced")) + return + + container = context.get_container() + version = context.entry.environment_setup_version + + # Validate the gold query first, and deliberately do NOT catch its failure: a gold that doesn't + # compile/run is a harness or dataset bug, not the agent's fault, so it must fail the run loudly + # and get fixed rather than being silently scored or excluded. + gold_rows = execute_al_query(context.entry.gold_query, container, version, context.repo_path, "gold") + + try: + generated_rows = execute_al_query(generated_query, container, version, context.repo_path, "generated") + except (BuildError, BuildTimeoutExpired) as e: + logger.exception(f"Generated query failed to compile/run for {context.entry.instance_id}") + self.save_result(context, ExecutionBasedEvaluationResult.create_build_failure(context, output=generated_query, error_message=str(e))) + return + + resolved = result_sets_match(generated_rows, gold_rows, context.entry.ordered) + error_message = None if resolved else f"Result set mismatch: generated {len(generated_rows)} rows vs gold {len(gold_rows)} rows" + result = ExecutionBasedEvaluationResult.create_result(context, output=generated_query, build=True, resolved=resolved, error_message=error_message) + logger.info(f"{context.entry.instance_id}: build=True resolved={resolved}") + self.save_result(context, result) diff --git a/src/bcbench/operations/__init__.py b/src/bcbench/operations/__init__.py index 3469ab806..50b961115 100644 --- a/src/bcbench/operations/__init__.py +++ b/src/bcbench/operations/__init__.py @@ -6,10 +6,12 @@ build_ps_dataset_tests_script, build_ps_test_script, copy_symbol_apps, + execute_al_query, resolve_artifact_version_root, run_tests, + wrap_query_as_api, ) -from bcbench.operations.filesystem_operations import remove_tree +from bcbench.operations.filesystem_operations import clear_directory, remove_tree from bcbench.operations.git_operations import ( apply_patch, checkout_commit, @@ -38,10 +40,12 @@ "checkout_commit", "clean_project_paths", "clean_repo", + "clear_directory", "clone_repo_at_revision", "commit_changes", "copy_problem_statement_folder", "copy_symbol_apps", + "execute_al_query", "extract_tests_from_patch", "fetch_commit_if_missing", "remove_tree", @@ -54,4 +58,5 @@ "setup_instructions_from_config", "setup_repo_prebuild", "stage_and_get_diff", + "wrap_query_as_api", ] diff --git a/src/bcbench/operations/bc_operations.py b/src/bcbench/operations/bc_operations.py index 88ea254d2..b9a047513 100644 --- a/src/bcbench/operations/bc_operations.py +++ b/src/bcbench/operations/bc_operations.py @@ -13,6 +13,8 @@ from bcbench.dataset.dataset_entry import _BugFixTestGenBase from bcbench.exceptions import BuildError, BuildTimeoutExpired, TestExecutionError, TestExecutionTimeoutExpired from bcbench.logger import get_logger +from bcbench.operations.filesystem_operations import remove_tree +from bcbench.operations.setup_operations import bootstrap_app_json from bcbench.types import ContainerConfig logger = get_logger(__name__) @@ -235,3 +237,182 @@ def run_test_suite(test_entries: list[TestEntry], expectation: Literal["Pass", " except subprocess.TimeoutExpired: logger.exception(f"Test execution timed out after {_config.timeout.test_execution} seconds") raise TestExecutionTimeoutExpired(test_entries_json, _config.timeout.test_execution) from None + + +# --- data-query category: compile + run an AL query and capture its rows via a wrapped API query --- + +# API metadata injected into a generated/gold query so it is exposed over OData and can be fetched. +_QUERY_API_PUBLISHER = "bcbench" +_QUERY_API_GROUP = "eval" +_QUERY_API_VERSION = "v1.0" + + +def _safe_object_name(object_id: int) -> str: + """A short, unique, always-valid query object name. + + The object name is irrelevant to a query's result set (we score by comparing data, not + identifiers), but AL requires it to be a valid identifier of <=30 characters and unique in + the tenant. Normalizing it keeps the benchmark focused on query logic instead of failing an + otherwise-correct query just because the agent chose a long/descriptive name (AL0305). + """ + return f"BCBenchQuery{object_id}" + + +def _entity_set_name(object_id: int) -> str: + """Per-object OData entity set so the generated and gold API queries don't collide on route.""" + return f"bcbenchResults{object_id}" + + +def _entity_name(object_id: int) -> str: + return f"bcbenchResult{object_id}" + + +def _query_api_properties(object_id: int) -> str: + return ( + "QueryType = API;\n" + f" APIPublisher = '{_QUERY_API_PUBLISHER}';\n" + f" APIGroup = '{_QUERY_API_GROUP}';\n" + f" APIVersion = '{_QUERY_API_VERSION}';\n" + f" EntityName = '{_entity_name(object_id)}';\n" + f" EntitySetName = '{_entity_set_name(object_id)}';" + ) + + +def wrap_query_as_api(query_text: str, object_id: int) -> str: + """Turn a plain AL query object into an API query the harness can fetch over OData. + + Reassigns the object id and normalizes the object name (so generated and gold apps don't + collide and long names don't cause AL0305), drops any existing ``QueryType`` line, and + injects the API properties right after the object's opening brace. Pure string transform so + it can be unit-tested without a container. + """ + import re + + safe_name = _safe_object_name(object_id) + # AL keywords are case-insensitive; match `query`/`QueryType` in any casing. + text, replaced = re.subn( + r'(\bquery\s+)\d+\s+("(?:[^"\\]|\\.)*"|\w+)', + rf"\g<1>{object_id} {safe_name}", + query_text, + count=1, + flags=re.IGNORECASE, + ) + if replaced == 0: + raise BuildError("query-wrap", f"No AL query object declaration found in generated output:\n{query_text}") + + text = re.sub(r"\bQueryType\s*=\s*\w+\s*;", "", text, count=1, flags=re.IGNORECASE) + + brace_index = text.find("{") + if brace_index == -1: + raise BuildError("query-wrap", f"Generated query has no object body ('{{' not found):\n{query_text}") + return f"{text[: brace_index + 1]}\n {_query_api_properties(object_id)}\n{text[brace_index + 1 :]}" + + +_QUERY_RUN_TEMPLATE = Template( + """ +Import-Module BcContainerHelper -Force -DisableNameChecking +Import-Module '$app_utils_path' -Force +$$ErrorActionPreference = 'Stop' + +$$password = ConvertTo-SecureString '$password' -AsPlainText -Force +$$credential = New-Object System.Management.Automation.PSCredential('$username', $$password) + +# Remove any app left installed by a previous run of the same suffix so re-running against the +# same container doesn't fail with an object-ID conflict on the fixed 50100/50101 range. +UnInstall-BcContainerApp -containerName '$container_name' -name '$app_name' -publisher '$app_publisher' -force -doNotSaveData -ErrorAction SilentlyContinue +UnPublish-BcContainerApp -containerName '$container_name' -name '$app_name' -publisher '$app_publisher' -ErrorAction SilentlyContinue + +# Compile + publish the wrapped API query with the same proven helper the other categories use +# (clears/sets an explicit .alpackages symbol folder, GenerateReportLayout=No, ForceSync, +# dependencyPublishingOption=ignore) so Base Application symbols resolve reliably. +Invoke-AppBuildAndPublish -containerName '$container_name' -appProjectFolder '$app_dir' -credential $$credential -skipVerification -useDevEndpoint + +try { + # Read the query's rows over the OData/API endpoint from *inside* the container, so we don't depend + # on host->container name resolution or published ports (the runner does not update its hosts file). + # Basic auth header is built by hand rather than via -Credential: PowerShell 7 (used inside the + # container) refuses -Credential over plain HTTP, and a manual header works on both 5.1 and 7. + $$json = Invoke-ScriptInBcContainer -containerName '$container_name' -argumentList $$credential, '$publisher', '$group', '$version', '$entity_set' -scriptblock { + param($$cred, $$pub, $$grp, $$ver, $$eset) + $$pair = "$$($$cred.UserName):$$($$cred.GetNetworkCredential().Password)" + $$headers = @{ Authorization = 'Basic ' + [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes($$pair)) } + $$base = 'http://localhost:7048/BC/api' + $$companyId = (Invoke-RestMethod -Uri "$$base/v2.0/companies" -Headers $$headers).value[0].id + # Follow @odata.nextLink so large result sets aren't silently truncated to the first page. + $$rows = [System.Collections.Generic.List[object]]::new() + $$uri = "$$base/$$pub/$$grp/$$ver/companies($$companyId)/$$eset" + while ($$uri) { + $$page = Invoke-RestMethod -Uri $$uri -Headers $$headers + if ($$null -ne $$page.value) { foreach ($$row in $$page.value) { $$rows.Add($$row) } } + $$uri = $$page.'@odata.nextLink' + } + $$rows | ConvertTo-Json -Depth 10 -Compress + } + $$json | Out-File -FilePath '$result_file' -Encoding utf8 +} +finally { + # Best-effort teardown so the container doesn't accumulate throwaway apps between runs. + UnInstall-BcContainerApp -containerName '$container_name' -name '$app_name' -publisher '$app_publisher' -force -doNotSaveData -ErrorAction SilentlyContinue + UnPublish-BcContainerApp -containerName '$container_name' -name '$app_name' -publisher '$app_publisher' -ErrorAction SilentlyContinue +} +""".strip() +) + + +def execute_al_query(query_text: str, container: ContainerConfig, version: str, work_root: Path, suffix: str) -> list[dict]: + """Compile + publish an AL query (wrapped as an API query) to the container and return its rows. + + Builds a throwaway app under ``work_root/.bcbench-query-``, compiles + publishes it, + then reads the query's OData endpoint. Raises :class:`BuildError` if the query does not + compile or publish. + + NOTE: the container-side steps (compile/publish/OData fetch) require a running BC container + and have not been validated locally; the wrapping and comparison logic are unit-tested. + """ + import json + + object_id = 50100 if suffix == "generated" else 50101 + app_dir = work_root / f".bcbench-query-{suffix}" + if app_dir.exists(): + remove_tree(app_dir) + + app_name = f"BC-Bench Query {suffix}" + app_publisher = "BC-Bench" + bootstrap_app_json(app_dir, app_name, version, id_range=(object_id, object_id), publisher=app_publisher) + (app_dir / "query.al").write_text(wrap_query_as_api(query_text, object_id), encoding="utf-8") + # Symbols are downloaded into an explicit .alpackages folder by Invoke-AppBuildAndPublish (below). + + result_file = app_dir / "result.json" + app_utils_path = _config.paths.ps_script_path / "AppUtils.psm1" + ps_script = _QUERY_RUN_TEMPLATE.substitute( + app_utils_path=_escape_ps_string(str(app_utils_path)), + container_name=_escape_ps_string(container.name), + username=_escape_ps_string(container.username), + password=_escape_ps_string(container.password), + app_dir=_escape_ps_string(str(app_dir)), + app_name=_escape_ps_string(app_name), + app_publisher=_escape_ps_string(app_publisher), + publisher=_QUERY_API_PUBLISHER, + group=_QUERY_API_GROUP, + version=_QUERY_API_VERSION, + entity_set=_entity_set_name(object_id), + result_file=_escape_ps_string(str(result_file)), + ) + + try: + subprocess.run( + ["pwsh", "-NoProfile", "-NonInteractive", "-Command", ps_script], + cwd=work_root, + capture_output=True, + check=True, + text=True, + timeout=_config.timeout.build_app, + ) + except subprocess.CalledProcessError as e: + logger.debug(f"Query compile/publish/fetch failed ({suffix}): {e.stdout}\n{e.stderr}") + raise BuildError(f"query-{suffix}", (e.stdout or "") + (e.stderr or "")) from None + except subprocess.TimeoutExpired: + raise BuildTimeoutExpired(f"query-{suffix}", _config.timeout.build_app) from None + + rows = json.loads(result_file.read_text(encoding="utf-8-sig") or "[]") + return rows if isinstance(rows, list) else [rows] diff --git a/src/bcbench/operations/filesystem_operations.py b/src/bcbench/operations/filesystem_operations.py index 3a3b35fa1..4b745aad7 100644 --- a/src/bcbench/operations/filesystem_operations.py +++ b/src/bcbench/operations/filesystem_operations.py @@ -18,3 +18,18 @@ def remove_tree(path: Path) -> None: Use when `shutil.rmtree` fails due to read-only files, which usually occur in secondary runs on Windows. """ shutil.rmtree(path, onexc=_force_remove_readonly) + + +def clear_directory(path: Path) -> None: + """ + Remove everything inside a directory, leaving the directory itself in place. + + Use instead of `remove_tree` when the directory should survive. + """ + path.mkdir(parents=True, exist_ok=True) + for child in path.iterdir(): + if child.is_dir(): + remove_tree(child) + else: + child.chmod(stat.S_IWRITE) + child.unlink() diff --git a/src/bcbench/operations/skills_operations.py b/src/bcbench/operations/skills_operations.py index ba5fe9b34..b3bda008e 100644 --- a/src/bcbench/operations/skills_operations.py +++ b/src/bcbench/operations/skills_operations.py @@ -9,14 +9,24 @@ logger = get_logger(__name__) -def setup_agent_skills(agent_config: dict, entry: BaseDatasetEntry, repo_path: Path, harness: AgentHarness) -> bool: +def setup_agent_skills( + agent_config: dict, + entry: BaseDatasetEntry, + repo_path: Path, + harness: AgentHarness, + skills_enabled_override: bool | None = None, +) -> bool: """ Setup skills in the repository if available. + Args: + skills_enabled_override: When not None, takes precedence over ``config.yaml``'s + ``skills.enabled`` (used to toggle skills per run via the ``--skills`` CLI flag). + Returns: True if skills were copied, False if skills are disabled. """ - skills_enabled: bool = agent_config["skills"]["enabled"] + skills_enabled: bool = agent_config["skills"]["enabled"] if skills_enabled_override is None else skills_enabled_override if skills_enabled: source_skills: Path = _get_source_instructions_path(entry.customization_profile) diff --git a/src/bcbench/results/base.py b/src/bcbench/results/base.py index 0106efe3b..b6e5a2a43 100644 --- a/src/bcbench/results/base.py +++ b/src/bcbench/results/base.py @@ -107,6 +107,11 @@ def create_success(cls, context: "EvaluationContext", output: str) -> Self: def create_build_failure(cls, context: "EvaluationContext", output: str, error_message: str) -> Self: return cls(**cls._base_fields(context), output=output, error_message=error_message, resolved=False, build=False) + @classmethod + def create_result(cls, context: "EvaluationContext", output: str, *, build: bool, resolved: bool, error_message: str | None = None) -> Self: + """General factory for execution outcomes, e.g. compiled+ran but produced the wrong result (build=True, resolved=False).""" + return cls(**cls._base_fields(context), output=output, build=build, resolved=resolved, error_message=error_message) + @property def status_label(self) -> str: if self.timeout: diff --git a/src/bcbench/results/summary.py b/src/bcbench/results/summary.py index 4a01b762d..dad89c875 100644 --- a/src/bcbench/results/summary.py +++ b/src/bcbench/results/summary.py @@ -166,6 +166,7 @@ def from_results(cls, results: Sequence[BaseEvaluationResult], run_id: str) -> " return summary.model_copy( update={ + "total": total, "resolved": resolved, "failed": total - resolved, "build": build, diff --git a/src/bcbench/types.py b/src/bcbench/types.py index 00b7fb7ff..4abfb4beb 100644 --- a/src/bcbench/types.py +++ b/src/bcbench/types.py @@ -225,6 +225,7 @@ class EvaluationCategory(StrEnum): TEST_GENERATION = "test-generation" CODE_REVIEW = "code-review" NL2AL = "nl2al" + DATA_QUERY = "data-query" # Implement an approved extensibility request (add an event/extension point) as an AL code change. # The sibling ext-advisor category is planned but not yet implemented. EXT_REQUEST_IMPLEMENT = "extensibility-request-implement" @@ -244,6 +245,8 @@ def dataset_path(self) -> Path: return get_config().paths.dataset_dir / "codereview.jsonl" case EvaluationCategory.NL2AL: return get_config().paths.dataset_dir / "nl2al.jsonl" + case EvaluationCategory.DATA_QUERY: + return get_config().paths.dataset_dir / "dataquery.jsonl" case EvaluationCategory.EXT_REQUEST_IMPLEMENT: return get_config().paths.dataset_dir / "extensibility_request_implement.jsonl" case EvaluationCategory.EXT_REQUEST_TRIAGE: @@ -253,7 +256,7 @@ def dataset_path(self) -> Path: @property def entry_class(self) -> type[BaseDatasetEntry]: - from bcbench.dataset import BugFixEntry, CodeReviewEntry, ExtRequestImplementEntry, ExtRequestTriageEntry, NL2ALEntry, TestGenEntry + from bcbench.dataset import BugFixEntry, CodeReviewEntry, DataQueryEntry, ExtRequestImplementEntry, ExtRequestTriageEntry, NL2ALEntry, TestGenEntry match self: case EvaluationCategory.BUG_FIX: @@ -264,6 +267,8 @@ def entry_class(self) -> type[BaseDatasetEntry]: return CodeReviewEntry case EvaluationCategory.NL2AL: return NL2ALEntry + case EvaluationCategory.DATA_QUERY: + return DataQueryEntry case EvaluationCategory.EXT_REQUEST_IMPLEMENT: return ExtRequestImplementEntry case EvaluationCategory.EXT_REQUEST_TRIAGE: @@ -273,7 +278,7 @@ def entry_class(self) -> type[BaseDatasetEntry]: @property def result_class(self) -> type[BaseEvaluationResult]: - from bcbench.results.base import JudgeBasedEvaluationResult + from bcbench.results.base import ExecutionBasedEvaluationResult, JudgeBasedEvaluationResult from bcbench.results.bugfix import BugFixResult from bcbench.results.codereview import CodeReviewResult from bcbench.results.testgeneration import TestGenerationResult @@ -287,6 +292,8 @@ def result_class(self) -> type[BaseEvaluationResult]: return CodeReviewResult case EvaluationCategory.NL2AL: return JudgeBasedEvaluationResult + case EvaluationCategory.DATA_QUERY: + return ExecutionBasedEvaluationResult case EvaluationCategory.EXT_REQUEST_IMPLEMENT: return JudgeBasedEvaluationResult case EvaluationCategory.EXT_REQUEST_TRIAGE: @@ -309,6 +316,8 @@ def summary_class(self) -> type[EvaluationResultSummary]: return CodeReviewResultSummary case EvaluationCategory.NL2AL: return JudgeBasedEvaluationResultSummary + case EvaluationCategory.DATA_QUERY: + return ExecutionBasedEvaluationResultSummary case EvaluationCategory.EXT_REQUEST_IMPLEMENT: return JudgeBasedEvaluationResultSummary case EvaluationCategory.EXT_REQUEST_TRIAGE: @@ -330,6 +339,8 @@ def aggregate_class(self) -> type[LeaderboardAggregate]: return CodeReviewLeaderboardAggregate case EvaluationCategory.NL2AL: return JudgeBasedLeaderboardAggregate + case EvaluationCategory.DATA_QUERY: + return ExecutionBasedLeaderboardAggregate case EvaluationCategory.EXT_REQUEST_IMPLEMENT: return JudgeBasedLeaderboardAggregate case EvaluationCategory.EXT_REQUEST_TRIAGE: @@ -339,7 +350,7 @@ def aggregate_class(self) -> type[LeaderboardAggregate]: @property def pipeline(self) -> EvaluationPipeline: - from bcbench.evaluate import BugFixPipeline, CodeReviewPipeline, ExtRequestImplementPipeline, ExtRequestTriagePipeline, NL2ALPipeline, TestGenerationPipeline + from bcbench.evaluate import BugFixPipeline, CodeReviewPipeline, DataQueryPipeline, ExtRequestImplementPipeline, ExtRequestTriagePipeline, NL2ALPipeline, TestGenerationPipeline match self: case EvaluationCategory.BUG_FIX: @@ -350,6 +361,8 @@ def pipeline(self) -> EvaluationPipeline: return CodeReviewPipeline() case EvaluationCategory.NL2AL: return NL2ALPipeline() + case EvaluationCategory.DATA_QUERY: + return DataQueryPipeline() case EvaluationCategory.EXT_REQUEST_IMPLEMENT: return ExtRequestImplementPipeline() case EvaluationCategory.EXT_REQUEST_TRIAGE: @@ -373,6 +386,8 @@ def evaluators(self) -> list[str]: return ["precision_score", "recall_score", "f1_score", "valid_review_output"] case EvaluationCategory.NL2AL: return ["lm_checklist"] + case EvaluationCategory.DATA_QUERY: + return ["resolution_rate", "build_rate"] case EvaluationCategory.EXT_REQUEST_IMPLEMENT: return ["lm_checklist"] case EvaluationCategory.EXT_REQUEST_TRIAGE: @@ -390,6 +405,8 @@ def core_score(self) -> str: return "F1Score" case EvaluationCategory.NL2AL | EvaluationCategory.EXT_REQUEST_IMPLEMENT | EvaluationCategory.EXT_REQUEST_TRIAGE: return "test_passed" + case EvaluationCategory.DATA_QUERY: + return "ResolutionRate" raise ValueError(f"Unknown evaluation category: {self}") @@ -397,7 +414,7 @@ def core_score(self) -> str: def requires_container(self) -> bool: """Whether evaluating this category builds/runs AL code and therefore needs a BC container.""" match self: - case EvaluationCategory.BUG_FIX | EvaluationCategory.TEST_GENERATION: + case EvaluationCategory.BUG_FIX | EvaluationCategory.TEST_GENERATION | EvaluationCategory.DATA_QUERY: return True case EvaluationCategory.CODE_REVIEW | EvaluationCategory.NL2AL | EvaluationCategory.EXT_REQUEST_IMPLEMENT | EvaluationCategory.EXT_REQUEST_TRIAGE: return False @@ -418,7 +435,7 @@ def runner(self) -> str: Only categories that require building BaseApp needs self-hosted runners. """ match self: - case EvaluationCategory.BUG_FIX | EvaluationCategory.TEST_GENERATION: + case EvaluationCategory.BUG_FIX | EvaluationCategory.TEST_GENERATION | EvaluationCategory.DATA_QUERY: return "GitHub-BCBench" case EvaluationCategory.CODE_REVIEW | EvaluationCategory.EXT_REQUEST_IMPLEMENT | EvaluationCategory.EXT_REQUEST_TRIAGE: return "ubuntu-latest" diff --git a/tests/conftest.py b/tests/conftest.py index c3c572ac9..11c2404fd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,7 +13,7 @@ import pytest -from bcbench.dataset import BaseDatasetEntry, BugFixEntry, ExtRequestImplementEntry, ExtRequestTriageEntry, ManagedLabel, NL2ALEntry, TestEntry +from bcbench.dataset import BaseDatasetEntry, BugFixEntry, DataQueryEntry, ExtRequestImplementEntry, ExtRequestTriageEntry, ManagedLabel, NL2ALEntry, TestEntry from bcbench.dataset.codereview import CodeReviewEntry, ReviewComment, Severity from bcbench.dataset.dataset_entry import EntryMetadata, _BugFixTestGenBase from bcbench.evaluate.review_parsing import parse_review_output @@ -352,6 +352,35 @@ def sample_nl2al_entry() -> NL2ALEntry: return create_nl2al_entry() +VALID_DATA_QUERY_PROMPT = "Return the total sales amount per customer." +VALID_GOLD_QUERY = ( + 'query 50100 SalesByCustomer\n{\n QueryType = Normal;\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; "No.") { }\n }\n }\n}' +) + + +def create_data_query_entry( + instance_id: str = "dataquery__sales-by-customer-1", + environment_setup_version: str = VALID_ENVIRONMENT_VERSION, + nl_prompt: str = VALID_DATA_QUERY_PROMPT, + created_at: str = VALID_CREATED_AT, + gold_query: str = VALID_GOLD_QUERY, + ordered: bool = False, +) -> DataQueryEntry: + return DataQueryEntry( + instance_id=instance_id, + environment_setup_version=environment_setup_version, + nl_prompt=nl_prompt, + created_at=created_at, + gold_query=gold_query, + ordered=ordered, + ) + + +@pytest.fixture +def sample_data_query_entry() -> DataQueryEntry: + return create_data_query_entry() + + def create_ext_implement_entry( instance_id: str = "microsoftInternal__NAV-Ext_Request_Impl-30361", repo: str = "microsoftInternal/NAV", diff --git a/tests/test_agent_skills.py b/tests/test_agent_skills.py index 21aafc7e2..4fa918a03 100644 --- a/tests/test_agent_skills.py +++ b/tests/test_agent_skills.py @@ -8,7 +8,7 @@ import pytest -from bcbench.dataset import RepoGroundedEntry +from bcbench.dataset import BaseDatasetEntry, RepoGroundedEntry from bcbench.operations import setup_agent_skills from bcbench.operations.instruction_operations import _get_source_instructions_path from bcbench.types import AgentHarness @@ -160,3 +160,45 @@ def test_skills_disabled(): assert result is False assert not (repo_path / ".github" / "skills").exists() + + +def test_skills_override_enables_when_config_disabled(): + """--skills (override=True) enables skills even when config.yaml has them disabled.""" + with TemporaryDirectory() as tmpdir: + repo_path = Path(tmpdir) + entry = MagicMock(spec=BaseDatasetEntry) + entry.customization_profile = "microsoftInternal-NAV" + config = {"skills": {"enabled": False}} + + result = setup_agent_skills(config, entry, repo_path, harness=AgentHarness.COPILOT, skills_enabled_override=True) + + assert result is True + assert (repo_path / ".github" / "skills").exists() + + +def test_skills_override_disables_when_config_enabled(): + """override=False wins over an enabled config, so no skills are copied.""" + with TemporaryDirectory() as tmpdir: + repo_path = Path(tmpdir) + entry = MagicMock(spec=BaseDatasetEntry) + entry.customization_profile = "microsoftInternal-NAV" + config = {"skills": {"enabled": True}} + + result = setup_agent_skills(config, entry, repo_path, harness=AgentHarness.COPILOT, skills_enabled_override=False) + + assert result is False + assert not (repo_path / ".github" / "skills").exists() + + +def test_skills_override_none_falls_back_to_config(): + """override=None (default) preserves the config-driven behavior.""" + with TemporaryDirectory() as tmpdir: + repo_path = Path(tmpdir) + entry = MagicMock(spec=BaseDatasetEntry) + entry.customization_profile = "microsoftInternal-NAV" + config = {"skills": {"enabled": False}} + + result = setup_agent_skills(config, entry, repo_path, harness=AgentHarness.COPILOT, skills_enabled_override=None) + + assert result is False + assert not (repo_path / ".github" / "skills").exists() diff --git a/tests/test_dataquery_evaluation.py b/tests/test_dataquery_evaluation.py new file mode 100644 index 000000000..dabe01dce --- /dev/null +++ b/tests/test_dataquery_evaluation.py @@ -0,0 +1,192 @@ +import json + +import pytest + +from bcbench.evaluate.dataquery import result_sets_match +from bcbench.exceptions import BuildError +from bcbench.operations import bc_operations, wrap_query_as_api +from bcbench.types import ContainerConfig + + +class TestResultSetsMatch: + def test_identical_rows_match(self): + rows = [{"No": "C1", "Total": 100}, {"No": "C2", "Total": 200}] + assert result_sets_match(rows, rows) + + def test_row_order_ignored_when_unordered(self): + generated = [{"No": "C2", "Total": 200}, {"No": "C1", "Total": 100}] + gold = [{"No": "C1", "Total": 100}, {"No": "C2", "Total": 200}] + assert result_sets_match(generated, gold, ordered=False) + + def test_row_order_enforced_when_ordered(self): + generated = [{"No": "C2", "Total": 200}, {"No": "C1", "Total": 100}] + gold = [{"No": "C1", "Total": 100}, {"No": "C2", "Total": 200}] + assert not result_sets_match(generated, gold, ordered=True) + + def test_numeric_normalization(self): + # Amounts arrive as numeric JSON types on both sides; scale differences must not matter. + assert result_sets_match([{"Total": 500}], [{"Total": 500.0}]) + + def test_column_names_ignored(self): + assert result_sets_match([{"ItemNo": "I1", "Qty": 5}], [{"No": "I1", "Total": 5}]) + + def test_odata_metadata_keys_ignored(self): + generated = [{"@odata.etag": "W/abc", "No": "C1", "Total": 100}] + gold = [{"No": "C1", "Total": 100}] + assert result_sets_match(generated, gold) + + def test_mismatch_detected(self): + assert not result_sets_match([{"No": "C1", "Total": 100}], [{"No": "C1", "Total": 999}]) + + def test_different_row_count_mismatch(self): + assert not result_sets_match([{"No": "C1"}], [{"No": "C1"}, {"No": "C2"}]) + + def test_close_but_distinct_values_do_not_match(self): + # Guards against numeric rounding collapsing distinct values into a false positive. + assert not result_sets_match([{"Total": 1.00001}], [{"Total": 1.00002}]) + + def test_high_precision_preserved(self): + assert result_sets_match([{"Total": 1.000000001}], [{"Total": 1.000000001}]) + assert not result_sets_match([{"Total": 1.000000001}], [{"Total": 1.000000002}]) + + def test_scale_insensitive(self): + assert result_sets_match([{"Total": 500}], [{"Total": 500.00}]) + + def test_digit_only_code_strings_not_collapsed(self): + # BC Code/No. fields are JSON strings even when digit-only: "001" and "1" are DISTINCT records + # and must never be scored as matching just because they are numerically equal. + assert not result_sets_match([{"No": "001"}], [{"No": "1"}]) + assert not result_sets_match([{"No": "0010"}], [{"No": "10"}]) + + def test_identical_code_strings_match(self): + assert result_sets_match([{"No": "001", "Name": "Acme"}], [{"No": "001", "Name": "Acme"}]) + + def test_numeric_string_not_coerced_to_number(self): + # A code that happens to look like a scaled number must not match the numeric value 1. + assert not result_sets_match([{"Key": "1.0"}], [{"Key": 1}]) + + +class TestWrapQueryAsApi: + PLAIN_QUERY = 'query 50100 MyQuery\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; "No.") { }\n }\n }\n}' + LONG_NAME_QUERY = 'query 50100 "Items on Open Sales and Purchase Orders"\n{\n elements\n {\n dataitem(Item; Item)\n {\n column(No; "No.") { }\n }\n }\n}' + + def test_reassigns_object_id_and_name(self): + wrapped = wrap_query_as_api(self.PLAIN_QUERY, 50101) + assert "query 50101 BCBenchQuery50101" in wrapped + assert "query 50100" not in wrapped + assert "MyQuery" not in wrapped + + def test_normalizes_overlong_quoted_name(self): + # A descriptive >30-char name would trip AL0305; the harness normalizes it away. + wrapped = wrap_query_as_api(self.LONG_NAME_QUERY, 50100) + assert "query 50100 BCBenchQuery50100" in wrapped + assert "Items on Open Sales and Purchase Orders" not in wrapped + + def test_injects_api_properties(self): + wrapped = wrap_query_as_api(self.PLAIN_QUERY, 50100) + assert "QueryType = API;" in wrapped + assert "APIPublisher = 'bcbench';" in wrapped + assert "EntitySetName = 'bcbenchResults50100';" in wrapped + + def test_generated_and_gold_use_distinct_entity_sets(self): + # Both apps can be published to the same tenant; distinct entity sets avoid an OData route collision. + generated = wrap_query_as_api(self.PLAIN_QUERY, 50100) + gold = wrap_query_as_api(self.PLAIN_QUERY, 50101) + assert "EntitySetName = 'bcbenchResults50100';" in generated + assert "EntitySetName = 'bcbenchResults50101';" in gold + + def test_drops_existing_querytype(self): + wrapped = wrap_query_as_api(self.PLAIN_QUERY, 50100) + assert "QueryType = Normal;" not in wrapped + + def test_preserves_query_body(self): + wrapped = wrap_query_as_api(self.PLAIN_QUERY, 50100) + assert "dataitem(Customer; Customer)" in wrapped + assert 'column(No; "No.")' in wrapped + + def test_uppercase_query_keyword_reassigned(self): + wrapped = wrap_query_as_api('Query 50123 "My Q"\n{\n elements { }\n}', 50100) + assert "50100 BCBenchQuery50100" in wrapped + assert "50123" not in wrapped + + def test_compact_and_cased_querytype_removed(self): + # QueryType on the same line as the brace (no leading newline) and in any casing must still + # be stripped, else the injected QueryType = API duplicates the property. + wrapped = wrap_query_as_api("query 50100 Q\n{ querytype = Normal; elements { } }", 50100) + assert wrapped.count("QueryType") == 1 + assert "QueryType = API;" in wrapped + + def test_missing_brace_raises_builderror(self): + with pytest.raises(BuildError): + wrap_query_as_api("query 50100 MyQuery no body here", 50100) + + def test_no_query_declaration_raises_builderror(self): + with pytest.raises(BuildError): + wrap_query_as_api("codeunit 50100 NotAQuery { }", 50100) + + +def test_execute_al_query_bootstraps_app_manifest(tmp_path, monkeypatch): + app_dir = tmp_path / ".bcbench-query-generated" + + def write_empty_result(*args, **kwargs): + (app_dir / "result.json").write_text("[]", encoding="utf-8") + + monkeypatch.setattr(bc_operations.subprocess, "run", write_empty_result) + + rows = bc_operations.execute_al_query( + 'query 50100 MyQuery { elements { dataitem(Customer; Customer) { column(No; "No.") { } } } }', + ContainerConfig(name="bcserver", username="admin", password="password"), + "26.0.12345.0", + tmp_path, + "generated", + ) + + manifest = json.loads((app_dir / "app.json").read_text(encoding="utf-8")) + assert rows == [] + assert manifest["name"] == "BC-Bench Query generated" + assert manifest["idRanges"] == [{"from": 50100, "to": 50100}] + assert manifest["runtime"] == "15.0" + + +class TestQueryRunTemplate: + def _render(self): + return bc_operations._QUERY_RUN_TEMPLATE.substitute( + app_utils_path="AppUtils.psm1", + container_name="c", + username="u", + password="p", + app_dir="d", + app_name="BC-Bench Query generated", + app_publisher="BC-Bench", + publisher=bc_operations._QUERY_API_PUBLISHER, + group=bc_operations._QUERY_API_GROUP, + version=bc_operations._QUERY_API_VERSION, + entity_set=bc_operations._entity_set_name(50100), + result_file="r", + ) + + def test_uses_proven_build_helper(self): + assert "Invoke-AppBuildAndPublish" in self._render() + + def test_fetches_from_inside_container(self): + script = self._render() + assert "Invoke-ScriptInBcContainer" in script + assert "http://localhost:7048/BC/api" in script + + def test_does_not_use_credential_over_http(self): + # PowerShell 7 (inside the container) refuses -Credential over plain HTTP; we must build a + # Basic auth header by hand instead. + script = self._render() + assert "-Credential" not in script.split("Invoke-ScriptInBcContainer", 1)[1] + assert "Authorization" in script + assert "Basic " in script + + def test_follows_odata_nextlink(self): + # Result sets larger than one OData page must not be silently truncated. + assert "@odata.nextLink" in self._render() + + def test_uninstalls_throwaway_app(self): + # Re-running against the same container must not fail with an object-ID conflict. + script = self._render() + assert "UnPublish-BcContainerApp" in script + assert "UnInstall-BcContainerApp" in script diff --git a/tests/test_type_exhaustiveness.py b/tests/test_type_exhaustiveness.py index 7fb68a7d7..882a79641 100644 --- a/tests/test_type_exhaustiveness.py +++ b/tests/test_type_exhaustiveness.py @@ -2,7 +2,7 @@ import pytest -from bcbench.dataset import BugFixEntry, CodeReviewEntry, ExtRequestImplementEntry, ExtRequestTriageEntry, NL2ALEntry +from bcbench.dataset import BugFixEntry, CodeReviewEntry, DataQueryEntry, ExtRequestImplementEntry, ExtRequestTriageEntry, NL2ALEntry from bcbench.dataset.codereview import ReviewComment, Severity from bcbench.types import AgentHarness, AgentMetrics, EvaluationCategory @@ -58,7 +58,11 @@ def test_all_categories_have_aggregate_classes(): def test_all_categories_handled_in_get_expected_output( - sample_dataset_entry_with_problem_statement: BugFixEntry, sample_nl2al_entry: NL2ALEntry, sample_ext_implement_entry: "ExtRequestImplementEntry", sample_ext_triage_entry: "ExtRequestTriageEntry" + sample_dataset_entry_with_problem_statement: BugFixEntry, + sample_nl2al_entry: NL2ALEntry, + sample_data_query_entry: DataQueryEntry, + sample_ext_implement_entry: ExtRequestImplementEntry, + sample_ext_triage_entry: ExtRequestTriageEntry, ): for category in EvaluationCategory: entry_cls = category.entry_class @@ -75,6 +79,8 @@ def test_all_categories_handled_in_get_expected_output( ) elif entry_cls is NL2ALEntry: entry = sample_nl2al_entry + elif entry_cls is DataQueryEntry: + entry = sample_data_query_entry elif entry_cls is ExtRequestImplementEntry: entry = sample_ext_implement_entry elif entry_cls is ExtRequestTriageEntry: