Add data-query category: AL query-generation benchmark

.github/workflows/claude-evaluation.yml

@@ -24,6 +24,7 @@ on:
           - "bug-fix"
           - "test-generation"
           - "code-review"
+          - "data-query"
       test-run:
         description: "Indicate this is a test run (with few entries)"
         required: false
@@ -39,6 +40,11 @@ on:
         required: false
         default: false
         type: boolean
+      skills:
+        description: "Enable agent skills"
+        required: false
+        default: false
+        type: boolean
       repeat:
         description: "Number of times to run sequentially (ignored for test runs)"
         required: false
@@ -140,7 +146,8 @@ jobs:
             --repo-path "${{ steps.setup-env.outputs.repo_path }}" `
             --output-dir "${{ env.EVALUATION_RESULTS_DIR }}" `
             ${{ inputs.al-mcp && '--al-mcp' || '' }} `
-            ${{ inputs.al-lsp && '--al-lsp' || '' }}
+            ${{ inputs.al-lsp && '--al-lsp' || '' }} `
+            ${{ inputs.skills && '--skills' || '' }}
 
       - name: Upload evaluation results
         uses: actions/upload-artifact@v6
@@ -176,4 +183,4 @@ jobs:
       workflow-file: claude-evaluation.yml
       repeat: ${{ inputs.repeat }}
       workflow-inputs: |
-        {"model": "${{ inputs.model }}", "category": "${{ inputs.category }}", "test-run": "${{ inputs.test-run }}", "al-mcp": "${{ inputs.al-mcp }}", "al-lsp": "${{ inputs.al-lsp }}", "git-ref": "${{ inputs.git-ref || github.ref_name }}"}
+        {"model": "${{ inputs.model }}", "category": "${{ inputs.category }}", "test-run": "${{ inputs.test-run }}", "al-mcp": "${{ inputs.al-mcp }}", "al-lsp": "${{ inputs.al-lsp }}", "skills": "${{ inputs.skills }}", "git-ref": "${{ inputs.git-ref || github.ref_name }}"}

.github/workflows/copilot-evaluation.yml

@@ -31,6 +31,7 @@ on:
           - "bug-fix"
           - "test-generation"
           - "code-review"
+          - "data-query"
       test-run:
         description: "Indicate this is a test run (with few entries)"
         required: false
@@ -46,6 +47,11 @@ on:
         required: false
         default: false
         type: boolean
+      skills:
+        description: "Enable agent skills"
+        required: false
+        default: false
+        type: boolean
       repeat:
         description: "Number of times to run sequentially (ignored for test runs)"
         required: false
@@ -145,7 +151,8 @@ jobs:
             --repo-path "${{ steps.setup-env.outputs.repo_path }}" `
             --output-dir "${{ env.EVALUATION_RESULTS_DIR }}" `
             ${{ inputs.al-mcp && '--al-mcp' || '' }} `
-            ${{ inputs.al-lsp && '--al-lsp' || '' }}
+            ${{ inputs.al-lsp && '--al-lsp' || '' }} `
+            ${{ inputs.skills && '--skills' || '' }}
 
       - name: Upload evaluation results
         uses: actions/upload-artifact@v6
@@ -181,4 +188,4 @@ jobs:
       workflow-file: copilot-evaluation.yml
       repeat: ${{ inputs.repeat }}
       workflow-inputs: |
-        {"model": "${{ inputs.model }}", "category": "${{ inputs.category }}", "test-run": "${{ inputs.test-run }}", "al-mcp": "${{ inputs.al-mcp }}", "al-lsp": "${{ inputs.al-lsp }}", "git-ref": "${{ inputs.git-ref || github.ref_name }}"}
+        {"model": "${{ inputs.model }}", "category": "${{ inputs.category }}", "test-run": "${{ inputs.test-run }}", "al-mcp": "${{ inputs.al-mcp }}", "al-lsp": "${{ inputs.al-lsp }}", "skills": "${{ inputs.skills }}", "git-ref": "${{ inputs.git-ref || github.ref_name }}"}

dataset/dataquery.jsonl

@@ -0,0 +1,11 @@
+{"instance_id": "dataquery__outstanding-sales-value-by-customer-1", "environment_setup_version": "28.3", "created_at": "2026-07-10", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one open sales order line, return the customer's number, the customer's name, and their total outstanding amount. Use open sales order lines (Sales Line records whose document type is Order) and sum the line 'Outstanding Amount' field (which is net of VAT). Customers with no open sales order lines must not appear.", "ordered": false, "gold_query": "query 50100 OutstandingSalesByCustomer\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(Customer; Customer)\n        {\n            column(No; \"No.\") { }\n            column(Name; Name) { }\n            dataitem(SalesLine; \"Sales Line\")\n            {\n                DataItemLink = \"Sell-to Customer No.\" = Customer.\"No.\";\n                DataItemTableFilter = \"Document Type\" = const(Order);\n                column(TotalOutstanding; \"Outstanding Amount\") { Method = Sum; }\n            }\n        }\n    }\n}"}
+{"instance_id": "dataquery__total-sold-quantity-by-item-1", "environment_setup_version": "28.3", "created_at": "2026-07-10", "metadata": {"area": "inventory"}, "nl_prompt": "Across all posted sales invoice lines whose type is Item, return each item's number together with the total quantity sold (the sum of the line Quantity). Produce one row per item number.", "ordered": false, "gold_query": "query 50100 SoldQuantityByItem\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n        {\n            DataItemTableFilter = Type = const(Item);\n            column(ItemNo; \"No.\") { }\n            column(TotalQuantity; Quantity) { Method = Sum; }\n        }\n    }\n}"}
+{"instance_id": "dataquery__avg-invoice-amount-by-country-1", "environment_setup_version": "28.3", "created_at": "2026-07-10", "metadata": {"area": "finance"}, "nl_prompt": "For each country/region, return the country/region code and the average posted sales invoice line amount. Average the 'Amount' field (net of VAT) over all posted sales invoice lines, grouping the lines by their bill-to customer's Country/Region Code.", "ordered": false, "gold_query": "query 50100 AvgInvoiceAmountByCountry\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(Customer; Customer)\n        {\n            column(CountryRegionCode; \"Country/Region Code\") { }\n            dataitem(SalesInvoiceHeader; \"Sales Invoice Header\")\n            {\n                DataItemLink = \"Bill-to Customer No.\" = Customer.\"No.\";\n                dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n                {\n                    DataItemLink = \"Document No.\" = SalesInvoiceHeader.\"No.\";\n                    column(AvgAmount; Amount) { Method = Average; }\n                }\n            }\n        }\n    }\n}"}
+{"instance_id": "dataquery__total-purchase-amount-by-vendor-1", "environment_setup_version": "28.3", "created_at": "2026-07-10", "metadata": {"area": "purchasing"}, "nl_prompt": "For each vendor that has at least one posted purchase invoice line, return the vendor's number, the vendor's name, and their total posted purchase amount. Sum the 'Amount' field (net of VAT) from posted purchase invoice lines. Vendors with no posted purchase invoice lines must not appear.", "ordered": false, "gold_query": "query 50100 PurchaseAmountByVendor\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(Vendor; Vendor)\n        {\n            column(No; \"No.\") { }\n            column(Name; Name) { }\n            dataitem(PurchInvLine; \"Purch. Inv. Line\")\n            {\n                DataItemLink = \"Buy-from Vendor No.\" = Vendor.\"No.\";\n                column(TotalAmount; Amount) { Method = Sum; }\n            }\n        }\n    }\n}"}
+{"instance_id": "dataquery__open-sales-order-count-by-customer-1", "environment_setup_version": "28.3", "created_at": "2026-07-13", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one open sales order, return the customer's number, the customer's name, and the number of open sales orders they have. An open sales order is a sales document whose document type is Order; count the order documents (headers), not the order lines. Customers with no open sales orders must not appear.", "ordered": false, "gold_query": "query 50100 OpenSalesOrdersByCustomer\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(Customer; Customer)\n        {\n            column(No; \"No.\") { }\n            column(Name; Name) { }\n            dataitem(SalesHeader; \"Sales Header\")\n            {\n                DataItemLink = \"Sell-to Customer No.\" = Customer.\"No.\";\n                DataItemTableFilter = \"Document Type\" = const(Order);\n                column(OrderCount) { Method = Count; }\n            }\n        }\n    }\n}"}
+{"instance_id": "dataquery__opportunity-count-by-status-1", "environment_setup_version": "28.3", "created_at": "2026-07-10", "metadata": {"area": "crm"}, "nl_prompt": "Return the number of CRM opportunities in each status. Group the opportunities by their Status field and, for each status value that occurs, output the status and the count of opportunities with that status.", "ordered": false, "gold_query": "query 50100 OpportunityCountByStatus\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(Opportunity; Opportunity)\n        {\n            column(Status; Status) { }\n            column(OpportunityCount) { Method = Count; }\n        }\n    }\n}"}
+{"instance_id": "dataquery__customer-count-by-country-1", "environment_setup_version": "28.3", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "Group all customers by their Country/Region Code and return, for each distinct code, the Country/Region Code and the number of customers that have it. Include customers whose Country/Region Code is blank as their own group. Count the Customer records (one row per distinct Country/Region Code).", "ordered": false, "gold_query": "query 50100 CustomerCountByCountry\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(Customer; Customer)\n        {\n            column(CountryRegionCode; \"Country/Region Code\") { }\n            column(CustomerCount) { Method = Count; }\n        }\n    }\n}"}
+{"instance_id": "dataquery__outstanding-purchase-value-by-vendor-1", "environment_setup_version": "28.3", "created_at": "2026-07-28", "metadata": {"area": "purchase"}, "nl_prompt": "For each vendor that has at least one open purchase order line, return the vendor's number, the vendor's name, and their total outstanding amount. Use open purchase order lines (Purchase Line records whose Document Type is Order) and sum the line 'Outstanding Amount' field (net of VAT). Vendors with no open purchase order lines must not appear.", "ordered": false, "gold_query": "query 50100 OutstandingPurchaseByVendor\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(Vendor; Vendor)\n        {\n            column(No; \"No.\") { }\n            column(Name; Name) { }\n            dataitem(PurchaseLine; \"Purchase Line\")\n            {\n                DataItemLink = \"Buy-from Vendor No.\" = Vendor.\"No.\";\n                DataItemTableFilter = \"Document Type\" = const(Order);\n                column(TotalOutstanding; \"Outstanding Amount\") { Method = Sum; }\n            }\n        }\n    }\n}"}
+{"instance_id": "dataquery__total-posted-sales-amount-by-customer-1", "environment_setup_version": "28.3", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one posted sales invoice line, return the customer's number, the customer's name, and their total posted sales amount. Join posted sales invoice headers to their lines via Document No., group by the header's Bill-to Customer No., and sum the line 'Amount' field (net of VAT). Customers with no posted sales invoice lines must not appear.", "ordered": false, "gold_query": "query 50100 PostedSalesAmountByCustomer\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(Customer; Customer)\n        {\n            column(No; \"No.\") { }\n            column(Name; Name) { }\n            dataitem(SalesInvoiceHeader; \"Sales Invoice Header\")\n            {\n                DataItemLink = \"Bill-to Customer No.\" = Customer.\"No.\";\n                dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n                {\n                    DataItemLink = \"Document No.\" = SalesInvoiceHeader.\"No.\";\n                    column(TotalAmount; Amount) { Method = Sum; }\n                }\n            }\n        }\n    }\n}"}
+{"instance_id": "dataquery__line-count-per-open-sales-order-1", "environment_setup_version": "28.3", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "For each open sales order, return the order's document number and the number of lines it has. Use Sales Line records whose Document Type is Order, group them by Document No., and count the lines per order (one row per order document number).", "ordered": false, "gold_query": "query 50100 LineCountPerOpenSalesOrder\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(SalesLine; \"Sales Line\")\n        {\n            DataItemTableFilter = \"Document Type\" = const(Order);\n            column(DocumentNo; \"Document No.\") { }\n            column(LineCount) { Method = Count; }\n        }\n    }\n}"}
+{"instance_id": "dataquery__total-purchased-quantity-by-item-1", "environment_setup_version": "28.3", "created_at": "2026-07-28", "metadata": {"area": "purchase"}, "nl_prompt": "Across all posted purchase invoice lines whose Type is Item, return each item's number together with the total purchased quantity (the sum of the line Quantity). Produce one row per item number.", "ordered": false, "gold_query": "query 50100 PurchasedQuantityByItem\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(PurchInvLine; \"Purch. Inv. Line\")\n        {\n            DataItemTableFilter = Type = const(Item);\n            column(ItemNo; \"No.\") { }\n            column(TotalQuantity; Quantity) { Method = Sum; }\n        }\n    }\n}"}

docs/data-query.md

@@ -0,0 +1,46 @@
+---
+layout: default
+title: Data Query - BC-Bench
+---
+
+# Data Query
+
+This category benchmarks an agent's ability to **generate Business Central AL queries** from a natural-language data question — an offline query-generation benchmark. There is **no MCP server and no live server in the loop**: the agent writes an AL query, and the query is evaluated deterministically.
+
+Given a question, the agent authors a single AL `query` object and writes it to `query.al`. The harness then **compiles and runs both the generated query and a gold reference query** against a fixed dataset (the BC container's Contoso demo data) and compares the result sets.
+
+## How it is scored
+
+Data Query is **execution-based** (like bug-fix), with no LLM judge:
+
+- **build** — the generated query compiled and ran.
+- **resolved** (the headline `ResolutionRate`) — the generated query's result set **matches the gold query's**. Rows are compared by value (numbers normalized, column names/order ignored); row order is ignored unless the entry marks the question as `ordered`.
+
+To execute a query, the harness wraps it as an API query (injecting `APIPublisher`/`APIGroup`/`APIVersion`/`EntitySetName`), publishes a throwaway app to the container, and reads the query's OData endpoint. This runs on the `GitHub-BCBench` self-hosted runner (`requires_container = True`).
+
+> This complements the AI Test Toolkit evals in the BC platform repo: those test the **MCP server** end-to-end, while BC-Bench benchmarks **models/agents** on query generation.
+
+## Dataset
+
+Each entry has an `nl_prompt` (the question) and a `gold_query` (the reference AL query whose result set defines "correct"), plus `environment_setup_version` (the BC artifact) and `ordered`. See `dataset/dataquery.jsonl`.
+
+## Running it (no local containers)
+
+Trigger it from the GitHub **Actions** tab — the self-hosted `GitHub-BCBench` runner provisions the BC container for you (a stock **sandbox artifact with Cronus/Contoso demo data** — no special build is needed, since the query is just compiled and run):
+
+1. Actions → **Evaluation with GitHub Copilot** (or **Evaluation with Claude Code**) → **Run workflow**.
+2. Set **category** = `data-query`, pick a **model**, leave **test-run** = `true` for a quick 2-entry run.
+3. The run: provisions the container → the agent writes `query.al` → the harness compiles + runs the generated and gold queries → compares result sets → `summarize-results` reports `ResolutionRate` / `BuildRate`.
+
+`data-query` sets `requires_container = True`; its container setup **skips the repo clone** (there is no repo — the agent generates from scratch) and just stands up the sandbox container.
+
+### Local (optional)
+
+```bash
+uv run bcbench evaluate copilot dataquery__outstanding-sales-value-by-customer-1 \
+  --category data-query --container-name <bc-sandbox> --username admin --password <pw>
+```
+
+The optional `al-mcp` / `al-lsp` levers give the agent AL compiler/language-server feedback while it authors the query.
+
+[← Back to Home](index.md)

docs/index.md

@@ -12,6 +12,7 @@ A benchmark for evaluating AI coding agents on real-world **Business Central (AL
 | [Bug Fixing](bug-fix.md) | Follows [SWE-Bench](https://www.swebench.com/) methodology to evaluate bug fixing in AL code |
 | [Test Generation](test-generation.md) | "Reverses" SWE-Bench: Generates reproduction tests (TDD) instead of fixes |
 | [Code Review](code-review.md) | Reviews AL pull requests; scored with Precision / Recall / F1 against gold findings |
+| [Data Query](data-query.md) | Generates AL queries from natural-language data questions; scored deterministically by running the query and comparing its result set to a gold query |
 
 ## Diagnostics
 

scripts/BCBenchUtils.psm1

@@ -490,7 +490,7 @@ function Get-BCBenchDatasetPath {
     param(
         [Parameter(Mandatory = $true)]
         # Category validation lives only here: every caller resolves the dataset path through this function, so there's no need to duplicate ValidateSet on each caller.
-        [ValidateSet("bug-fix", "test-generation", "code-review", "nl2al")]
+        [ValidateSet("bug-fix", "test-generation", "code-review", "nl2al", "data-query")]
         [string] $Category
     )
 
@@ -499,6 +499,7 @@ function Get-BCBenchDatasetPath {
         "test-generation" { $DatasetName = "bcbench.jsonl" }
         "code-review" { $DatasetName = "codereview.jsonl" }
         "nl2al" { $DatasetName = "nl2al.jsonl" }
+        "data-query" { $DatasetName = "dataquery.jsonl" }
     }
 
     [string] $projectRoot = Split-Path $PSScriptRoot -Parent

scripts/Setup-ContainerAndRepository.ps1

@@ -72,7 +72,10 @@ if (-not $SkipRepo) {
     Invoke-GitCloneWithRetry -RepoUrl $cloneInfo.Url -Token $cloneInfo.Token -ClonePath $RepoPath -CommitSha $commitSha -SparseCheckoutPaths $cloneInfo.SparseCheckoutPaths
 }
 else {
-    Write-Log "Skipping repository clone (SkipRepo flag set)" -Level Info
+    # Categories that scaffold their own workspace still need the folder to exist: it is shared into
+    # the container below, and Compile-AppInBcContainer throws for any path not shared with it.
+    Write-Log "Skipping repository clone (SkipRepo flag set); creating empty workspace at $RepoPath" -Level Info
+    New-Item -ItemType Directory -Path $RepoPath -Force | Out-Null
 }
 
 if (-not $SkipContainer) {

src/bcbench/agent/claude/agent.py

@@ -26,6 +26,7 @@ def run_claude_code(
     output_dir: Path,
     al_mcp: bool = False,
     al_lsp: bool = False,
+    skills: bool = False,
     container_name: str = "bcbench",
 ) -> tuple[AgentMetrics | None, ExperimentConfiguration]:
     """Run Claude Code on a single dataset entry.
@@ -46,7 +47,7 @@ def run_claude_code(
     mcp_config_json, mcp_server_names = build_mcp_config(claude_config, entry, repo_path, al_mcp=al_mcp, container_name=container_name)
     lsp_plugin_dir: Path | None = build_al_lsp_plugin(entry, category, repo_path, AgentType.CLAUDE, al_lsp=al_lsp, container_name=container_name)
     instructions_enabled: bool = setup_instructions_from_config(claude_config, entry, repo_path, agent_type=AgentType.CLAUDE)
-    skills_enabled: bool = setup_agent_skills(claude_config, entry, repo_path, agent_type=AgentType.CLAUDE)
+    skills_enabled: bool = setup_agent_skills(claude_config, entry, repo_path, agent_type=AgentType.CLAUDE, skills_enabled_override=skills)
     custom_agent: str | None = setup_custom_agent(claude_config, entry, repo_path, agent_type=AgentType.CLAUDE)
     tool_log_path: Path = setup_hooks(repo_path, AgentType.CLAUDE, output_dir)
     plugins: dict[str, Path] = resolve_config_plugins(claude_config)