Add data-query category: AL query-generation benchmark

.github/workflows/claude-evaluation.yml

@@ -24,6 +24,7 @@ on:
           - "bug-fix"
           - "test-generation"
           - "code-review"
+          - "data-query"
       test-run:
         description: "Indicate this is a test run (with few entries)"
         required: false

.github/workflows/copilot-evaluation.yml

@@ -31,6 +31,7 @@ on:
           - "bug-fix"
           - "test-generation"
           - "code-review"
+          - "data-query"
       test-run:
         description: "Indicate this is a test run (with few entries)"
         required: false

dataset/dataquery.jsonl

@@ -0,0 +1,11 @@
+{"instance_id": "dataquery__outstanding-sales-value-by-customer-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-10", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one open sales order line, return the customer's number, the customer's name, and their total outstanding amount. Use open sales order lines (Sales Line records whose document type is Order) and sum the line 'Outstanding Amount' field (which is net of VAT). Customers with no open sales order lines must not appear.", "ordered": false, "gold_query": "query 50100 OutstandingSalesByCustomer\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(Customer; Customer)\n        {\n            column(No; \"No.\") { }\n            column(Name; Name) { }\n            dataitem(SalesLine; \"Sales Line\")\n            {\n                DataItemLink = \"Sell-to Customer No.\" = Customer.\"No.\";\n                DataItemTableFilter = \"Document Type\" = const(Order);\n                column(TotalOutstanding; \"Outstanding Amount\") { Method = Sum; }\n            }\n        }\n    }\n}"}
+{"instance_id": "dataquery__total-sold-quantity-by-item-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-10", "metadata": {"area": "inventory"}, "nl_prompt": "Across all posted sales invoice lines whose type is Item, return each item's number together with the total quantity sold (the sum of the line Quantity). Produce one row per item number.", "ordered": false, "gold_query": "query 50100 SoldQuantityByItem\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n        {\n            DataItemTableFilter = Type = const(Item);\n            column(ItemNo; \"No.\") { }\n            column(TotalQuantity; Quantity) { Method = Sum; }\n        }\n    }\n}"}
+{"instance_id": "dataquery__avg-invoice-amount-by-country-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-10", "metadata": {"area": "finance"}, "nl_prompt": "For each country/region, return the country/region code and the average posted sales invoice line amount. Average the 'Amount' field (net of VAT) over all posted sales invoice lines, grouping the lines by their bill-to customer's Country/Region Code.", "ordered": false, "gold_query": "query 50100 AvgInvoiceAmountByCountry\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(Customer; Customer)\n        {\n            column(CountryRegionCode; \"Country/Region Code\") { }\n            dataitem(SalesInvoiceHeader; \"Sales Invoice Header\")\n            {\n                DataItemLink = \"Bill-to Customer No.\" = Customer.\"No.\";\n                dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n                {\n                    DataItemLink = \"Document No.\" = SalesInvoiceHeader.\"No.\";\n                    column(AvgAmount; Amount) { Method = Average; }\n                }\n            }\n        }\n    }\n}"}
+{"instance_id": "dataquery__total-purchase-amount-by-vendor-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-10", "metadata": {"area": "purchasing"}, "nl_prompt": "For each vendor that has at least one posted purchase invoice line, return the vendor's number, the vendor's name, and their total posted purchase amount. Sum the 'Amount' field (net of VAT) from posted purchase invoice lines. Vendors with no posted purchase invoice lines must not appear.", "ordered": false, "gold_query": "query 50100 PurchaseAmountByVendor\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(Vendor; Vendor)\n        {\n            column(No; \"No.\") { }\n            column(Name; Name) { }\n            dataitem(PurchInvLine; \"Purch. Inv. Line\")\n            {\n                DataItemLink = \"Buy-from Vendor No.\" = Vendor.\"No.\";\n                column(TotalAmount; Amount) { Method = Sum; }\n            }\n        }\n    }\n}"}
+{"instance_id": "dataquery__open-sales-order-count-by-customer-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-13", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one open sales order, return the customer's number, the customer's name, and the number of open sales orders they have. An open sales order is a sales document whose document type is Order; count the order documents (headers), not the order lines. Customers with no open sales orders must not appear.", "ordered": false, "gold_query": "query 50100 OpenSalesOrdersByCustomer\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(Customer; Customer)\n        {\n            column(No; \"No.\") { }\n            column(Name; Name) { }\n            dataitem(SalesHeader; \"Sales Header\")\n            {\n                DataItemLink = \"Sell-to Customer No.\" = Customer.\"No.\";\n                DataItemTableFilter = \"Document Type\" = const(Order);\n                column(OrderCount) { Method = Count; }\n            }\n        }\n    }\n}"}
+{"instance_id": "dataquery__opportunity-count-by-status-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-10", "metadata": {"area": "crm"}, "nl_prompt": "Return the number of CRM opportunities in each status. Group the opportunities by their Status field and, for each status value that occurs, output the status and the count of opportunities with that status.", "ordered": false, "gold_query": "query 50100 OpportunityCountByStatus\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(Opportunity; Opportunity)\n        {\n            column(Status; Status) { }\n            column(OpportunityCount) { Method = Count; }\n        }\n    }\n}"}
+{"instance_id": "dataquery__customer-count-by-country-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "Group all customers by their Country/Region Code and return, for each distinct code, the Country/Region Code and the number of customers that have it. Include customers whose Country/Region Code is blank as their own group. Count the Customer records (one row per distinct Country/Region Code).", "ordered": false, "gold_query": "query 50100 CustomerCountByCountry\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(Customer; Customer)\n        {\n            column(CountryRegionCode; \"Country/Region Code\") { }\n            column(CustomerCount) { Method = Count; }\n        }\n    }\n}"}
+{"instance_id": "dataquery__outstanding-purchase-value-by-vendor-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-28", "metadata": {"area": "purchase"}, "nl_prompt": "For each vendor that has at least one open purchase order line, return the vendor's number, the vendor's name, and their total outstanding amount. Use open purchase order lines (Purchase Line records whose Document Type is Order) and sum the line 'Outstanding Amount' field (net of VAT). Vendors with no open purchase order lines must not appear.", "ordered": false, "gold_query": "query 50100 OutstandingPurchaseByVendor\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(Vendor; Vendor)\n        {\n            column(No; \"No.\") { }\n            column(Name; Name) { }\n            dataitem(PurchaseLine; \"Purchase Line\")\n            {\n                DataItemLink = \"Buy-from Vendor No.\" = Vendor.\"No.\";\n                DataItemTableFilter = \"Document Type\" = const(Order);\n                column(TotalOutstanding; \"Outstanding Amount\") { Method = Sum; }\n            }\n        }\n    }\n}"}
+{"instance_id": "dataquery__total-posted-sales-amount-by-customer-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one posted sales invoice line, return the customer's number, the customer's name, and their total posted sales amount. Join posted sales invoice headers to their lines via Document No., group by the header's Bill-to Customer No., and sum the line 'Amount' field (net of VAT). Customers with no posted sales invoice lines must not appear.", "ordered": false, "gold_query": "query 50100 PostedSalesAmountByCustomer\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(Customer; Customer)\n        {\n            column(No; \"No.\") { }\n            column(Name; Name) { }\n            dataitem(SalesInvoiceHeader; \"Sales Invoice Header\")\n            {\n                DataItemLink = \"Bill-to Customer No.\" = Customer.\"No.\";\n                dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n                {\n                    DataItemLink = \"Document No.\" = SalesInvoiceHeader.\"No.\";\n                    column(TotalAmount; Amount) { Method = Sum; }\n                }\n            }\n        }\n    }\n}"}
+{"instance_id": "dataquery__line-count-per-open-sales-order-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "For each open sales order, return the order's document number and the number of lines it has. Use Sales Line records whose Document Type is Order, group them by Document No., and count the lines per order (one row per order document number).", "ordered": false, "gold_query": "query 50100 LineCountPerOpenSalesOrder\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(SalesLine; \"Sales Line\")\n        {\n            DataItemTableFilter = \"Document Type\" = const(Order);\n            column(DocumentNo; \"Document No.\") { }\n            column(LineCount) { Method = Count; }\n        }\n    }\n}"}
+{"instance_id": "dataquery__total-purchased-quantity-by-item-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-28", "metadata": {"area": "purchase"}, "nl_prompt": "Across all posted purchase invoice lines whose Type is Item, return each item's number together with the total purchased quantity (the sum of the line Quantity). Produce one row per item number.", "ordered": false, "gold_query": "query 50100 PurchasedQuantityByItem\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(PurchInvLine; \"Purch. Inv. Line\")\n        {\n            DataItemTableFilter = Type = const(Item);\n            column(ItemNo; \"No.\") { }\n            column(TotalQuantity; Quantity) { Method = Sum; }\n        }\n    }\n}"}

docs/data-query.md

@@ -0,0 +1,46 @@
+---
+layout: default
+title: Data Query - BC-Bench
+---
+
+# Data Query
+
+This category benchmarks an agent's ability to **generate Business Central AL queries** from a natural-language data question — an offline query-generation benchmark. There is **no MCP server and no live server in the loop**: the agent writes an AL query, and the query is evaluated deterministically.
+
+Given a question, the agent authors a single AL `query` object and writes it to `query.al`. The harness then **compiles and runs both the generated query and a gold reference query** against a fixed dataset (the BC container's Contoso demo data) and compares the result sets.
+
+## How it is scored
+
+Data Query is **execution-based** (like bug-fix), with no LLM judge:
+
+- **build** — the generated query compiled and ran.
+- **resolved** (the headline `ResolutionRate`) — the generated query's result set **matches the gold query's**. Rows are compared by value (numbers normalized, column names/order ignored); row order is ignored unless the entry marks the question as `ordered`.
+
+To execute a query, the harness wraps it as an API query (injecting `APIPublisher`/`APIGroup`/`APIVersion`/`EntitySetName`), publishes a throwaway app to the container, and reads the query's OData endpoint. This runs on the `GitHub-BCBench` self-hosted runner (`requires_container = True`).
+
+> This complements the AI Test Toolkit evals in the BC platform repo: those test the **MCP server** end-to-end, while BC-Bench benchmarks **models/agents** on query generation.
+
+## Dataset
+
+Each entry has an `nl_prompt` (the question) and a `gold_query` (the reference AL query whose result set defines "correct"), plus `environment_setup_version` (the BC artifact) and `ordered`. See `dataset/dataquery.jsonl`.
+
+## Running it (no local containers)
+
+Trigger it from the GitHub **Actions** tab — the self-hosted `GitHub-BCBench` runner provisions the BC container for you (a stock **sandbox artifact with Cronus/Contoso demo data** — no special build is needed, since the query is just compiled and run):
+
+1. Actions → **Evaluation with GitHub Copilot** (or **Evaluation with Claude Code**) → **Run workflow**.
+2. Set **category** = `data-query`, pick a **model**, leave **test-run** = `true` for a quick 2-entry run.
+3. The run: provisions the container → the agent writes `query.al` → the harness compiles + runs the generated and gold queries → compares result sets → `summarize-results` reports `ResolutionRate` / `BuildRate`.
+
+`data-query` sets `requires_container = True`; its container setup **skips the repo clone** (there is no repo — the agent generates from scratch) and just stands up the sandbox container.
+
+### Local (optional)
+
+```bash
+uv run bcbench evaluate copilot dataquery__outstanding-sales-value-by-customer-1 \
+  --category data-query --container-name <bc-sandbox> --username admin --password <pw>
+```
+
+The optional `al-mcp` / `al-lsp` levers give the agent AL compiler/language-server feedback while it authors the query.
+
+[← Back to Home](index.md)

docs/index.md

@@ -12,6 +12,7 @@ A benchmark for evaluating AI coding agents on real-world **Business Central (AL
 | [Bug Fixing](bug-fix.md) | Follows [SWE-Bench](https://www.swebench.com/) methodology to evaluate bug fixing in AL code |
 | [Test Generation](test-generation.md) | "Reverses" SWE-Bench: Generates reproduction tests (TDD) instead of fixes |
 | [Code Review](code-review.md) | Reviews AL pull requests; scored with Precision / Recall / F1 against gold findings |
+| [Data Query](data-query.md) | Generates AL queries from natural-language data questions; scored deterministically by running the query and comparing its result set to a gold query |
 
 ## Diagnostics
 

scripts/BCBenchUtils.psm1

@@ -490,7 +490,7 @@ function Get-BCBenchDatasetPath {
     param(
         [Parameter(Mandatory = $true)]
         # Category validation lives only here: every caller resolves the dataset path through this function, so there's no need to duplicate ValidateSet on each caller.
-        [ValidateSet("bug-fix", "test-generation", "code-review", "nl2al")]
+        [ValidateSet("bug-fix", "test-generation", "code-review", "nl2al", "data-query")]
         [string] $Category
     )
 
@@ -499,6 +499,7 @@ function Get-BCBenchDatasetPath {
         "test-generation" { $DatasetName = "bcbench.jsonl" }
         "code-review" { $DatasetName = "codereview.jsonl" }
         "nl2al" { $DatasetName = "nl2al.jsonl" }
+        "data-query" { $DatasetName = "dataquery.jsonl" }
     }
 
     [string] $projectRoot = Split-Path $PSScriptRoot -Parent

scripts/Setup-ContainerAndRepository.ps1

@@ -61,11 +61,21 @@ if (Test-Path $RepoPath) {
     throw "Repository already exists at $RepoPath. This indicates the machine was not properly cleaned up from a previous run."
 }
 
-[hashtable] $cloneInfo = Get-RepoCloneInfo -Entry $entries[0]
-[string] $commitSha = $entries[0].base_commit
+# Some categories (e.g. data-query) generate code from scratch rather than editing an existing
+# repository, so there is nothing to clone -- the agent just needs an empty working directory.
+[string[]] $noCloneCategories = @('data-query')
 
-Write-Log "Cloning repository $($entries[0].repo) to $RepoPath" -Level Info
-Invoke-GitCloneWithRetry -RepoUrl $cloneInfo.Url -Token $cloneInfo.Token -ClonePath $RepoPath -CommitSha $commitSha -SparseCheckoutPaths $cloneInfo.SparseCheckoutPaths
+if ($Category -in $noCloneCategories) {
+    Write-Log "Category '$Category' needs no repository clone; creating empty workspace at $RepoPath" -Level Info
+    New-Item -ItemType Directory -Path $RepoPath -Force | Out-Null
+}
+else {
+    [hashtable] $cloneInfo = Get-RepoCloneInfo -Entry $entries[0]
+    [string] $commitSha = $entries[0].base_commit
+
+    Write-Log "Cloning repository $($entries[0].repo) to $RepoPath" -Level Info
+    Invoke-GitCloneWithRetry -RepoUrl $cloneInfo.Url -Token $cloneInfo.Token -ClonePath $RepoPath -CommitSha $commitSha -SparseCheckoutPaths $cloneInfo.SparseCheckoutPaths
+}
 
 if (-not $SkipContainer) {
     [PSCredential]$credential = Get-BCCredential -Username $Username -Password $Password

src/bcbench/agent/shared/config.yaml

@@ -66,6 +66,28 @@ prompt:
 
     If there are no findings, write an empty array. Write only valid JSON to review.json, with no surrounding markdown or commentary.
 
+  data-query-template: |
+    You are writing a Microsoft Dynamics 365 Business Central AL query to answer a data
+    question. Proceed without asking for confirmation.
+
+    Task: author a single AL `query` object that returns the data needed to answer the
+    question below, and write it to a file named `query.al` in {{repo_path}}.
+
+    Requirements:
+    - Write exactly one AL `query` object with an object id in the range 50100..50149 and a
+      name. Reference real Business Central tables and fields.
+    - Return the columns needed to answer the question. Use column `Method` (Sum, Count,
+      Average, Min, Max) for aggregates and `DataItemLink` to join dataitems.
+    - Do NOT add API properties (QueryType, APIPublisher, APIGroup, APIVersion, EntityName,
+      EntitySetName) — the evaluation harness adds those. Write only the query logic
+      (query id/name, elements, dataitem(s), columns, filters, order by).
+    - The file must contain only the query object, nothing else.
+
+    Question:
+    {{task}}
+
+    You MUST write query.al before finishing; if you do not, there is no output to evaluate.
+
 # controls:
 # 1. whether to copy custom instructions from `src/bcbench/agent/shared/instructions/<sanitized-repo>/`
 #    - Copilot: copies to repo/.github/ and renames AGENTS.md to copilot-instructions.md