Add data-query category: AL query-generation benchmark

.github/workflows/claude-evaluation.yml

@@ -24,6 +24,7 @@ on:
           - "bug-fix"
           - "test-generation"
           - "code-review"
+          - "data-query"
       test-run:
         description: "Indicate this is a test run (with few entries)"
         required: false

.github/workflows/copilot-evaluation.yml

@@ -31,6 +31,7 @@ on:
           - "bug-fix"
           - "test-generation"
           - "code-review"
+          - "data-query"
       test-run:
         description: "Indicate this is a test run (with few entries)"
         required: false

dataset/dataquery.jsonl

@@ -0,0 +1,6 @@
+{"instance_id": "dataquery__outstanding-sales-value-by-customer-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-10", "metadata": {"area": "sales"}, "nl_prompt": "For each customer, what is the total outstanding value across their open sales orders? Include the customer's number and name.", "ordered": false, "gold_query": "query 50100 OutstandingSalesByCustomer\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(Customer; Customer)\n        {\n            column(No; \"No.\") { }\n            column(Name; Name) { }\n            dataitem(SalesLine; \"Sales Line\")\n            {\n                DataItemLink = \"Sell-to Customer No.\" = Customer.\"No.\";\n                DataItemTableFilter = \"Document Type\" = const(Order);\n                column(TotalOutstanding; \"Outstanding Amount\") { Method = Sum; }\n            }\n        }\n    }\n}"}
+{"instance_id": "dataquery__total-sold-quantity-by-item-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-10", "metadata": {"area": "inventory"}, "nl_prompt": "What is the total sold quantity per item across all posted sales invoice lines? Return each item number and its total quantity.", "ordered": false, "gold_query": "query 50100 SoldQuantityByItem\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n        {\n            DataItemTableFilter = Type = const(Item);\n            column(ItemNo; \"No.\") { }\n            column(TotalQuantity; Quantity) { Method = Sum; }\n        }\n    }\n}"}
+{"instance_id": "dataquery__avg-invoice-amount-by-country-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-10", "metadata": {"area": "finance"}, "nl_prompt": "For each country/region, what is the average posted sales invoice line amount? Group the posted sales invoice lines by the bill-to customer's country/region code.", "ordered": false, "gold_query": "query 50100 AvgInvoiceAmountByCountry\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(Customer; Customer)\n        {\n            column(CountryRegionCode; \"Country/Region Code\") { }\n            dataitem(SalesInvoiceHeader; \"Sales Invoice Header\")\n            {\n                DataItemLink = \"Bill-to Customer No.\" = Customer.\"No.\";\n                dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n                {\n                    DataItemLink = \"Document No.\" = SalesInvoiceHeader.\"No.\";\n                    column(AvgAmount; Amount) { Method = Average; }\n                }\n            }\n        }\n    }\n}"}
+{"instance_id": "dataquery__total-purchase-amount-by-vendor-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-10", "metadata": {"area": "purchasing"}, "nl_prompt": "What is the total posted purchase invoice amount per vendor? Include the vendor's number and name.", "ordered": false, "gold_query": "query 50100 PurchaseAmountByVendor\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(Vendor; Vendor)\n        {\n            column(No; \"No.\") { }\n            column(Name; Name) { }\n            dataitem(PurchInvLine; \"Purch. Inv. Line\")\n            {\n                DataItemLink = \"Buy-from Vendor No.\" = Vendor.\"No.\";\n                column(TotalAmount; Amount) { Method = Sum; }\n            }\n        }\n    }\n}"}
+{"instance_id": "dataquery__items-on-both-open-orders-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-10", "metadata": {"area": "inventory"}, "nl_prompt": "Which items appear on both open sales orders and open purchase orders? Return the item numbers.", "ordered": false, "gold_query": "query 50100 ItemsOnBothOpenOrders\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(SalesLine; \"Sales Line\")\n        {\n            DataItemTableFilter = \"Document Type\" = const(Order), Type = const(Item);\n            column(ItemNo; \"No.\") { }\n            dataitem(PurchaseLine; \"Purchase Line\")\n            {\n                DataItemLink = \"No.\" = SalesLine.\"No.\";\n                DataItemTableFilter = \"Document Type\" = const(Order), Type = const(Item);\n            }\n        }\n    }\n}"}
+{"instance_id": "dataquery__opportunity-count-by-status-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-10", "metadata": {"area": "crm"}, "nl_prompt": "How many CRM opportunities are there in each status?", "ordered": false, "gold_query": "query 50100 OpportunityCountByStatus\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(Opportunity; Opportunity)\n        {\n            column(Status; Status) { }\n            column(OpportunityCount; \"No.\") { Method = Count; }\n        }\n    }\n}"}

docs/data-query.md

@@ -0,0 +1,46 @@
+---
+layout: default
+title: Data Query - BC-Bench
+---
+
+# Data Query
+
+This category benchmarks an agent's ability to **generate Business Central AL queries** from a natural-language data question — an offline query-generation benchmark. There is **no MCP server and no live server in the loop**: the agent writes an AL query, and the query is evaluated deterministically.
+
+Given a question, the agent authors a single AL `query` object and writes it to `query.al`. The harness then **compiles and runs both the generated query and a gold reference query** against a fixed dataset (the BC container's Contoso demo data) and compares the result sets.
+
+## How it is scored
+
+Data Query is **execution-based** (like bug-fix), with no LLM judge:
+
+- **build** — the generated query compiled and ran.
+- **resolved** (the headline `ResolutionRate`) — the generated query's result set **matches the gold query's**. Rows are compared by value (numbers normalized, column names/order ignored); row order is ignored unless the entry marks the question as `ordered`.
+
+To execute a query, the harness wraps it as an API query (injecting `APIPublisher`/`APIGroup`/`APIVersion`/`EntitySetName`), publishes a throwaway app to the container, and reads the query's OData endpoint. This runs on the `GitHub-BCBench` self-hosted runner (`requires_container = True`).
+
+> This complements the AI Test Toolkit evals in the BC platform repo: those test the **MCP server** end-to-end, while BC-Bench benchmarks **models/agents** on query generation.
+
+## Dataset
+
+Each entry has an `nl_prompt` (the question) and a `gold_query` (the reference AL query whose result set defines "correct"), plus `environment_setup_version` (the BC artifact) and `ordered`. See `dataset/dataquery.jsonl`.
+
+## Running it (no local containers)
+
+Trigger it from the GitHub **Actions** tab — the self-hosted `GitHub-BCBench` runner provisions the BC container for you (a stock **sandbox artifact with Cronus/Contoso demo data** — no special build is needed, since the query is just compiled and run):
+
+1. Actions → **Evaluation with GitHub Copilot** (or **Evaluation with Claude Code**) → **Run workflow**.
+2. Set **category** = `data-query`, pick a **model**, leave **test-run** = `true` for a quick 2-entry run.
+3. The run: provisions the container → the agent writes `query.al` → the harness compiles + runs the generated and gold queries → compares result sets → `summarize-results` reports `ResolutionRate` / `BuildRate`.
+
+`data-query` sets `requires_container = True`; its container setup **skips the repo clone** (there is no repo — the agent generates from scratch) and just stands up the sandbox container.
+
+### Local (optional)
+
+```bash
+uv run bcbench evaluate copilot dataquery__outstanding-sales-value-by-customer-1 \
+  --category data-query --container-name <bc-sandbox> --username admin --password <pw>
+```
+
+The optional `al-mcp` / `al-lsp` levers give the agent AL compiler/language-server feedback while it authors the query.
+
+[← Back to Home](index.md)

docs/index.md

@@ -12,6 +12,7 @@ A benchmark for evaluating AI coding agents on real-world **Business Central (AL
 | [Bug Fixing](bug-fix.md) | Follows [SWE-Bench](https://www.swebench.com/) methodology to evaluate bug fixing in AL code |
 | [Test Generation](test-generation.md) | "Reverses" SWE-Bench: Generates reproduction tests (TDD) instead of fixes |
 | [Code Review](code-review.md) | Reviews AL pull requests; scored with Precision / Recall / F1 against gold findings |
+| [Data Query](data-query.md) | Generates AL queries from natural-language data questions; scored deterministically by running the query and comparing its result set to a gold query |
 
 ## Diagnostics
 

scripts/BCBenchUtils.psm1

@@ -490,7 +490,7 @@ function Get-BCBenchDatasetPath {
     param(
         [Parameter(Mandatory = $true)]
         # Category validation lives only here: every caller resolves the dataset path through this function, so there's no need to duplicate ValidateSet on each caller.
-        [ValidateSet("bug-fix", "test-generation", "code-review", "nl2al")]
+        [ValidateSet("bug-fix", "test-generation", "code-review", "nl2al", "data-query")]
         [string] $Category
     )
 
@@ -499,6 +499,7 @@ function Get-BCBenchDatasetPath {
         "test-generation" { $DatasetName = "bcbench.jsonl" }
         "code-review" { $DatasetName = "codereview.jsonl" }
         "nl2al" { $DatasetName = "nl2al.jsonl" }
+        "data-query" { $DatasetName = "dataquery.jsonl" }
     }
 
     [string] $projectRoot = Split-Path $PSScriptRoot -Parent

scripts/Setup-ContainerAndRepository.ps1

@@ -61,11 +61,21 @@ if (Test-Path $RepoPath) {
     throw "Repository already exists at $RepoPath. This indicates the machine was not properly cleaned up from a previous run."
 }
 
-[hashtable] $cloneInfo = Get-RepoCloneInfo -Entry $entries[0]
-[string] $commitSha = $entries[0].base_commit
+# Some categories (e.g. data-query) generate code from scratch rather than editing an existing
+# repository, so there is nothing to clone -- the agent just needs an empty working directory.
+[string[]] $noCloneCategories = @('data-query')
 
-Write-Log "Cloning repository $($entries[0].repo) to $RepoPath" -Level Info
-Invoke-GitCloneWithRetry -RepoUrl $cloneInfo.Url -Token $cloneInfo.Token -ClonePath $RepoPath -CommitSha $commitSha -SparseCheckoutPaths $cloneInfo.SparseCheckoutPaths
+if ($Category -in $noCloneCategories) {
+    Write-Log "Category '$Category' needs no repository clone; creating empty workspace at $RepoPath" -Level Info
+    New-Item -ItemType Directory -Path $RepoPath -Force | Out-Null
+}
+else {
+    [hashtable] $cloneInfo = Get-RepoCloneInfo -Entry $entries[0]
+    [string] $commitSha = $entries[0].base_commit
+
+    Write-Log "Cloning repository $($entries[0].repo) to $RepoPath" -Level Info
+    Invoke-GitCloneWithRetry -RepoUrl $cloneInfo.Url -Token $cloneInfo.Token -ClonePath $RepoPath -CommitSha $commitSha -SparseCheckoutPaths $cloneInfo.SparseCheckoutPaths
+}
 
 if (-not $SkipContainer) {
     [PSCredential]$credential = Get-BCCredential -Username $Username -Password $Password

src/bcbench/agent/shared/config.yaml

@@ -66,6 +66,28 @@ prompt:
 
     If there are no findings, write an empty array. Write only valid JSON to review.json, with no surrounding markdown or commentary.
 
+  data-query-template: |
+    You are writing a Microsoft Dynamics 365 Business Central AL query to answer a data
+    question. Proceed without asking for confirmation.
+
+    Task: author a single AL `query` object that returns the data needed to answer the
+    question below, and write it to a file named `query.al` in {{repo_path}}.
+
+    Requirements:
+    - Write exactly one AL `query` object with an object id in the range 50100..50149 and a
+      name. Reference real Business Central tables and fields.
+    - Return the columns needed to answer the question. Use column `Method` (Sum, Count,
+      Average, Min, Max) for aggregates and `DataItemLink` to join dataitems.
+    - Do NOT add API properties (QueryType, APIPublisher, APIGroup, APIVersion, EntityName,
+      EntitySetName) — the evaluation harness adds those. Write only the query logic
+      (query id/name, elements, dataitem(s), columns, filters, order by).
+    - The file must contain only the query object, nothing else.
+
+    Question:
+    {{task}}
+
+    You MUST write query.al before finishing; if you do not, there is no output to evaluate.
+
 # controls:
 # 1. whether to copy custom instructions from `src/bcbench/agent/shared/instructions/<sanitized-repo>/`
 #    - Copilot: copies to repo/.github/ and renames AGENTS.md to copilot-instructions.md

src/bcbench/agent/shared/instructions/dataquery-bc/skills/al-query-authoring/SKILL.md

@@ -0,0 +1,51 @@
+---
+name: al-query-authoring
+description: Guide for authoring Business Central AL query objects that answer data questions (joins, aggregates, filters, sorting). Use this when asked to write an AL query that returns Business Central data such as customers, vendors, items, sales, purchases, projects, or opportunities.
+---
+
+Write a single, compilable AL `query` object that returns exactly the data needed to answer
+the question. Reference real Business Central tables and fields — a query that does not
+compile, or returns the wrong data, fails.
+
+## Structure
+
+```al
+query 50100 TopCustomersBySales
+{
+    QueryType = Normal;
+
+    elements
+    {
+        dataitem(Customer; Customer)
+        {
+            column(No; "No.") { }
+            column(Name; Name) { }
+            dataitem(SalesLine; "Sales Line")
+            {
+                DataItemLink = "Sell-to Customer No." = Customer."No.";
+                DataItemTableFilter = "Document Type" = const(Order);
+                column(OutstandingAmount; "Outstanding Amount") { Method = Sum; }
+            }
+        }
+    }
+}
+```
+
+## Rules of thumb
+
+- **Aggregate** a column with a method: `column(Total; "Amount (LCY)") { Method = Sum; }`
+  (also `Average`, `Count`, `Min`, `Max`). Non-aggregated columns become the GROUP BY.
+- **Join** by nesting a `dataitem` and linking it: `DataItemLink = "<child field>" = Parent."<field>";`.
+- **Filter** rows with `DataItemTableFilter = "<field>" = const(<value>);` (e.g. an Option
+  like `Document Type`) or a range/expression.
+- **Order** with `OrderBy { descending(<column>); }` when the question asks for ranking or
+  "top N" (combine with `TopNumberOfRows` where appropriate).
+- **Quote** any field or table name that contains spaces or special characters: `"No."`,
+  `"Sales Line"`, `"Amount (LCY)"`.
+- Prefer stored fields; FlowFields and Option fields are supported.
+
+## Common pitfalls
+
+- Don't invent table or field names — use the real Business Central schema.
+- Return only the columns the question needs; extra or missing columns change the result set.
+- One `query` object per file.

src/bcbench/commands/evaluate.py

@@ -308,7 +308,7 @@ def evaluate(self, context: EvaluationContext[BaseDatasetEntry]) -> None:
         logger.info("Mock pipeline: Generating random evaluation result")
 
         match context.category:
-            case EvaluationCategory.BUG_FIX | EvaluationCategory.TEST_GENERATION:
+            case EvaluationCategory.BUG_FIX | EvaluationCategory.TEST_GENERATION | EvaluationCategory.DATA_QUERY:
                 scenarios = ["success", "build-fail"]
             case EvaluationCategory.CODE_REVIEW:
                 scenarios = ["invalid", "valid"]

src/bcbench/dataset/__init__.py

@@ -1,12 +1,13 @@
 """Dataset module for querying, validating and analyze dataset entries."""
 
 from bcbench.dataset.codereview import CodeReviewEntry, ReviewComment, Severity
-from bcbench.dataset.dataset_entry import BaseDatasetEntry, BugFixEntry, NL2ALEntry, TestEntry, TestGenEntry
+from bcbench.dataset.dataset_entry import BaseDatasetEntry, BugFixEntry, DataQueryEntry, NL2ALEntry, TestEntry, TestGenEntry
 
 __all__ = [
     "BaseDatasetEntry",
     "BugFixEntry",
     "CodeReviewEntry",
+    "DataQueryEntry",
     "NL2ALEntry",
     "ReviewComment",
     "Severity",

src/bcbench/dataset/dataset_entry.py

@@ -14,7 +14,7 @@
 
 _config = get_config()
 
-__all__ = ["BaseDatasetEntry", "BugFixEntry", "NL2ALEntry", "TestEntry", "TestGenEntry"]
+__all__ = ["BaseDatasetEntry", "BugFixEntry", "DataQueryEntry", "NL2ALEntry", "TestEntry", "TestGenEntry"]
 
 
 class TestEntry(BaseModel):
@@ -168,3 +168,27 @@ def get_task(self) -> str:
 
     def get_expected_output(self) -> Checklist:
         return {"assertions": self.expected}
+
+
+class DataQueryEntry(BaseDatasetEntry):
+    """Dataset entry for the data-query category — generate an AL query that answers a data question.
+
+    Execution-based: the agent authors an AL query; evaluation compiles + runs both the generated
+    query and the gold query against a fixed dataset (Contoso in a BC container) and compares the
+    result sets. No repo scaffold, so base_commit / patch are relaxed to optional.
+    """
+
+    base_commit: str | None = None
+    patch: str = ""
+
+    nl_prompt: Annotated[str, Field(min_length=1, pattern=r"^[^\x00]*$")]
+    gold_query: Annotated[str, Field(min_length=1, pattern=r"^[^\x00]*$")]
+    # Whether row order is significant when comparing result sets (e.g. the question asks for a
+    # specific ranking). Defaults to False: result sets are compared order-insensitively.
+    ordered: bool = False
+
+    def get_task(self) -> str:
+        return self.nl_prompt
+
+    def get_expected_output(self) -> str:
+        return self.gold_query

src/bcbench/evaluate/__init__.py

@@ -3,7 +3,8 @@
 from bcbench.evaluate.base import EvaluationPipeline
 from bcbench.evaluate.bugfix import BugFixPipeline
 from bcbench.evaluate.codereview import CodeReviewPipeline
+from bcbench.evaluate.dataquery import DataQueryPipeline
 from bcbench.evaluate.nl2al import NL2ALPipeline
 from bcbench.evaluate.testgeneration import TestGenerationPipeline
 
-__all__ = ["BugFixPipeline", "CodeReviewPipeline", "EvaluationPipeline", "NL2ALPipeline", "TestGenerationPipeline"]
+__all__ = ["BugFixPipeline", "CodeReviewPipeline", "DataQueryPipeline", "EvaluationPipeline", "NL2ALPipeline", "TestGenerationPipeline"]