microsoft · vyta · Jul 29, 2026
@@ -29,7 +29,6 @@
     "docs/docusaurus/coverage/**",
     "dependency-pinning-artifacts/**",
     "evals/results/**",
-    "evals/beval/**/results/**",
     "vally-results/**",
     "evals/agent-behavior/eval.yaml"
   ],
@@ -136,6 +135,7 @@
     "octocat",
     "onclick",
     "optimi",
+    "overbroad",
     "packageable",
     "parseable",
     "pascalcase",
@@ -159,6 +159,7 @@
     "stimul",
     "subcat",
     "subdirs",
+    "tabindex",
     "terroris",
     "thiserror",
     "trustworthi",

@@ -14,7 +14,7 @@ Items in the Highest Priority Rules section from attached instructions files ove
 * Breaking changes are acceptable.
 * Backward-compatibility layers or legacy support are added only when explicitly requested.
 * Tests, scripts, and one-off markdown docs are created or modified only when the requested change or its directly required support work needs them.
-* Before a dependency-backed npm command, establish the relevant package root with `npm ci` when no successful installation for its current lockfile is known. Treat the repository root, `docs/docusaurus`, and `evals/beval` independently; do not substitute `npm install` or reinstall a known-current root.
+* Before a dependency-backed npm command, establish the relevant package root with `npm ci` when no successful installation for its current lockfile is known. Treat the repository root and `docs/docusaurus` independently; do not substitute `npm install` or reinstall a known-current root.
 * Generic validation uses local-safe commands and does not select `ci:*` commands. A command in a plan, README, template, prior log, catalog, or error is a reference, not an execution request. A task that specifically asks to run or reproduce a named CI lane may use its ordinary `ci:*` command.
 * Browser installation, model or moderation environments, service startup, credentials, execution outside the sandbox, interactive UI, and adjacent CI lanes are separate actions. Do not infer them from generic validation or a failed command.
 

@@ -5,7 +5,6 @@ updates:
   directories:
   - /
   - /docs/docusaurus
-  - /evals/beval
   groups:
     npm-dependencies:
       patterns:

@@ -1,6 +1,6 @@
 ---
 description: "Command taxonomy and CI-owned validation conventions for package scripts, eval wrappers, and owning workflows"
-applyTo: 'package.json, docs/docusaurus/package.json, scripts/evals/**, .github/workflows/docusaurus-tests.yml, .github/workflows/eval-validation.yml, .github/workflows/beval.yml, .github/workflows/pr-validation.yml'
+applyTo: 'package.json, docs/docusaurus/package.json, scripts/evals/**, .github/workflows/docusaurus-tests.yml, .github/workflows/eval-validation.yml, .github/workflows/agent-conformance.yml, .github/workflows/pr-validation.yml'
 ---
 
 # CI-Owned Validation Instructions
@@ -29,7 +29,7 @@ The `ci:*` prefix communicates ownership and default agent behavior. It does not
 
 * Coordinate a breaking package-script rename with every owning workflow consumer in the same change set.
 * Preserve workflow change detection, gates, arguments, outputs, artifacts, summaries, permissions, and failure behavior when changing only an entry-point name.
-* Keep Beval service orchestration and direct workflow invocation unchanged unless a separate requirement justifies a package entry point.
+* Keep agent-conformance suite orchestration and its `workflow_call` invocation unchanged unless a separate requirement justifies a package entry point.
 * Use existing repository syntax, workflow, and source checks plus hosted CI evidence. Do not add a command-policy test subsystem or dedicated policy job solely for this taxonomy.
 
 ## Prerequisites and evidence

@@ -0,0 +1,62 @@
+# Copyright (c) 2026 Microsoft Corporation. All rights reserved.
+# SPDX-License-Identifier: MIT
+name: Agent Conformance
+
+on:
+  workflow_call:
+    secrets:
+      copilot-github-token:
+        required: true
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  conformance:
+    name: Conformance (${{ matrix.suite }})
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        suite:
+          - agent-conformance-accessibility-planner
+          - agent-conformance-dt-coach
+          - agent-conformance-privacy-planner
+          - agent-conformance-rai-planner
+          - agent-conformance-security-planner
+          - agent-conformance-sssc-planner
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Setup Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: "24"
+          cache: "npm"
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Run behavioral conformance suite
+        env:
+          COPILOT_GITHUB_TOKEN: ${{ secrets.copilot-github-token }}
+        run: npx vally eval --suite ${{ matrix.suite }}
+
+      - name: Upload results
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        if: always()
+        with:
+          name: ${{ matrix.suite }}-${{ github.run_id }}
+          path: evals/results/
+          retention-days: 30
@@ -69,7 +69,8 @@ jobs:
           # @github/copilot-linuxmusl-arm64 and @github/copilot-linuxmusl-x64
           # added in v1.0.63) use a non-SPDX proprietary license
           # (LicenseRef-bad-see-license-in-license.md); it is GitHub's own
-          # CLI toolchain, deliberately used in beval.yml.
+          # CLI toolchain, pulled transitively via @github/copilot-sdk by the
+          # vally copilot-sdk executor used to run the eval suites.
           # pkg:npm/hve-core is the private root package (never published to npm).
           # pkg:pypi/certifi uses MPL-2.0 (Mozilla CA bundle).
           # pkg:pypi/charset-normalizer is MIT licensed but some older wheel

@@ -101,3 +101,11 @@ jobs:
       contents: read
       issues: write
 
+  agent-conformance:
+    name: Agent Behavioral Conformance
+    uses: ./.github/workflows/agent-conformance.yml
+    permissions:
+      contents: read
+    secrets:
+      copilot-github-token: ${{ secrets.COPILOT_GITHUB_TOKEN }}
+
@@ -490,8 +490,5 @@ docs/docusaurus/build/
 docs/docusaurus/.docusaurus/
 docs/docusaurus/node_modules/
 
-# Beval evaluation results
-evals/beval/**/results/
-
 # Playwright test output
 test-results/
@@ -24,3 +24,33 @@ suites:
     filter:
       category: script-validation
 
+  agent-conformance-accessibility-planner:
+    description: Behavioral conformance regression suite for the Accessibility Planner agent
+    filter:
+      category: agent-conformance-accessibility-planner
+
+  agent-conformance-dt-coach:
+    description: Behavioral conformance regression suite for the DT Coach agent
+    filter:
+      category: agent-conformance-dt-coach
+
+  agent-conformance-privacy-planner:
+    description: Behavioral conformance regression suite for the Privacy Planner agent
+    filter:
+      category: agent-conformance-privacy-planner
+
+  agent-conformance-rai-planner:
+    description: Behavioral conformance regression suite for the RAI Planner agent
+    filter:
+      category: agent-conformance-rai-planner
+
+  agent-conformance-security-planner:
+    description: Behavioral conformance regression suite for the Security Planner agent
+    filter:
+      category: agent-conformance-security-planner
+
+  agent-conformance-sssc-planner:
+    description: Behavioral conformance regression suite for the SSSC Planner agent
+    filter:
+      category: agent-conformance-sssc-planner
+
@@ -3,7 +3,7 @@ title: Validation Commands and CI-Owned Lanes
 description: Choose local-safe validation defaults and reproduce CI-owned documentation and evaluation lanes when their prerequisites are available
 sidebar_position: 12
 author: Microsoft
-ms.date: 2026-07-17
+ms.date: 2026-07-28
 ms.topic: how-to
 keywords:
   - validation
@@ -56,7 +56,6 @@ the reproducible bootstrap path.
 |-------------------|-----------------------------------------------------------------|
 | Repository root   | Root validation, scripts, and `ci:eval:*` commands              |
 | `docs/docusaurus` | Docusaurus lint, component test, build, and Playwright commands |
-| `evals/beval`     | The Beval workflow and its package-specific dependencies        |
 
 Installing dependencies for one root does not provision the other roots. The
 root commands that delegate to Docusaurus still need the Docusaurus package
@@ -113,6 +112,7 @@ output in `logs/` while diagnosing a failure.
 | Eval execution       | `npm run ci:eval:execute`                                                                    | Manifest, Vally, Copilot credential, and a noninteractive service-capable environment; model-backed and potentially costly; writes `logs/eval-summary.json` and per-spec results |
 | General eval suites  | `npm run ci:eval:run`                                                                        | Vally and model access; model-backed and potentially costly                                                                                                                      |
 | One suite            | `npm run ci:eval:run:skills`, `npm run ci:eval:run:agents`, or `npm run ci:eval:run:scripts` | Same model and service prerequisites as the selected suite                                                                                                                       |
+| Agent conformance    | `npm run ci:eval:run:conformance`                                                            | Vally and model access; runs the six planner-agent conformance suites in sequence and stops at the first failing suite                                                           |
 | Result comparison    | `npm run ci:eval:compare`                                                                    | Existing Vally result sets; compares prior outputs without selecting another suite                                                                                               |
 | Prompt behavior      | `npm run ci:eval:behavior-prompts`                                                           | Vally and model access; runs the prompt conformance spec                                                                                                                         |
 | Instruction behavior | `npm run ci:eval:behavior-instructions`                                                      | Vally and model access; runs the instruction conformance spec                                                                                                                    |
@@ -172,24 +172,26 @@ Only `npm run ci:eval:agent:dashboard:open` is interactive and opens the
 generated dashboard. Keep it separate from unattended validation or report
 generation.
 
-## Beval workflow
+## Agent conformance workflow
 
-Beval is a CI-owned workflow with its own package root at `evals/beval`. The
-workflow has a 30-minute timeout and requires `COPILOT_TOKEN`. It installs that
-package root, starts Copilot ACP agent and judge services on TCP ports 3000 and
-3001, verifies both ports, then runs this existing invocation:
+Agent conformance is a CI-owned workflow that runs the six planner-agent
+suites under `evals/agent-conformance/`. It is invoked from
+`weekly-validation.yml` through `workflow_call`, has a 30-minute timeout, and
+requires the `copilot-github-token` secret exported as `COPILOT_GITHUB_TOKEN`.
+The workflow runs one matrix leg per suite with `fail-fast: false`, so a single
+failing agent does not mask the others:
 
 ```bash
-beval -c evals/beval/dt-coach/eval.config.yaml run --cases evals/beval/dt-coach/cases/ --agent evals/beval/dt-coach/agent.yaml -m validation -o evals/beval/dt-coach/results/results.json
+npx vally eval --suite agent-conformance-dt-coach
 ```
 
-Results remain under `evals/beval/dt-coach/results/` and the workflow uploads
-them as the `beval-results-${{ github.run_id }}` artifact. Run `npm ci` in
-`evals/beval` before a deliberate local reproduction, and establish the two
-services and credential through an operator-managed environment. Do not ask for
-or transmit the credential through chat. Do not treat Beval as part of
-`validate:local`, infer its prerequisites from generic validation, or add a root
-package wrapper solely for naming consistency.
+Results are written under `evals/results/` and uploaded as a per-suite
+artifact. Each stimulus launches the agent artifact in turn 0 and delivers the
+case query in turn 1, then grades the response with a model judge plus advisory
+wall-time and content budgets. Establish the credential through an
+operator-managed environment before a deliberate local reproduction. Do not ask
+for or transmit the credential through chat, do not treat this lane as part of
+`validate:local`, and do not infer its prerequisites from generic validation.
 
 ## Review and cleanup