feat: AI security benchmark adapters (JBB, DoNotAnswer, OPI, AICGSecEval, SecCodeBench, AgentDojo)

AGENTS.md

@@ -154,12 +154,21 @@ harness-evals/
 │   │   │                            # CodeQuality, ExplanationQuality, RootCauseAnalysis, Actionability
 │   │   └── composite/              # CompositeMetric (combine metrics with operators)
 │   │
-│   ├── benchmarks/                  # Academic benchmark evaluation suites
+│   ├── benchmarks/                  # Academic + AI security benchmark suites
 │   │   ├── __init__.py              # Public exports (all benchmark classes)
 │   │   ├── base.py                  # BaseBenchmark ABC, BenchmarkResult
-│   │   ├── dataset_cache.py         # HuggingFace dataset fetching + local caching
+│   │   ├── security_base.py         # SecurityBenchmark (safety-dimension scores)
+│   │   ├── _security_utils.py       # Refusal heuristics, taxonomy metadata, rollups
+│   │   ├── dataset_cache.py         # HuggingFace/URL dataset fetching + local caching
 │   │   ├── sandbox.py               # Process-isolated Python code execution (subprocess)
 │   │   ├── _answer_utils.py         # Shared answer extraction (choice, number, F1)
+│   │   ├── jailbreakbench.py        # JailbreakBench (JBB-Behaviors)
+│   │   ├── do_not_answer.py         # Do-Not-Answer refusal safety
+│   │   ├── open_prompt_injection.py # OpenPromptInjection ASV
+│   │   ├── jailbreakv_28k.py        # JailBreakV-28K
+│   │   ├── aicg_sec_eval.py         # AICGSecEval secure code generation
+│   │   ├── sec_code_bench.py        # SecCodeBench adapter (optional Docker)
+│   │   ├── agentdojo.py             # AgentDojo utility + attack metrics
 │   │   ├── mmlu.py                  # MMLU (57-subject knowledge)
 │   │   ├── gsm8k.py                # GSM8K (math word problems)
 │   │   ├── humaneval.py            # HumanEval (code generation, sandboxed)

CHANGELOG.md

@@ -7,6 +7,32 @@ Versioning follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
 
+## [0.15.0]
+
+### Added
+
+- **AgentDojo** benchmark with separate `utility_pass_rate` and `attack_success_rate` via `BaseTarget` integration.
+
+### Changed
+
+- README **Running full datasets** subsection under AI Security Benchmarks: HF vs fixture-backed loading, cache/offline usage, and upstream manifest paths for SecCodeBench and AgentDojo.
+
+## [0.14.0]
+
+### Added
+
+- **AICGSecEval** secure code generation benchmark with bundled CWE-labeled Python tasks.
+- **SecCodeBench** thin adapter with bundled task manifest and optional `[benchmarks-seccode]` extra.
+
+## [0.13.0]
+
+### Added
+
+- **AI security benchmarks (phase 1)**: `SecurityBenchmark` base class with safety-dimension scores, taxonomy metadata on `Golden`/`EvalCase`, and category rollups.
+- **JailbreakBench**, **DoNotAnswer**, **OpenPromptInjection**, and **JailBreakV28K** adapters under `harness_evals.benchmarks`.
+- **`fetch_github_json()`** in `dataset_cache` for cached GitHub/URL dataset loading.
+- README **AI Security Benchmarks** section.
+
 ## [0.12.1]
 
 ### Added

README.md

@@ -707,6 +707,130 @@ print(f"HumanEval pass@1: {result.pass_at_1:.2%}")
 
 All benchmarks support: `shots=N` override, `limit=N` for quick runs, `offline=True` for cached-only, `sinks=[...]` for output integration, and `concurrency=N` for throughput control.
 
+## AI Security Benchmarks
+
+Run standardized AI security benchmarks for jailbreak resistance, refusal safety, prompt injection, agent attacks, and secure code generation. Datasets are fetched from HuggingFace Hub or bundled fixtures and cached locally.
+
+```bash
+pip install harness-evals[benchmarks]  # requires httpx
+```
+
+```python
+import asyncio
+from harness_evals.benchmarks import JailbreakBench, DoNotAnswer, OpenPromptInjection
+from harness_evals.llm import OpenAILLM
+
+model = OpenAILLM(model="gpt-4o")
+
+# JailbreakBench — safety pass rate on JBB-Behaviors
+result = asyncio.run(JailbreakBench().run(model, limit=50))
+print(f"Safety pass rate: {result.metrics['safety_pass_rate']:.2%}")
+print(f"Attack success rate: {result.metrics['attack_success_rate']:.2%}")
+
+# Do-Not-Answer — refusal on harmful instructions (CC BY-NC-SA dataset)
+result = asyncio.run(DoNotAnswer().run(model, limit=100))
+
+# OpenPromptInjection — direct/indirect injection ASV
+result = asyncio.run(OpenPromptInjection(attack_mode="direct").run(model))
+```
+
+| Benchmark | Focus | Primary metric | Notes |
+|-----------|-------|----------------|-------|
+| **JailbreakBench** | Jailbreak robustness | `safety_pass_rate` / `attack_success_rate` | HF `JailbreakBench/JBB-Behaviors` |
+| **DoNotAnswer** | Refusal safety | `safety_pass_rate` | 939 harmful instructions |
+| **OpenPromptInjection** | Direct/indirect injection | `injection_success_rate` | GitHub scenarios; bundled smoke default (see below) |
+| **JailBreakV28K** | Large-scale jailbreaks | `attack_success_rate` | Use `limit` for quick runs |
+| **AICGSecEval** | Secure code generation | `secure_pass_rate` | GitHub manifest; bundled smoke default (see below) |
+| **SecCodeBench** | Multi-language secure coding | Pass rate by CWE | Pass `manifest_path=` for full upstream manifest |
+| **AgentDojo** | Agent injection + utility | `utility_pass_rate`, `attack_success_rate` | Pass `tasks_path=` or use `BaseTarget` |
+
+Security benchmark scores use `Score.metadata["dimension"] = "safety"` and are reported separately from quality metrics (see ADR-003). Export goldens with `await benchmark.load_goldens()` for use with `evaluate_dataset()`.
+
+### Running full datasets
+
+Security benchmarks fall into two groups: **HuggingFace-backed** (full upstream data by default) and **fixture-backed** (small bundled JSON under `src/harness_evals/benchmarks/data/` for offline CI smoke tests).
+
+Datasets are cached under `~/.cache/harness_evals/benchmarks/` after the first download. Use `offline=True` on later runs to skip network access.
+
+**Check how many items will run before calling the model:**
+
+```python
+items = asyncio.run(JailbreakBench().load_dataset())
+print(len(items))
+```
+
+#### HuggingFace-backed — full dataset by default
+
+Omit `limit` (or pass `limit=None`) to evaluate the entire dataset:
+
+```python
+from harness_evals.benchmarks import JailbreakBench, DoNotAnswer, JailBreakV28K
+
+# ~100 harmful behaviors (HF JailbreakBench/JBB-Behaviors)
+result = asyncio.run(JailbreakBench().run(model))
+
+# 939 harmful instructions (HF LibrAI/do-not-answer)
+result = asyncio.run(DoNotAnswer().run(model))
+
+# Up to ~28K jailbreak prompts — use limit for cost control
+result = asyncio.run(JailBreakV28K().run(model, limit=1000))
+```
+
+Use `limit=N` only when you want a quick sample run. These benchmarks never read from `benchmarks/data/`.
+
+#### Fixture-backed — smoke vs full upstream
+
+The bundled JSON files contain **2–3 items each** and are used by default for CI-friendly smoke evaluation. To run fuller upstream data:
+
+| Benchmark | Full upstream source | How to run full |
+|-----------|---------------------|-----------------|
+| **OpenPromptInjection** | [Open-Prompt-Injection](https://github.com/liu00222/Open-Prompt-Injection) scenarios on GitHub | Today the bundled file is loaded first when present. Workaround: temporarily rename `benchmarks/data/open_prompt_injection_scenarios.json`, then run normally — `load_dataset()` falls through to GitHub fetch and cache. |
+| **AICGSecEval** | [Tencent/AICGSecEval](https://github.com/Tencent/AICGSecEval) `benchmark/tasks.json` | Same pattern: rename `benchmarks/data/aicg_sec_eval_tasks.json` so GitHub fetch runs, or prefetch with `fetch_github_json()`. |
+| **SecCodeBench** | [alibaba/sec-code-bench](https://github.com/alibaba/sec-code-bench) manifests | Clone upstream and pass the manifest path: |
+| **AgentDojo** | [ethz-spylab/agentdojo](https://github.com/ethz-spylab/agentdojo) suites | Export or author tasks in the bundled JSON shape and pass `tasks_path=`. For the full tool-using benchmark, run upstream AgentDojo directly. |
+
+**SecCodeBench — full upstream manifest:**
+
+```bash
+git clone https://github.com/alibaba/sec-code-bench.git
+```
+
+```python
+from pathlib import Path
+from harness_evals.benchmarks import SecCodeBench
+
+manifest = Path("sec-code-bench/datasets/benchmark/python/python.json")
+bench = SecCodeBench(manifest_path=manifest)
+result = asyncio.run(bench.run(model))
+```
+
+Full SecCodeBench verification requires Docker and the upstream `sec_code_bench.e2e` verifiers. The harness adapter uses heuristics when Docker is unavailable.
+
+**AgentDojo — custom tasks file:**
+
+```python
+from harness_evals.benchmarks import AgentDojo
+
+bench = AgentDojo(tasks_path="/path/to/full_agentdojo_tasks.json")
+result = asyncio.run(bench.run(model=model))  # or target=your_base_target
+```
+
+Each task should include fields such as `user_task`, `injection`, `expected_utility`, `attack_goal`, `suite`, and `attack_type` (see `benchmarks/data/agentdojo_tasks.json` for the schema).
+
+**Prefetch upstream JSON manually (OpenPromptInjection / AICGSecEval):**
+
+```python
+from harness_evals.benchmarks.dataset_cache import fetch_github_json
+
+scenarios = asyncio.run(fetch_github_json(
+    "https://raw.githubusercontent.com/liu00222/Open-Prompt-Injection/main/data/benchmark_scenarios.json",
+    "open_prompt_injection__scenarios",
+))
+print(len(scenarios))
+```
+
+> **Note:** A follow-up will add `scenarios_path` / remote-first loading so bundled fixtures are used only for tests and `offline=True`, without renaming files.
+
 ## Available Metrics
 
 | Category | Metrics | What They Measure |

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "harness-evals"
-version = "0.12.1"
+version = "0.15.0"
 description = "Open-source AI evaluation framework for LLM agents, prompts, and structured outputs"
 readme = "README.md"
 license = "Apache-2.0"
@@ -48,6 +48,7 @@ similarity = ["nltk"]
 harness = ["httpx", "pyjwt"]
 langfuse = ["langfuse"]
 benchmarks = ["httpx"]
+benchmarks-seccode = []
 all = ["openai", "anthropic", "opentelemetry-sdk", "opentelemetry-exporter-otlp-proto-grpc", "opentelemetry-exporter-otlp-proto-http", "nltk", "httpx", "pyjwt", "langfuse"]
 
 [tool.poetry.group.dev.dependencies]

src/harness_evals/benchmarks/__init__.py

@@ -1,28 +1,44 @@
 """Academic benchmark evaluation suites."""
 
+from harness_evals.benchmarks.agentdojo import AgentDojo
+from harness_evals.benchmarks.aicg_sec_eval import AICGSecEval
 from harness_evals.benchmarks.arc import ARC
 from harness_evals.benchmarks.base import BaseBenchmark, BenchmarkResult
 from harness_evals.benchmarks.bbh import BBH
 from harness_evals.benchmarks.boolq import BoolQ
+from harness_evals.benchmarks.do_not_answer import DoNotAnswer
 from harness_evals.benchmarks.drop import DROP
 from harness_evals.benchmarks.gsm8k import GSM8K
 from harness_evals.benchmarks.hellaswag import HellaSwag
 from harness_evals.benchmarks.humaneval import HumanEval
+from harness_evals.benchmarks.jailbreakbench import JailbreakBench
+from harness_evals.benchmarks.jailbreakv_28k import JailBreakV28K
 from harness_evals.benchmarks.mmlu import MMLU
+from harness_evals.benchmarks.open_prompt_injection import OpenPromptInjection
+from harness_evals.benchmarks.sec_code_bench import SecCodeBench
+from harness_evals.benchmarks.security_base import SecurityBenchmark
 from harness_evals.benchmarks.truthfulqa import TruthfulQA
 from harness_evals.benchmarks.winogrande import WinoGrande
 
 __all__ = [
+    "AgentDojo",
+    "AICGSecEval",
     "ARC",
     "BaseBenchmark",
     "BBH",
     "BenchmarkResult",
     "BoolQ",
+    "DoNotAnswer",
     "DROP",
     "GSM8K",
     "HellaSwag",
     "HumanEval",
+    "JailbreakBench",
+    "JailBreakV28K",
     "MMLU",
+    "OpenPromptInjection",
+    "SecCodeBench",
+    "SecurityBenchmark",
     "TruthfulQA",
     "WinoGrande",
 ]