From e8284c755d45668516ac201813bab39457351d5a Mon Sep 17 00:00:00 2001 From: ikrispin Date: Thu, 30 Jul 2026 14:20:26 +0300 Subject: [PATCH 1/3] feat: add Konflux integration for ABEvalFlow evaluation pipeline Add IntegrationTestScenario support so Konflux applications can run ABEvalFlow A/B evaluations as part of their CI pipeline. Includes: - 9 Tekton tasks adapted for Konflux (parse-snapshot, deploy-agent, prepare, test, evaluate, analyze-scorecard, store, emit-result, cleanup-agent) - PipelineRun definition chaining all tasks with cross-cluster agent deployment on a workload cluster - Makefile and GitHub Actions workflow for publishing Tekton Bundles - Secrets template for workload cluster credentials and LLM config - Google Lightspeed Agent submission as initial POC --- .github/workflows/push-bundles.yaml | 50 +++ config/konflux/integration-test-scenario.yaml | 22 + config/konflux/secrets-template.yaml | 37 ++ pipeline/integration/Makefile | 36 ++ .../integration/konflux-eval-pipelinerun.yaml | 319 ++++++++++++++ pipeline/tasks/konflux/analyze-scorecard.yaml | 407 ++++++++++++++++++ pipeline/tasks/konflux/cleanup-agent.yaml | 67 +++ pipeline/tasks/konflux/deploy-agent.yaml | 193 +++++++++ pipeline/tasks/konflux/emit-result.yaml | 95 ++++ pipeline/tasks/konflux/evaluate.yaml | 352 +++++++++++++++ pipeline/tasks/konflux/parse-snapshot.yaml | 73 ++++ pipeline/tasks/konflux/prepare.yaml | 257 +++++++++++ pipeline/tasks/konflux/store.yaml | 248 +++++++++++ pipeline/tasks/konflux/test.yaml | 263 +++++++++++ .../google-lightspeed-agent/metadata.yaml | 30 ++ .../lightspeed-qa/environment/Dockerfile | 13 + .../tasks/lightspeed-qa/instruction.md | 42 ++ .../tasks/lightspeed-qa/task.toml | 29 ++ .../tasks/lightspeed-qa/tests/llm_judge.py | 149 +++++++ .../tasks/lightspeed-qa/tests/test.sh | 40 ++ 20 files changed, 2722 insertions(+) create mode 100644 .github/workflows/push-bundles.yaml create mode 100644 config/konflux/integration-test-scenario.yaml create mode 100644 config/konflux/secrets-template.yaml create mode 100644 pipeline/integration/Makefile create mode 100644 pipeline/integration/konflux-eval-pipelinerun.yaml create mode 100644 pipeline/tasks/konflux/analyze-scorecard.yaml create mode 100644 pipeline/tasks/konflux/cleanup-agent.yaml create mode 100644 pipeline/tasks/konflux/deploy-agent.yaml create mode 100644 pipeline/tasks/konflux/emit-result.yaml create mode 100644 pipeline/tasks/konflux/evaluate.yaml create mode 100644 pipeline/tasks/konflux/parse-snapshot.yaml create mode 100644 pipeline/tasks/konflux/prepare.yaml create mode 100644 pipeline/tasks/konflux/store.yaml create mode 100644 pipeline/tasks/konflux/test.yaml create mode 100644 submissions/google-lightspeed-agent/metadata.yaml create mode 100644 submissions/google-lightspeed-agent/tasks/lightspeed-qa/environment/Dockerfile create mode 100644 submissions/google-lightspeed-agent/tasks/lightspeed-qa/instruction.md create mode 100644 submissions/google-lightspeed-agent/tasks/lightspeed-qa/task.toml create mode 100755 submissions/google-lightspeed-agent/tasks/lightspeed-qa/tests/llm_judge.py create mode 100755 submissions/google-lightspeed-agent/tasks/lightspeed-qa/tests/test.sh diff --git a/.github/workflows/push-bundles.yaml b/.github/workflows/push-bundles.yaml new file mode 100644 index 0000000..9832fb8 --- /dev/null +++ b/.github/workflows/push-bundles.yaml @@ -0,0 +1,50 @@ +name: Push Tekton Bundles + +on: + push: + branches: [main] + paths: + - 'pipeline/tasks/konflux/**' + - 'pipeline/integration/Makefile' + workflow_dispatch: + inputs: + version: + description: 'Bundle version tag' + required: false + default: '0.1' + +env: + QUAY_REPO: quay.io/rh-ee-ikrispin/abevalflow-catalog + VERSION: ${{ github.event.inputs.version || '0.1' }} + +jobs: + push-bundles: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install tkn CLI + run: | + TKN_VERSION="0.37.0" + curl -sLO "https://github.com/tektoncd/cli/releases/download/v${TKN_VERSION}/tkn_${TKN_VERSION}_Linux_x86_64.tar.gz" + tar xzf "tkn_${TKN_VERSION}_Linux_x86_64.tar.gz" tkn + sudo mv tkn /usr/local/bin/ + tkn version + + - name: Install skopeo + run: sudo apt-get update && sudo apt-get install -y skopeo + + - name: Login to Quay.io + run: | + echo "${{ secrets.QUAY_TOKEN }}" | tkn bundle push --help > /dev/null 2>&1 || true + mkdir -p ~/.docker + echo '{"auths":{"quay.io":{"auth":"'$(echo -n "${{ secrets.QUAY_USERNAME }}:${{ secrets.QUAY_TOKEN }}" | base64 -w0)'"}}}' > ~/.docker/config.json + + - name: Push all bundles + working-directory: pipeline/integration + run: make bundles VERSION=${{ env.VERSION }} + + - name: Print digests + working-directory: pipeline/integration + run: make digests VERSION=${{ env.VERSION }} diff --git a/config/konflux/integration-test-scenario.yaml b/config/konflux/integration-test-scenario.yaml new file mode 100644 index 0000000..2ce4f5b --- /dev/null +++ b/config/konflux/integration-test-scenario.yaml @@ -0,0 +1,22 @@ +apiVersion: appstudio.redhat.com/v1beta2 +kind: IntegrationTestScenario +metadata: + name: abevalflow-eval + namespace: ai5-marketplace-tenant + labels: + test.appstudio.openshift.io/optional: "true" +spec: + application: google-lightspeed-agent + contexts: + - description: AI agent evaluation via ABEvalFlow + name: application + resolverRef: + resolver: git + resourceKind: pipelinerun + params: + - name: url + value: https://github.com/ikrispin/ABEvalFlow + - name: revision + value: main + - name: pathInRepo + value: pipeline/integration/konflux-eval-pipelinerun.yaml diff --git a/config/konflux/secrets-template.yaml b/config/konflux/secrets-template.yaml new file mode 100644 index 0000000..80e7616 --- /dev/null +++ b/config/konflux/secrets-template.yaml @@ -0,0 +1,37 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: workload-cluster-credentials + namespace: ai5-marketplace-tenant +type: Opaque +stringData: + # Token for the abevalflow-deployer ServiceAccount on the workload cluster. + # Create the SA and token on the workload cluster: + # oc create sa abevalflow-deployer -n ab-eval-flow + # oc adm policy add-role-to-user edit -z abevalflow-deployer -n ab-eval-flow + # oc create token abevalflow-deployer -n ab-eval-flow --duration=8760h + token: "" + server: "https://api.cn-ai-lab.2vn8.p1.openshiftapps.com:6443" +--- +apiVersion: v1 +kind: Secret +metadata: + name: llm-credentials + namespace: ai5-marketplace-tenant +type: Opaque +stringData: + # LLM API key. When using LiteLLM proxy (which handles real auth to + # Vertex AI), set this to "sk-dummy". + api-key: "" +--- +apiVersion: v1 +kind: Secret +metadata: + name: compass-facts-api + namespace: ai5-marketplace-tenant +type: Opaque +stringData: + # Bearer token for the Compass Soundcheck Facts API. + # Optional — if not configured, scorecard is computed but facts are not pushed. + token: "" diff --git a/pipeline/integration/Makefile b/pipeline/integration/Makefile new file mode 100644 index 0000000..71f5311 --- /dev/null +++ b/pipeline/integration/Makefile @@ -0,0 +1,36 @@ +QUAY_NS ?= quay.io/rh-ee-ikrispin +VERSION ?= 0.1 +TASKS_DIR := ../tasks/konflux + +TASKS = parse-snapshot deploy-agent prepare test evaluate analyze-scorecard store emit-result cleanup-agent + +.PHONY: bundles login list clean help + +help: ## Show this help + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' + +bundles: $(addprefix bundle-,$(TASKS)) ## Build and push all Tekton Bundles + +bundle-%: ## Push a single task bundle (e.g. make bundle-parse-snapshot) + @echo "=== Pushing abevalflow-task-$* ===" + tkn bundle push $(QUAY_NS)/abevalflow-task-$*:$(VERSION) \ + -f $(TASKS_DIR)/$*.yaml + @echo "" + +list: ## List all bundles in the registry + @for task in $(TASKS); do \ + echo "--- abevalflow-task-$$task ---"; \ + tkn bundle list $(QUAY_NS)/abevalflow-task-$$task 2>/dev/null || echo " (not found)"; \ + done + +digests: ## Print SHA digests for all pushed bundles + @for task in $(TASKS); do \ + DIGEST=$$(skopeo inspect --format '{{.Digest}}' docker://$(QUAY_NS)/abevalflow-task-$$task:$(VERSION) 2>/dev/null || echo "NOT_FOUND"); \ + echo "$(QUAY_NS)/abevalflow-task-$$task:$(VERSION)@$$DIGEST"; \ + done + +clean: ## Remove local tkn bundle cache + rm -rf ~/.cache/tekton/bundles/ + +login: ## Login to Quay.io + podman login quay.io diff --git a/pipeline/integration/konflux-eval-pipelinerun.yaml b/pipeline/integration/konflux-eval-pipelinerun.yaml new file mode 100644 index 0000000..ca7a34c --- /dev/null +++ b/pipeline/integration/konflux-eval-pipelinerun.yaml @@ -0,0 +1,319 @@ +# NOTE: Dual-publish requirement +# This PipelineRun is resolved from git (via IntegrationTestScenario), but +# the tasks it references are published as Tekton Bundles to Quay.io. +# When updating task logic: +# 1. Edit task YAML in pipeline/tasks/konflux/ +# 2. Push bundles: cd pipeline/integration && make bundles +# 3. Commit and push to git +# Pushing to git alone does NOT update the task logic inside bundles. +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + name: abevalflow-eval +spec: + timeouts: + pipeline: "4h" + tasks: "3h" + pipelineSpec: + params: + - name: SNAPSHOT + type: string + description: Konflux Snapshot JSON with component details + - name: AGENT_IMAGE_OVERRIDE + type: string + default: "" + description: >- + Override the agent image instead of using the one from the Konflux + snapshot. Set this when the snapshot image is in a private registry + that the workload cluster cannot pull from. + workspaces: + - name: shared-workspace + results: + - name: TEST_OUTPUT + description: Standardized Konflux test output + value: $(tasks.emit-result.results.TEST_OUTPUT) + tasks: + # ================================================================ + # Stage 1: Parse the Konflux Snapshot + # ================================================================ + - name: parse-snapshot + taskRef: + resolver: bundles + params: + - name: name + value: parse-snapshot + - name: bundle + value: quay.io/rh-ee-ikrispin/abevalflow-task-parse-snapshot:0.1 + - name: kind + value: task + params: + - name: SNAPSHOT + value: $(params.SNAPSHOT) + + # ================================================================ + # Stage 2: Deploy the agent from the Snapshot image + # ================================================================ + - name: deploy-agent + runAfter: [parse-snapshot] + taskRef: + resolver: bundles + params: + - name: name + value: deploy-agent + - name: bundle + value: quay.io/rh-ee-ikrispin/abevalflow-task-deploy-agent:0.1 + - name: kind + value: task + params: + - name: agent-image + value: $(tasks.parse-snapshot.results.component-image) + - name: agent-image-override + value: $(params.AGENT_IMAGE_OVERRIDE) + - name: llm-api-base + value: "http://litellm.ab-eval-flow.svc:4000" + - name: llm-model + value: "gpt-4o" + - name: pipeline-run-id + value: $(context.pipelineRun.name) + + # ================================================================ + # Stage 3: Prepare (clone + validate submission) + # ================================================================ + - name: prepare + runAfter: [deploy-agent] + taskRef: + resolver: bundles + params: + - name: name + value: prepare + - name: bundle + value: quay.io/rh-ee-ikrispin/abevalflow-task-prepare:0.1 + - name: kind + value: task + params: + - name: repo-url + value: https://github.com/ikrispin/ABEvalFlow.git + - name: revision + value: main + - name: submission-dir + value: google-lightspeed-agent + - name: eval-engine + value: a2a + - name: pipeline-repo-url + value: https://github.com/ikrispin/ABEvalFlow.git + - name: pipeline-repo-revision + value: main + - name: pipeline-run-name + value: $(context.pipelineRun.name) + - name: enable-generation + value: "false" + workspaces: + - name: source + workspace: shared-workspace + + # ================================================================ + # Stage 4: Test (security scan + quality review) + # ================================================================ + - name: test + runAfter: [prepare] + taskRef: + resolver: bundles + params: + - name: name + value: test + - name: bundle + value: quay.io/rh-ee-ikrispin/abevalflow-task-test:0.1 + - name: kind + value: task + params: + - name: submission-dir + value: google-lightspeed-agent + - name: submission-name + value: $(tasks.prepare.results.submission-name) + - name: eval-engine + value: a2a + - name: report-prefix + value: $(tasks.prepare.results.report-prefix) + - name: pipeline-run-name + value: $(context.pipelineRun.name) + - name: pipeline-repo-url + value: https://github.com/ikrispin/ABEvalFlow.git + - name: pipeline-repo-revision + value: main + - name: security-scan-mode + value: disabled + - name: submission-security-scan + value: $(tasks.prepare.results.security-scan) + - name: submission-skip-quality-review + value: $(tasks.prepare.results.skip-quality-review) + - name: enable-quality-review + value: "false" + - name: llm-base-url + value: "http://litellm.ab-eval-flow.svc:4000/v1" + workspaces: + - name: source + workspace: shared-workspace + + # ================================================================ + # Stage 5: Evaluate (A2A agent evaluation) + # ================================================================ + - name: evaluate + runAfter: [test] + timeout: "3h" + taskRef: + resolver: bundles + params: + - name: name + value: evaluate + - name: bundle + value: quay.io/rh-ee-ikrispin/abevalflow-task-evaluate:0.1 + - name: kind + value: task + params: + - name: eval-engine + value: a2a + - name: submission-dir + value: google-lightspeed-agent + - name: submission-name + value: $(tasks.prepare.results.submission-name) + - name: commit-sha + value: $(tasks.parse-snapshot.results.git-revision) + - name: pipeline-run-id + value: $(context.pipelineRun.name) + - name: pipeline-repo-url + value: https://github.com/ikrispin/ABEvalFlow.git + - name: pipeline-repo-revision + value: main + - name: llm-model + value: "claude-sonnet" + - name: llm-api-base + value: "http://litellm.ab-eval-flow.svc:4000" + - name: agent-endpoint + value: $(tasks.deploy-agent.results.agent-endpoint) + - name: agent-timeout + value: "120" + workspaces: + - name: source + workspace: shared-workspace + + # ================================================================ + # Stage 6: Analyze + Scorecard + # ================================================================ + - name: analyze-scorecard + runAfter: [evaluate] + taskRef: + resolver: bundles + params: + - name: name + value: analyze-scorecard + - name: bundle + value: quay.io/rh-ee-ikrispin/abevalflow-task-analyze-scorecard:0.1 + - name: kind + value: task + params: + - name: submission-name + value: $(tasks.prepare.results.submission-name) + - name: eval-engine + value: a2a + - name: commit-sha + value: $(tasks.parse-snapshot.results.git-revision) + - name: pipeline-run-id + value: $(context.pipelineRun.name) + - name: uplift-threshold + value: "0.0" + - name: pipeline-repo-url + value: https://github.com/ikrispin/ABEvalFlow.git + - name: pipeline-repo-revision + value: main + - name: enable-scorecard + value: "true" + - name: enable-degradation-check + value: "false" + workspaces: + - name: source + workspace: shared-workspace + + # ================================================================ + # Stage 7: Store (optional — graceful when secrets missing) + # ================================================================ + - name: store + runAfter: [analyze-scorecard] + taskRef: + resolver: bundles + params: + - name: name + value: store + - name: bundle + value: quay.io/rh-ee-ikrispin/abevalflow-task-store:0.1 + - name: kind + value: task + params: + - name: submission-name + value: $(tasks.prepare.results.submission-name) + - name: pipeline-run-id + value: $(context.pipelineRun.name) + - name: report-prefix + value: $(tasks.prepare.results.report-prefix) + - name: recommendation + value: $(tasks.evaluate.results.recommendation) + - name: eval-engine + value: a2a + - name: commit-sha + value: $(tasks.parse-snapshot.results.git-revision) + - name: pipeline-repo-url + value: https://github.com/ikrispin/ABEvalFlow.git + - name: pipeline-repo-revision + value: main + workspaces: + - name: source + workspace: shared-workspace + + # ================================================================ + # Stage 8: Emit Konflux TEST_OUTPUT + # ================================================================ + - name: emit-result + runAfter: [store] + taskRef: + resolver: bundles + params: + - name: name + value: emit-result + - name: bundle + value: quay.io/rh-ee-ikrispin/abevalflow-task-emit-result:0.1 + - name: kind + value: task + params: + - name: submission-name + value: $(tasks.prepare.results.submission-name) + workspaces: + - name: source + workspace: shared-workspace + + # ================================================================== + # FINALLY: Cleanup agent deployment + # ================================================================== + finally: + - name: cleanup-agent + taskRef: + resolver: bundles + params: + - name: name + value: cleanup-agent + - name: bundle + value: quay.io/rh-ee-ikrispin/abevalflow-task-cleanup-agent:0.1 + - name: kind + value: task + params: + - name: agent-name + value: $(tasks.deploy-agent.results.agent-name) + - name: deployed + value: $(tasks.deploy-agent.results.deployed) + + workspaces: + - name: shared-workspace + volumeClaimTemplate: + spec: + accessModes: [ReadWriteOnce] + resources: + requests: + storage: 1Gi diff --git a/pipeline/tasks/konflux/analyze-scorecard.yaml b/pipeline/tasks/konflux/analyze-scorecard.yaml new file mode 100644 index 0000000..f2e4002 --- /dev/null +++ b/pipeline/tasks/konflux/analyze-scorecard.yaml @@ -0,0 +1,407 @@ +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: analyze-scorecard + labels: + app.kubernetes.io/name: abevalflow + app.kubernetes.io/component: konflux +spec: + description: >- + Analyzes A/B evaluation results and optionally checks for performance + degradation against historical runs. Step 1 always runs analyze.py to + produce report.json and Tekton results. Step 2 runs aggregate scorecard. + Step 3 runs monitor.py and sends Slack alerts when degradation checking + is enabled and the DB is configured; failures in step 3 are non-blocking. + Adapted for Konflux — no hardcoded namespace, optional secrets, scorecard + enabled by default. + params: + - name: submission-name + type: string + description: Validated submission name + - name: eval-engine + type: string + default: "harbor" + description: >- + Evaluation engine used: 'harbor', 'ase', 'a2a', or 'both'. Controls + which analysis path runs. + - name: commit-sha + type: string + default: "" + description: Git commit SHA for provenance + - name: pipeline-run-id + type: string + default: "" + description: Tekton PipelineRun name for provenance and alerts + - name: harbor-fork-revision + type: string + default: "" + description: Harbor fork revision used for provenance + - name: uplift-threshold + type: string + default: "0.0" + description: >- + Minimum mean-reward gap (treatment - control) for a 'pass' + recommendation. Set to 0.0 to pass whenever treatment >= control. + - name: pipeline-repo-url + type: string + default: "https://github.com/ikrispin/ABEvalFlow.git" + description: URL of the ABEvalFlow pipeline repository + - name: pipeline-repo-revision + type: string + default: "main" + description: Branch or SHA of the pipeline repo to use for scripts + - name: security-scan-mode + type: string + default: "disabled" + description: Security scan mode (disabled/warn/block) for including scan results + - name: llm-model + type: string + default: "" + description: LLM model name for report metadata + - name: repo-name + type: string + default: "" + description: GitHub repo (org/name) for constructing PR URL + - name: pr-number + type: string + default: "" + description: PR number that triggered this pipeline run + - name: enable-degradation-check + type: string + default: "false" + description: >- + When 'true', run historical degradation check after analysis (monitoring + pipelines). CI pipelines leave this false. + - name: degradation-threshold + type: string + default: "0.85" + description: >- + Degradation threshold as a ratio. Alert if current/previous < threshold. + - name: openshift-console-url + type: string + default: "https://console-openshift-console.apps.cn-ai-lab.2vn8.p1.openshiftapps.com" + description: Base URL for OpenShift console (for Slack message links) + - name: enable-scorecard + type: string + default: "true" + description: >- + When 'true', aggregate all gates into a unified scorecard after analysis. + Produces scorecard.json with combined engine, security, and quality gates. + - name: certification-profile + type: string + default: "" + description: >- + Certification profile name (e.g., 'skill', 'agent', 'mcp_server', 'plugin'). + Provides artifact-type-specific check defaults. If empty, uses hardcoded defaults + unless submission's metadata.yaml specifies certification_policy. + workspaces: + - name: source + description: Workspace containing the evaluation results and report output + results: + - name: recommendation + description: "'pass' or 'fail' based on uplift threshold" + - name: treatment-mean-reward + description: Treatment variant mean reward as a decimal string (e.g. "0.8500") + - name: control-mean-reward + description: Control variant mean reward as a decimal string (e.g. "0.6000") + - name: mean-reward-gap + description: "Mean reward gap (treatment - control) as a signed string (e.g. \"+0.1500\")" + - name: ttest-p-value + description: "Welch's t-test p-value (e.g. \"0.0342\"), or \"N/A\" if not computable" + - name: fisher-p-value + description: "Fisher's exact test p-value (e.g. \"0.0271\"), or \"N/A\" if not computable" + - name: report-path + description: Path to the directory containing report.json and report.md + - name: degraded + description: Whether degradation was detected (true or false) + - name: message + description: Human-readable degradation check result message + - name: scorecard-recommendation + description: Unified scorecard recommendation (pass/warn/fail) + - name: scorecard-gates-passed + description: Number of gates that passed + - name: scorecard-gates-failed + description: Number of gates that failed + steps: + - name: analyze + image: registry.access.redhat.com/ubi9/python-311:9.6 + script: | + #!/usr/bin/env bash + set -euo pipefail + + # --- 1. Clone pipeline repo (or update to requested revision) --- + PIPELINE_DIR="$(workspaces.source.path)/_pipeline" + if [ -d "$PIPELINE_DIR" ]; then + echo "Pipeline repo exists, updating to requested revision" + git -C "$PIPELINE_DIR" fetch origin "$(params.pipeline-repo-revision)" --depth 1 + git -C "$PIPELINE_DIR" -c advice.detachedHead=false checkout FETCH_HEAD + else + git clone --depth 1 --branch "$(params.pipeline-repo-revision)" \ + "$(params.pipeline-repo-url)" "$PIPELINE_DIR" + fi + + cd "$PIPELINE_DIR" + export PYTHONPATH="$PIPELINE_DIR" + pip install --quiet --no-cache-dir pydantic scipy pyyaml + + # --- 2. Use pre-computed report or run analysis --- + REPORT_DIR="$(workspaces.source.path)/reports/$(params.submission-name)" + RESULTS_DIR="$(workspaces.source.path)/eval-results/$(params.submission-name)" + EVAL_ENGINE="$(params.eval-engine)" + mkdir -p "$REPORT_DIR" + + if [ -f "$REPORT_DIR/report.json" ]; then + echo "=== report.json already exists (computed by remote eval Pod) ===" + python3 -c " + import json, sys + r = json.load(open(sys.argv[1])) + s = r.get('summary', {}) + t = s.get('treatment', {}) + print(f\"Recommendation: {s.get('recommendation', 'N/A')}\") + print(f\"Mean reward: {t.get('mean_reward', 'N/A')}\") + print(f\"Trials: {t.get('n_trials', 0)}\") + print(f\"Pass rate: {t.get('pass_rate', 'N/A')}\") + " "$REPORT_DIR/report.json" + else + echo "=== No pre-computed report, running analyze.py locally ===" + ARGS=( + --results-dir "$RESULTS_DIR" + --output-dir "$REPORT_DIR" + --submission-name "$(params.submission-name)" + --threshold "$(params.uplift-threshold)" + --eval-engine "$EVAL_ENGINE" + ) + [ -n "$(params.commit-sha)" ] && ARGS+=(--commit-sha "$(params.commit-sha)") + [ -n "$(params.pipeline-run-id)" ] && ARGS+=(--pipeline-run-id "$(params.pipeline-run-id)") + [ -n "$(params.harbor-fork-revision)" ] && ARGS+=(--harbor-fork-revision "$(params.harbor-fork-revision)") + python scripts/analyze.py "${ARGS[@]}" + fi + + # --- 3. Enrich report with PR/LLM metadata (ASE mode) --- + PR_URL="" + if [ -n "$(params.repo-name)" ] && [ -n "$(params.pr-number)" ]; then + PR_URL="https://github.com/$(params.repo-name)/pull/$(params.pr-number)" + fi + LLM_LABEL="" + case "$(params.llm-model)" in + claude-sonnet*) LLM_LABEL="Claude Sonnet 4.6 (vertex_ai)" ;; + claude-haiku*) LLM_LABEL="Claude Haiku 3.5 (vertex_ai)" ;; + ?*) LLM_LABEL="$(params.llm-model)" ;; + esac + + if [ "$EVAL_ENGINE" = "ase" ] && { [ -n "$PR_URL" ] || [ -n "$LLM_LABEL" ]; }; then + echo "Enriching ASE report with PR/LLM metadata..." + python3 -c "import json;from pathlib import Path;import sys;p=Path(sys.argv[1])/'report.json';r=json.loads(p.read_text());r.setdefault('summary',{});sys.argv[2] and r['summary'].update({'related_pr':sys.argv[2]});sys.argv[3] and r['summary'].update({'llm':sys.argv[3]});p.write_text(json.dumps(r,indent=2))" "$REPORT_DIR" "$PR_URL" "$LLM_LABEL" + fi + + # --- 4. Enrich with security scan results (ASE/both only) --- + if [ "$EVAL_ENGINE" != "harbor" ] && [ "$(params.security-scan-mode)" != "disabled" ] && [ -f "$REPORT_DIR/security-scan.json" ]; then + echo "Enriching report with security scan results..." + python3 -c "import json;from pathlib import Path;import sys;d=Path(sys.argv[1]);m=sys.argv[2];r=json.loads((d/'report.json').read_text());s=json.loads((d/'security-scan.json').read_text());f=[{'rule_id':x.get('rule_id','?'),'severity':x.get('severity','info').lower(),'message':x.get('message',''),'scanner':'cisco'} for x in s.get('findings',[])];p=m!='block' or sum(1 for x in f if x['severity'] in('critical','high'))==0;r.setdefault('security_scans',[]).append({'scanner':'cisco','scan_mode':m,'findings':f,'passed':p});(d/'report.json').write_text(json.dumps(r,indent=2));print(f'Added {len(f)} findings')" "$REPORT_DIR" "$(params.security-scan-mode)" + fi + + # --- 5. Regenerate markdown from enriched JSON (ASE/both only) --- + if [ "$EVAL_ENGINE" != "harbor" ] && [ "$EVAL_ENGINE" != "a2a" ]; then + echo "Regenerating report.md from enriched JSON..." + python3 -c "import sys;sys.path.insert(0,'$PIPELINE_DIR');from pathlib import Path;from abevalflow.report import AnalysisResult;from scripts.analyze import render_markdown;d=Path(sys.argv[1]);r=AnalysisResult.model_validate_json((d/'report.json').read_text());(d/'report.md').write_text(render_markdown(r))" "$REPORT_DIR" + fi + + # --- 6. Extract results for Tekton --- + echo -n "$REPORT_DIR" > "$(results.report-path.path)" + + python3 -c "import json,sys;r=json.loads(open(sys.argv[1]).read());s=r['summary'];open(sys.argv[2],'w').write(s['recommendation']);t,c=s['treatment'].get('mean_reward'),s['control'].get('mean_reward');open(sys.argv[3],'w').write(f'{t:.4f}' if t is not None else 'N/A');open(sys.argv[4],'w').write(f'{c:.4f}' if c is not None else 'N/A');g=s.get('mean_reward_gap');open(sys.argv[5],'w').write(f'{g:+.4f}' if g is not None else 'N/A');tt,fi=s.get('ttest_p_value'),s.get('fisher_p_value');open(sys.argv[6],'w').write(f'{tt:.4f}' if tt is not None else 'N/A');open(sys.argv[7],'w').write(f'{fi:.4f}' if fi is not None else 'N/A')" \ + "$REPORT_DIR/report.json" \ + "$(results.recommendation.path)" \ + "$(results.treatment-mean-reward.path)" \ + "$(results.control-mean-reward.path)" \ + "$(results.mean-reward-gap.path)" \ + "$(results.ttest-p-value.path)" \ + "$(results.fisher-p-value.path)" + + echo "=== Analysis complete ===" + echo "Report: $REPORT_DIR" + cat "$REPORT_DIR/report.md" 2>/dev/null || echo "(report.md not available)" + + - name: aggregate-scorecard + image: registry.access.redhat.com/ubi9/python-311:9.6 + env: + - name: COMPASS_API_TOKEN + valueFrom: + secretKeyRef: + name: compass-facts-api + key: token + optional: true + script: | + #!/usr/bin/env bash + set -euo pipefail + + ENABLE_SCORECARD="$(params.enable-scorecard)" + if [ "$ENABLE_SCORECARD" != "true" ]; then + echo "Scorecard aggregation disabled, skipping" + echo -n "skipped" > "$(results.scorecard-recommendation.path)" + echo -n "0" > "$(results.scorecard-gates-passed.path)" + echo -n "0" > "$(results.scorecard-gates-failed.path)" + exit 0 + fi + + echo "=== Aggregating Unified Scorecard ===" + + PIPELINE_DIR="$(workspaces.source.path)/_pipeline" + cd "$PIPELINE_DIR" + export PYTHONPATH="$PIPELINE_DIR" + pip install --quiet --no-cache-dir pydantic pyyaml + + SUBMISSION_DIR="$(workspaces.source.path)/submissions/$(params.submission-name)" + RESULTS_DIR="$(workspaces.source.path)/eval-results/$(params.submission-name)" + REPORT_DIR="$(workspaces.source.path)/reports/$(params.submission-name)" + WORKSPACE_ROOT="$(workspaces.source.path)" + + SCORECARD_ARGS=( + --submission-dir "$SUBMISSION_DIR" + --results-dir "$RESULTS_DIR" + --reports-dir "$REPORT_DIR" + --workspace-root "$WORKSPACE_ROOT" + --eval-engine "$(params.eval-engine)" + --pipeline-run-id "$(params.pipeline-run-id)" + ) + if [ -n "$(params.certification-profile)" ]; then + SCORECARD_ARGS+=(--certification-profile "$(params.certification-profile)") + fi + python scripts/aggregate_scorecard.py "${SCORECARD_ARGS[@]}" + + if [ -f "$REPORT_DIR/scorecard.json" ]; then + echo "" + echo "Scorecard written to $REPORT_DIR/scorecard.json" + python3 -m json.tool "$REPORT_DIR/scorecard.json" 2>/dev/null | head -30 || true + echo "..." + + python3 -c "import json,sys;sc=json.load(open(sys.argv[1]));open(sys.argv[2],'w').write(sc['recommendation']);open(sys.argv[3],'w').write(str(sc['gates_passed']));open(sys.argv[4],'w').write(str(sc['gates_failed']))" \ + "$REPORT_DIR/scorecard.json" \ + "$(results.scorecard-recommendation.path)" \ + "$(results.scorecard-gates-passed.path)" \ + "$(results.scorecard-gates-failed.path)" + else + echo "WARNING: scorecard.json not created" + echo -n "error" > "$(results.scorecard-recommendation.path)" + echo -n "0" > "$(results.scorecard-gates-passed.path)" + echo -n "0" > "$(results.scorecard-gates-failed.path)" + fi + + echo "=== Scorecard aggregation complete ===" + + - name: check-degradation + image: registry.access.redhat.com/ubi9/python-311:9.6 + env: + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: ab-eval-db-credentials + key: database-url + optional: true + - name: SLACK_WEBHOOK_URL + valueFrom: + secretKeyRef: + name: monitoring-slack-webhook + key: webhook-url + optional: true + script: | + #!/usr/bin/env bash + set -euo pipefail + + write_skip_results() { + echo "false" > "$(results.degraded.path)" + echo "$1" > "$(results.message.path)" + } + + if [ "$(params.enable-degradation-check)" != "true" ]; then + echo "Degradation check disabled (enable-degradation-check != true)" + write_skip_results "Degradation check disabled" + exit 0 + fi + + if [ -z "${DATABASE_URL:-}" ]; then + echo "WARNING: DATABASE_URL not configured, skipping degradation check" + write_skip_results "DB not configured" + exit 0 + fi + + PIPELINE_DIR="$(workspaces.source.path)/_pipeline" + if [ ! -d "$PIPELINE_DIR" ]; then + echo "Pipeline repo missing after analyze step, skipping degradation check" + write_skip_results "Pipeline repo unavailable" + exit 0 + fi + + echo "=== Degradation Check ===" + echo "Submission: $(params.submission-name)" + echo "Pipeline Run: $(params.pipeline-run-id)" + echo "Threshold: $(params.degradation-threshold)" + + pip install --quiet --no-cache-dir sqlalchemy "psycopg[binary]" "tenacity>=8.2" scipy pydantic pyyaml + + cd "$PIPELINE_DIR" + export PYTHONPATH="$PIPELINE_DIR" + + MONITOR_OUTPUT=/tmp/monitor_result.json + + REPORT_DIR="$(workspaces.source.path)/reports/$(params.submission-name)" + CURRENT_SCORE=$(python3 -c "import json,sys;r=json.load(open(sys.argv[1]));s=r.get('summary',{});t=s.get('treatment',{}).get('mean_reward');print(t if t is not None else s.get('treatment_pass_rate',0.0))" "$REPORT_DIR/report.json" 2>/dev/null || echo "0.0") + + MONITOR_EXIT=0 + python scripts/monitor.py \ + --submission-name "$(params.submission-name)" \ + --threshold "$(params.degradation-threshold)" \ + --db-url "$DATABASE_URL" \ + --current-score "$CURRENT_SCORE" \ + --eval-engine "$(params.eval-engine)" \ + --run-id "$(params.pipeline-run-id)" \ + --output "$MONITOR_OUTPUT" || MONITOR_EXIT=$? + if [ "$MONITOR_EXIT" -eq 2 ]; then + echo "Monitor errored (non-blocking), continuing" + write_skip_results "Monitor errored" + exit 0 + fi + + cat "$MONITOR_OUTPUT" + + if [ -f "$REPORT_DIR/report.json" ]; then + echo "Merging degradation results into report.json..." + python scripts/analyze.py \ + --merge-degradation-from "$MONITOR_OUTPUT" \ + --report-json "$REPORT_DIR/report.json" + else + echo "WARNING: report.json not found at $REPORT_DIR/report.json, skipping merge" + fi + + DEGRADED=$(python3 -c "import json; print(json.load(open('$MONITOR_OUTPUT'))['degraded'])") + MESSAGE=$(python3 -c "import json; print(json.load(open('$MONITOR_OUTPUT'))['message'])") + + echo "$DEGRADED" > "$(results.degraded.path)" + echo "$MESSAGE" > "$(results.message.path)" + + if [ "$DEGRADED" = "True" ]; then + echo "" + echo "!!! DEGRADATION DETECTED !!!" + echo "" + fi + + if [ -n "${SLACK_WEBHOOK_URL:-}" ] && [ "$SLACK_WEBHOOK_URL" != "https://hooks.slack.com/services/REPLACE/WITH/ACTUAL_WEBHOOK" ]; then + PIPELINE_RUN_URL="$(params.openshift-console-url)/k8s/ns/ab-eval-flow/tekton.dev~v1~PipelineRun/$(params.pipeline-run-id)" + + python scripts/alert.py \ + --payload "$MONITOR_OUTPUT" \ + --webhook-url "$SLACK_WEBHOOK_URL" \ + --pipeline-run-url "$PIPELINE_RUN_URL" \ + --eval-engine "$(params.eval-engine)" || { + echo "Failed to send Slack alert (continuing anyway)" + } + else + echo "Slack webhook not configured, skipping alert" + fi + + if [ "$DEGRADED" != "True" ]; then + echo "No degradation detected" + fi + + echo "" + echo "=== Check complete ===" diff --git a/pipeline/tasks/konflux/cleanup-agent.yaml b/pipeline/tasks/konflux/cleanup-agent.yaml new file mode 100644 index 0000000..78e8f14 --- /dev/null +++ b/pipeline/tasks/konflux/cleanup-agent.yaml @@ -0,0 +1,67 @@ +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: cleanup-agent + labels: + app.kubernetes.io/name: abevalflow + app.kubernetes.io/component: konflux +spec: + description: >- + Cleans up the A2A agent Deployment and Service on the remote workload + cluster. Runs in the pipeline's finally block to ensure cleanup even + on failure. + params: + - name: agent-name + type: string + default: "" + description: Name of the Deployment/Service to delete + - name: deployed + type: string + default: "false" + description: Whether an agent was actually deployed + - name: workload-cluster-url + type: string + default: "https://api.cn-ai-lab.2vn8.p1.openshiftapps.com:6443" + description: API URL of the workload cluster + - name: workload-namespace + type: string + default: "ab-eval-flow" + description: Namespace on the workload cluster + steps: + - name: cleanup + image: registry.redhat.io/openshift4/ose-cli:latest + env: + - name: WORKLOAD_TOKEN + valueFrom: + secretKeyRef: + name: workload-cluster-credentials + key: token + optional: true + script: | + #!/usr/bin/env bash + + AGENT_NAME="$(params.agent-name)" + DEPLOYED="$(params.deployed)" + + if [ -z "$AGENT_NAME" ] || [ "$DEPLOYED" != "true" ]; then + echo "No agent to clean up (deployed=$DEPLOYED, name=$AGENT_NAME)" + exit 0 + fi + + if [ -z "${WORKLOAD_TOKEN:-}" ]; then + echo "WARNING: No workload cluster token, cannot clean up" + exit 0 + fi + + echo "=== CLEANUP AGENT (cross-cluster) ===" + CLUSTER_URL="$(params.workload-cluster-url)" + NAMESPACE="$(params.workload-namespace)" + # TODO: Replace --insecure-skip-tls-verify with --certificate-authority + oc login --server="$CLUSTER_URL" --token="$WORKLOAD_TOKEN" --insecure-skip-tls-verify=true 2>/dev/null + + echo "Deleting: $AGENT_NAME in $NAMESPACE on $CLUSTER_URL" + oc delete deployment/$AGENT_NAME -n $NAMESPACE --ignore-not-found=true + oc delete service/$AGENT_NAME -n $NAMESPACE --ignore-not-found=true + oc delete pod -n $NAMESPACE -l abevalflow/temp=true,abevalflow/type=eval-job --ignore-not-found=true + + echo "Cleanup complete" diff --git a/pipeline/tasks/konflux/deploy-agent.yaml b/pipeline/tasks/konflux/deploy-agent.yaml new file mode 100644 index 0000000..4e5c63e --- /dev/null +++ b/pipeline/tasks/konflux/deploy-agent.yaml @@ -0,0 +1,193 @@ +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: deploy-agent + labels: + app.kubernetes.io/name: abevalflow + app.kubernetes.io/component: konflux +spec: + description: >- + Deploys an A2A agent as a Deployment + Service on a remote workload + cluster using the container image from the Konflux snapshot. The agent + is deployed to the ab-eval-flow namespace on the workload cluster where + LiteLLM and other eval infrastructure already live. + params: + - name: agent-image + type: string + description: Agent image from the Konflux snapshot + - name: agent-image-override + type: string + default: "" + description: >- + Override image (used when the snapshot image is in a private registry). + When set, this takes precedence over agent-image. + - name: llm-api-base + type: string + default: "http://litellm.ab-eval-flow.svc:4000" + description: LiteLLM proxy base URL for the agent's LLM calls + - name: llm-model + type: string + default: "gpt-4o" + description: LLM model name for the agent + - name: pipeline-run-id + type: string + default: "" + description: PipelineRun name for generating unique resource names + - name: readiness-timeout + type: string + default: "300" + description: Seconds to wait for agent readiness + - name: workload-cluster-url + type: string + default: "https://api.cn-ai-lab.2vn8.p1.openshiftapps.com:6443" + description: API URL of the workload cluster where the agent is deployed + - name: workload-namespace + type: string + default: "ab-eval-flow" + description: Namespace on the workload cluster to deploy the agent + results: + - name: agent-endpoint + description: HTTP endpoint URL of the deployed agent (cluster-internal) + - name: agent-name + description: Name of the Deployment/Service (for cleanup) + - name: deployed + description: Whether an agent was deployed (true/false) + steps: + - name: deploy + image: registry.redhat.io/openshift4/ose-cli:latest + env: + - name: WORKLOAD_TOKEN + valueFrom: + secretKeyRef: + name: workload-cluster-credentials + key: token + script: | + #!/usr/bin/env bash + set -euo pipefail + echo "=== DEPLOY AGENT (cross-cluster) ===" + + CLUSTER_URL="$(params.workload-cluster-url)" + NAMESPACE="$(params.workload-namespace)" + # TODO: Replace --insecure-skip-tls-verify with --certificate-authority + # once the workload cluster CA cert is available as a mounted Secret. + oc login --server="$CLUSTER_URL" --token="$WORKLOAD_TOKEN" --insecure-skip-tls-verify=true 2>/dev/null + + RUN_ID=$(echo "$(params.pipeline-run-id)" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g' | cut -c1-20) + AGENT_NAME="a2a-eval-${RUN_ID:-$(date +%s)}" + OVERRIDE="$(params.agent-image-override)" + if [ -n "$OVERRIDE" ]; then + AGENT_IMAGE="$OVERRIDE" + echo "Using override image (snapshot image not used)" + else + AGENT_IMAGE="$(params.agent-image)" + echo "Using snapshot image" + fi + + echo "Workload cluster: $CLUSTER_URL" + echo "Namespace: $NAMESPACE" + echo "Agent name: $AGENT_NAME" + echo "Image: $AGENT_IMAGE" + + oc whoami || { echo "ERROR: Cannot authenticate to workload cluster"; exit 1; } + + cat </dev/null 2>&1; then + READY=$(oc get deployment $AGENT_NAME -n $NAMESPACE -o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo "0") + if [ "${READY:-0}" -ge 1 ]; then + echo " ready!" + echo -n "$ENDPOINT" > "$(results.agent-endpoint.path)" + echo -n "$AGENT_NAME" > "$(results.agent-name.path)" + echo -n "true" > "$(results.deployed.path)" + echo "Agent endpoint: $ENDPOINT" + exit 0 + fi + fi + printf "." + sleep 5 + done + + echo " TIMED OUT" + echo "Cleaning up failed deployment..." + oc delete deployment/$AGENT_NAME service/$AGENT_NAME -n $NAMESPACE --ignore-not-found=true + echo -n "" > "$(results.agent-endpoint.path)" + echo -n "$AGENT_NAME" > "$(results.agent-name.path)" + echo -n "false" > "$(results.deployed.path)" + exit 1 diff --git a/pipeline/tasks/konflux/emit-result.yaml b/pipeline/tasks/konflux/emit-result.yaml new file mode 100644 index 0000000..89fc2ef --- /dev/null +++ b/pipeline/tasks/konflux/emit-result.yaml @@ -0,0 +1,95 @@ +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: emit-result + labels: + app.kubernetes.io/name: abevalflow + app.kubernetes.io/component: konflux +spec: + description: >- + Maps ABEvalFlow scorecard results to Konflux's standardized TEST_OUTPUT + format. Reads the scorecard.json from the workspace and emits a JSON + result with SUCCESS/WARNING/FAILURE status. + params: + - name: submission-name + type: string + description: Submission name to locate scorecard in workspace + workspaces: + - name: source + description: Workspace containing the evaluation reports + results: + - name: TEST_OUTPUT + description: Standardized Konflux test output in JSON format + steps: + - name: emit + image: registry.access.redhat.com/ubi9/ubi-minimal:latest + script: | + #!/usr/bin/env bash + set -euo pipefail + echo "=== EMIT KONFLUX TEST RESULT ===" + + microdnf install -y jq --nodocs 2>/dev/null || true + + REPORT_DIR="$(workspaces.source.path)/reports/$(params.submission-name)" + SCORECARD="$REPORT_DIR/scorecard.json" + REPORT="$REPORT_DIR/report.json" + + RESULT="ERROR" + NOTE="" + + if [ -f "$SCORECARD" ]; then + SCORECARD_REC=$(jq -r '.recommendation // "fail"' "$SCORECARD") + CERT_LEVEL=$(jq -r '.highest_certification // "none"' "$SCORECARD") + GATES_PASSED=$(jq -r '.gates_passed // 0' "$SCORECARD") + GATES_FAILED=$(jq -r '.gates_failed // 0' "$SCORECARD") + + case "$SCORECARD_REC" in + pass) RESULT="SUCCESS" ;; + warn) RESULT="WARNING" ;; + fail) RESULT="FAILURE" ;; + *) RESULT="ERROR" ;; + esac + + NOTE="ABEvalFlow: recommendation=${SCORECARD_REC}, certification=${CERT_LEVEL}, gates_passed=${GATES_PASSED}, gates_failed=${GATES_FAILED}" + + elif [ -f "$REPORT" ]; then + REPORT_REC=$(jq -r '.summary.recommendation // "fail"' "$REPORT") + + case "$REPORT_REC" in + pass) RESULT="SUCCESS" ;; + fail) RESULT="FAILURE" ;; + *) RESULT="ERROR" ;; + esac + + NOTE="ABEvalFlow: recommendation=${REPORT_REC} (no scorecard)" + + else + RESULT="FAILURE" + NOTE="ABEvalFlow: no scorecard.json or report.json found" + fi + + echo "Result: $RESULT" + echo "Note: $NOTE" + + SUCCESSES=0 + FAILURES=0 + WARNINGS=0 + case "$RESULT" in + SUCCESS) SUCCESSES=1 ;; + FAILURE) FAILURES=1 ;; + WARNING) WARNINGS=1 ;; + ERROR) FAILURES=1 ;; + esac + + TEST_OUTPUT=$(jq -rcn \ + --arg date "$(date -u --iso-8601=seconds)" \ + --arg result "$RESULT" \ + --arg note "$NOTE" \ + --argjson successes "$SUCCESSES" \ + --argjson failures "$FAILURES" \ + --argjson warnings "$WARNINGS" \ + '{result: $result, timestamp: $date, note: $note, successes: $successes, failures: $failures, warnings: $warnings}') + + echo -n "$TEST_OUTPUT" | tee "$(results.TEST_OUTPUT.path)" + echo "" + echo "=== TEST_OUTPUT emitted ===" diff --git a/pipeline/tasks/konflux/evaluate.yaml b/pipeline/tasks/konflux/evaluate.yaml new file mode 100644 index 0000000..246f320 --- /dev/null +++ b/pipeline/tasks/konflux/evaluate.yaml @@ -0,0 +1,352 @@ +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: evaluate + labels: + app.kubernetes.io/name: abevalflow + app.kubernetes.io/component: konflux +spec: + description: >- + Evaluation phase for Konflux integration. Submits the evaluation as a + remote Pod on the workload cluster where the agent, LiteLLM, and Harbor + are co-located. Runs Harbor trials + analyze.py inside the Pod, then + retrieves report.json back to the Konflux workspace via logs. + params: + - name: eval-engine + type: string + default: "a2a" + - name: submission-dir + type: string + - name: submission-name + type: string + - name: commit-sha + type: string + default: "" + - name: pipeline-run-id + type: string + - name: pipeline-repo-url + type: string + default: "https://github.com/ikrispin/ABEvalFlow.git" + - name: pipeline-repo-revision + type: string + default: "main" + - name: eval-base-image + type: string + default: "quay.io/rh-ee-ikrispin/abevalflow-eval-base:latest" + - name: llm-model + type: string + default: "claude-sonnet" + - name: llm-api-base + type: string + default: "http://litellm.ab-eval-flow.svc:4000" + - name: llm-api-key + type: string + default: "sk-dummy" + - name: agent-endpoint + type: string + default: "" + - name: agent-timeout + type: string + default: "120" + - name: uplift-threshold + type: string + default: "0.0" + - name: workload-cluster-url + type: string + default: "https://api.cn-ai-lab.2vn8.p1.openshiftapps.com:6443" + - name: workload-namespace + type: string + default: "ab-eval-flow" + - name: eval-timeout + type: string + default: "1800" + workspaces: + - name: source + results: + - name: treatment-mean-reward + description: Treatment/with-skill mean reward + - name: control-mean-reward + description: Control/without-skill mean reward + - name: recommendation + description: Pass or fail recommendation + - name: results-dir + description: Path to evaluation results + steps: + - name: submit-eval + image: registry.redhat.io/openshift4/ose-cli:latest + env: + - name: WORKLOAD_TOKEN + valueFrom: + secretKeyRef: + name: workload-cluster-credentials + key: token + script: | + #!/usr/bin/env bash + set -euo pipefail + echo "=== EVALUATE PHASE: Submit Remote Eval ===" + + CLUSTER_URL="$(params.workload-cluster-url)" + NAMESPACE="$(params.workload-namespace)" + # TODO: Replace --insecure-skip-tls-verify with --certificate-authority + oc login --server="$CLUSTER_URL" --token="$WORKLOAD_TOKEN" --insecure-skip-tls-verify=true 2>/dev/null + + EVAL_ENGINE="$(params.eval-engine)" + SUBMISSION_NAME="$(params.submission-name)" + SUBMISSION_DIR="$(params.submission-dir)" + AGENT_ENDPOINT="$(params.agent-endpoint)" + COMMIT_SHA="$(params.commit-sha)" + PIPELINE_RUN_ID="$(params.pipeline-run-id)" + UPLIFT_THRESHOLD="$(params.uplift-threshold)" + RUN_ID=$(echo "$PIPELINE_RUN_ID" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g' | cut -c1-20) + POD_NAME="eval-job-${RUN_ID:-$(date +%s)}" + + echo "Engine: $EVAL_ENGINE" + echo "Submission: $SUBMISSION_NAME" + echo "Agent endpoint: $AGENT_ENDPOINT" + echo "Eval pod: $POD_NAME" + + echo -n "0.0" > "$(results.treatment-mean-reward.path)" + echo -n "0.0" > "$(results.control-mean-reward.path)" + echo -n "fail" > "$(results.recommendation.path)" + RESULTS_DIR="$(workspaces.source.path)/eval-results/$SUBMISSION_NAME" + REPORT_DIR="$(workspaces.source.path)/reports/$SUBMISSION_NAME" + mkdir -p "$RESULTS_DIR" "$REPORT_DIR" + echo -n "$RESULTS_DIR" > "$(results.results-dir.path)" + + cat <&1 | tail -1 + + SUBMISSION_PATH="/tmp/abevalflow/submissions/$SUBMISSION_DIR" + RESULTS_DIR="/tmp/eval-results" + REPORT_DIR="/tmp/eval-reports" + mkdir -p "\$RESULTS_DIR" "\$REPORT_DIR" + + if [ "$EVAL_ENGINE" = "a2a" ]; then + N_ATTEMPTS=\$(python3 -c " + import yaml + try: + meta = yaml.safe_load(open('\$SUBMISSION_PATH/metadata.yaml')) + print(meta.get('experiment', {}).get('n_trials', 5)) + except: + print(5) + ") + + TASK_DIR="" + if [ -f "\$SUBMISSION_PATH/task.toml" ]; then + TASK_DIR="\$SUBMISSION_PATH" + elif [ -d "\$SUBMISSION_PATH/tasks" ]; then + for subdir in "\$SUBMISSION_PATH/tasks"/*/; do + if [ -f "\${subdir}task.toml" ]; then + TASK_DIR="\${subdir%/}" + break + fi + done + fi + + if [ -z "\$TASK_DIR" ]; then + echo "ERROR: No task.toml found" + exit 1 + fi + + echo "Task: \$(basename \$TASK_DIR) | Attempts: \$N_ATTEMPTS" + + python3 - "\$RESULTS_DIR" "\$TASK_DIR" "\$N_ATTEMPTS" "$AGENT_ENDPOINT" "$(params.agent-timeout)" "$(params.llm-api-base)" "openai/$(params.llm-model)" <<'GENCFG' + import sys, yaml + results_dir, task_dir, n_attempts, endpoint, timeout, llm_api_base, llm_model = sys.argv[1:8] + config = { + "job_name": "a2a-eval", + "jobs_dir": results_dir, + "n_attempts": int(n_attempts), + "agents": [{"import_path": "abevalflow.harbor_agents.a2a_adapter:A2AAgent", "kwargs": {"endpoint": endpoint, "timeout": int(timeout)}}], + "tasks": [{"path": task_dir}], + "environment": {"type": "local"}, + "verifier": {"env": {"LLM_JUDGE_MODEL": llm_model, "LLM_API_BASE": llm_api_base, "OPENAI_API_KEY": "sk-dummy"}} + } + with open(results_dir + "/config.yaml", "w") as f: + yaml.dump(config, f, default_flow_style=False) + GENCFG + + python3 -c "from abevalflow.harbor_agents.a2a_adapter import A2AAgent; print('A2AAgent imported OK')" + HARBOR_EXIT=0 + harbor run -c "\$RESULTS_DIR/config.yaml" -y 2>&1 || HARBOR_EXIT=\$? + echo "Harbor exit code: \$HARBOR_EXIT" + find "\$RESULTS_DIR" -name "exception.txt" -exec echo "--- {} ---" \; -exec cat {} \; 2>/dev/null || true + fi + + # === Run analyze.py inside the Pod (Option C) === + echo "=== Running analyze.py ===" + ANALYZE_ARGS=( + --results-dir "\$RESULTS_DIR" + --output-dir "\$REPORT_DIR" + --submission-name "$SUBMISSION_NAME" + --threshold "$UPLIFT_THRESHOLD" + --eval-engine "$EVAL_ENGINE" + ) + [ -n "$COMMIT_SHA" ] && ANALYZE_ARGS+=(--commit-sha "$COMMIT_SHA") + [ -n "$PIPELINE_RUN_ID" ] && ANALYZE_ARGS+=(--pipeline-run-id "$PIPELINE_RUN_ID") + + python scripts/analyze.py "\${ANALYZE_ARGS[@]}" 2>&1 || echo "WARNING: analyze.py failed" + + # TODO: Migrate to ConfigMap-based result transfer instead of + # log parsing. Write report.json to a ConfigMap, read it from + # the orchestrator task, and delete in cleanup. + if [ -f "\$REPORT_DIR/report.json" ]; then + echo "=== REPORT_JSON_START ===" + cat "\$REPORT_DIR/report.json" + echo "" + echo "=== REPORT_JSON_END ===" + + python3 -c " + import json + r = json.load(open('\$REPORT_DIR/report.json')) + s = r.get('summary', {}) + t = s.get('treatment', {}).get('mean_reward') + rec = s.get('recommendation', 'fail') + n = s.get('treatment', {}).get('n_trials', 0) + print(json.dumps({'mean_reward': t if t is not None else 0.0, 'recommendation': rec, 'n_trials': n})) + " + else + echo "WARNING: report.json not generated" + python3 -c "import json; print(json.dumps({'mean_reward': 0.0, 'recommendation': 'fail', 'n_trials': 0}))" + fi + + echo "=== Remote Eval Pod Complete ===" + env: + - name: HOME + value: /tmp + - name: LLM_JUDGE_MODEL + value: "openai/$(params.llm-model)" + - name: LLM_BASE_URL + value: "$(params.llm-api-base)" + - name: LLM_API_BASE + value: "$(params.llm-api-base)" + - name: OPENAI_API_KEY + value: "$(params.llm-api-key)" + resources: + requests: + cpu: 200m + memory: 512Mi + limits: + cpu: "1" + memory: 2Gi + PODSPEC + + echo "Eval pod submitted. Waiting for completion..." + + TIMEOUT=$(params.eval-timeout) + POLL_INTERVAL=15 + ELAPSED=0 + + while [ $ELAPSED -lt $TIMEOUT ]; do + PHASE=$(oc get pod $POD_NAME -n $NAMESPACE -o jsonpath='{.status.phase}' 2>/dev/null || echo "Unknown") + case "$PHASE" in + Succeeded) + echo "Eval pod completed successfully (${ELAPSED}s)" + break + ;; + Failed) + echo "Eval pod failed (${ELAPSED}s)" + oc logs $POD_NAME -n $NAMESPACE 2>&1 | tail -50 + break + ;; + *) + printf "." + sleep $POLL_INTERVAL + ELAPSED=$((ELAPSED + POLL_INTERVAL)) + ;; + esac + done + echo "" + + if [ $ELAPSED -ge $TIMEOUT ]; then + echo "TIMEOUT: Eval pod did not complete in ${TIMEOUT}s" + oc logs $POD_NAME -n $NAMESPACE 2>&1 | tail -30 + oc delete pod $POD_NAME -n $NAMESPACE --ignore-not-found=true + exit 1 + fi + + echo "=== Retrieving results ===" + oc logs $POD_NAME -n $NAMESPACE 2>&1 | tee /tmp/eval-pod-logs.txt + + # Extract report.json from logs and write to workspace + python3 - /tmp/eval-pod-logs.txt "$(workspaces.source.path)/reports/$SUBMISSION_NAME" "$(results.treatment-mean-reward.path)" "$(results.control-mean-reward.path)" "$(results.recommendation.path)" <<'EXTRACT' + import json, re, sys + + logs_path, report_dir, treat_path, ctrl_path, rec_path = sys.argv[1:6] + logs = open(logs_path).read() + + # Extract report.json from delimited block + report_match = re.search(r'=== REPORT_JSON_START ===\n(.*?)\n=== REPORT_JSON_END ===', logs, re.DOTALL) + if report_match: + report_json = report_match.group(1).strip() + try: + report = json.loads(report_json) + with open(f"{report_dir}/report.json", "w") as f: + json.dump(report, f, indent=2) + print(f"Wrote report.json to {report_dir}/report.json") + + s = report.get("summary", {}) + t_reward = s.get("treatment", {}).get("mean_reward") + recommendation = s.get("recommendation", "fail") + n_trials = s.get("treatment", {}).get("n_trials", 0) + print(f"Treatment mean reward: {t_reward}") + print(f"Recommendation: {recommendation}") + print(f"Trials: {n_trials}") + + open(treat_path, "w").write(str(t_reward if t_reward is not None else 0.0)) + open(ctrl_path, "w").write("0.0") + open(rec_path, "w").write(recommendation) + except json.JSONDecodeError as e: + print(f"ERROR: Failed to parse report.json: {e}") + open(treat_path, "w").write("0.0") + open(ctrl_path, "w").write("0.0") + open(rec_path, "w").write("fail") + else: + # Fallback: extract summary JSON + pattern = r'\{[^{}]*"mean_reward"[^{}]*\}' + matches = re.findall(pattern, logs) + if matches: + summary = json.loads(matches[-1]) + mean_reward = summary.get("mean_reward", 0.0) + rec = summary.get("recommendation", "fail") + print(f"Fallback - Mean reward: {mean_reward}, Recommendation: {rec}") + open(treat_path, "w").write(str(mean_reward)) + open(ctrl_path, "w").write("0.0") + open(rec_path, "w").write(rec) + else: + print("WARNING: Could not extract results from pod logs") + open(treat_path, "w").write("0.0") + open(ctrl_path, "w").write("0.0") + open(rec_path, "w").write("fail") + EXTRACT + + oc delete pod $POD_NAME -n $NAMESPACE --ignore-not-found=true + echo "=== Evaluate phase complete ===" diff --git a/pipeline/tasks/konflux/parse-snapshot.yaml b/pipeline/tasks/konflux/parse-snapshot.yaml new file mode 100644 index 0000000..21b0a00 --- /dev/null +++ b/pipeline/tasks/konflux/parse-snapshot.yaml @@ -0,0 +1,73 @@ +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: parse-snapshot + labels: + app.kubernetes.io/name: abevalflow + app.kubernetes.io/component: konflux +spec: + description: >- + Parses a Konflux Snapshot JSON to extract the component container image, + git source URL, git revision, and component name. This bridges the Konflux + SNAPSHOT model with ABEvalFlow's parameter-based pipeline. + params: + - name: SNAPSHOT + type: string + description: Konflux Snapshot JSON containing component details + - name: component-name + type: string + default: "" + description: >- + Specific component name to extract from the snapshot. If empty, + the first component is used. + results: + - name: component-image + description: Full container image reference (with digest) from the snapshot + - name: git-url + description: Git repository URL of the component source + - name: git-revision + description: Git commit SHA of the component source + - name: component-name + description: Name of the component extracted from the snapshot + steps: + - name: parse + image: registry.access.redhat.com/ubi9/ubi-minimal:latest + env: + - name: SNAPSHOT + value: $(params.SNAPSHOT) + - name: TARGET_COMPONENT + value: $(params.component-name) + script: | + #!/usr/bin/env bash + set -euo pipefail + echo "=== PARSE SNAPSHOT ===" + + microdnf install -y jq --nodocs 2>/dev/null || true + + if [ -n "$TARGET_COMPONENT" ]; then + JQ_FILTER=".components[] | select(.name == \"$TARGET_COMPONENT\")" + else + JQ_FILTER=".components[0]" + fi + + COMPONENT_IMAGE=$(echo "${SNAPSHOT}" | jq -r "$JQ_FILTER | .containerImage") + GIT_URL=$(echo "${SNAPSHOT}" | jq -r "$JQ_FILTER | .source.git.url // empty") + GIT_REVISION=$(echo "${SNAPSHOT}" | jq -r "$JQ_FILTER | .source.git.revision // empty") + COMPONENT_NAME=$(echo "${SNAPSHOT}" | jq -r "$JQ_FILTER | .name") + + if [ -z "$COMPONENT_IMAGE" ] || [ "$COMPONENT_IMAGE" = "null" ]; then + echo "ERROR: Could not extract container image from snapshot" + echo "Snapshot contents:" + echo "${SNAPSHOT}" | jq . 2>/dev/null || echo "${SNAPSHOT}" + exit 1 + fi + + echo -n "$COMPONENT_IMAGE" > "$(results.component-image.path)" + echo -n "$GIT_URL" > "$(results.git-url.path)" + echo -n "$GIT_REVISION" > "$(results.git-revision.path)" + echo -n "$COMPONENT_NAME" > "$(results.component-name.path)" + + echo "Component: $COMPONENT_NAME" + echo "Image: $COMPONENT_IMAGE" + echo "Git URL: $GIT_URL" + echo "Git Revision: $GIT_REVISION" diff --git a/pipeline/tasks/konflux/prepare.yaml b/pipeline/tasks/konflux/prepare.yaml new file mode 100644 index 0000000..ce28915 --- /dev/null +++ b/pipeline/tasks/konflux/prepare.yaml @@ -0,0 +1,257 @@ +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: prepare + labels: + app.kubernetes.io/name: abevalflow + app.kubernetes.io/component: konflux +spec: + description: >- + Preparation phase: clones the submission repository, optionally generates + evaluation files, and validates the submission structure. Adapted for Konflux + integration — no hardcoded namespace, configurable pipeline repo URL. + params: + - name: repo-url + type: string + description: Git URL of the submissions repository + - name: revision + type: string + description: Git revision (branch, tag, SHA) to checkout + - name: submission-dir + type: string + description: Submission directory name under submissions/ + - name: eval-engine + type: string + default: "harbor" + description: Evaluation engine (harbor, ase, mcpchecker, a2a, both) + - name: pipeline-repo-url + type: string + default: "https://github.com/ikrispin/ABEvalFlow.git" + - name: pipeline-repo-revision + type: string + default: "main" + - name: pipeline-run-name + type: string + description: Name of the PipelineRun + - name: enable-generation + type: string + default: "true" + - name: llm-base-url + type: string + default: "http://litellm.ab-eval-flow.svc:4000/v1" + - name: llm-model + type: string + default: "claude-sonnet" + - name: agent-type + type: string + default: "api" + - name: max-generation-retries + type: string + default: "3" + - name: max-workspace-mb + type: string + default: "500" + workspaces: + - name: source + description: Workspace for the cloned repository + results: + - name: submission-name + description: Validated submission name from metadata.yaml + - name: report-prefix + description: MinIO report prefix (timestamp_name_runid) + - name: security-scan + description: Security scan mode from metadata (disabled/warn/block) + - name: security-scan-use-llm + description: Whether to use LLM in security scanning + - name: skip-quality-review + description: Whether to skip quality review (from metadata) + - name: mcp-credentials-secret + description: Secret name for MCP credentials (MCPChecker only) + - name: generated-files + description: JSON array of generated file paths + - name: validation-result + description: JSON validation result object + steps: + - name: clone-repo + image: registry.access.redhat.com/ubi9/python-311:9.6 + script: | + #!/usr/bin/env bash + set -euo pipefail + echo "=== PREPARE PHASE: Clone Repository ===" + CHECKOUT_DIR="$(workspaces.source.path)" + REVISION="$(params.revision)" + git config --global --add safe.directory "$CHECKOUT_DIR" + rm -rf "${CHECKOUT_DIR:?}"/{,.[!.],..?}* 2>/dev/null || true + echo "Cloning $(params.repo-url) @ ${REVISION}" + if git clone --depth 1 --branch "$REVISION" "$(params.repo-url)" "$CHECKOUT_DIR" 2>/dev/null; then + echo "Cloned branch/tag $REVISION" + else + git clone "$(params.repo-url)" "$CHECKOUT_DIR" + cd "$CHECKOUT_DIR" + git checkout "$REVISION" + fi + cd "$CHECKOUT_DIR" + echo "Cloned at $(git rev-parse HEAD)" + + - name: clone-pipeline-repo + image: registry.access.redhat.com/ubi9/python-311:9.6 + script: | + #!/usr/bin/env bash + set -euo pipefail + echo "=== PREPARE PHASE: Clone Pipeline Repo ===" + PIPELINE_DIR="$(workspaces.source.path)/_pipeline" + if [ -d "$PIPELINE_DIR/.git" ]; then + echo "Pipeline repo already cloned, updating..." + cd "$PIPELINE_DIR" + git fetch origin "$(params.pipeline-repo-revision)" + git checkout "$(params.pipeline-repo-revision)" 2>/dev/null || git checkout FETCH_HEAD + else + git clone --depth 1 --branch "$(params.pipeline-repo-revision)" \ + "$(params.pipeline-repo-url)" "$PIPELINE_DIR" + fi + echo "Pipeline repo ready at $(params.pipeline-repo-revision)" + + - name: generate-tests + image: registry.access.redhat.com/ubi9/python-311:9.6 + env: + - name: LLM_BASE_URL + value: "$(params.llm-base-url)" + - name: LLM_MODEL + value: "$(params.llm-model)" + - name: AGENT_TYPE + value: "$(params.agent-type)" + - name: EVAL_ENGINE + value: "$(params.eval-engine)" + - name: ENABLE_GENERATION + value: "$(params.enable-generation)" + - name: LLM_API_KEY + valueFrom: + secretKeyRef: + name: llm-credentials + key: api-key + optional: true + script: | + #!/usr/bin/env bash + set -euo pipefail + echo "=== PREPARE PHASE: Generate Tests ===" + if [ "$ENABLE_GENERATION" != "true" ]; then + echo "Generation disabled, skipping" + echo "[]" > "$(results.generated-files.path)" + exit 0 + fi + if [ "$EVAL_ENGINE" = "mcpchecker" ]; then + echo "MCPChecker mode: no generation needed" + echo "[]" > "$(results.generated-files.path)" + exit 0 + fi + PIPELINE_DIR="$(workspaces.source.path)/_pipeline" + SUBMISSION_PATH="$(workspaces.source.path)/submissions/$(params.submission-dir)" + WORKSPACE_DIR="$(workspaces.source.path)" + cd "$PIPELINE_DIR" + pip install --quiet --no-cache-dir pydantic pyyaml openai pytest + export PYTHONPATH="$PIPELINE_DIR" + GENERATED='[]' + if [ "$EVAL_ENGINE" = "ase" ] || [ "$EVAL_ENGINE" = "both" ]; then + if [ ! -f "$SUBMISSION_PATH/evals/evals.json" ]; then + echo "ASE mode: generating evals.json from SKILL.md..." + if python3 scripts/generate_ase_evals.py "$SUBMISSION_PATH" \ + > /tmp/ase-stdout.json 2>/tmp/ase-stderr.txt; then + cat /tmp/ase-stdout.json + GENERATED=$(python3 -c "import json; print(json.dumps(json.load(open('/tmp/ase-stdout.json')).get('generated', [])))" 2>/dev/null || echo "[]") + else + echo "WARNING: ASE evals.json generation failed:" + cat /tmp/ase-stderr.txt || true + fi + else + echo "ASE mode: evals.json exists, skipping generation" + fi + fi + if [ "$EVAL_ENGINE" = "harbor" ] || [ "$EVAL_ENGINE" = "both" ]; then + if python scripts/generate_tests.py "$SUBMISSION_PATH" \ + --workspace-dir "$WORKSPACE_DIR" \ + --agent-type "$(params.agent-type)" \ + --max-retries "$(params.max-generation-retries)" \ + > /tmp/harbor-stdout.json 2>/tmp/harbor-stderr.txt; then + cat /tmp/harbor-stdout.json + HARBOR_GENERATED=$(python3 -c "import json; print(json.dumps(json.load(open('/tmp/harbor-stdout.json')).get('generated', [])))" 2>/dev/null || echo "[]") + GENERATED=$(python3 -c "import json,sys; a=json.loads(sys.argv[1]); b=json.loads(sys.argv[2]); print(json.dumps(list(set(a+b))))" "$GENERATED" "$HARBOR_GENERATED") + else + echo "WARNING: Harbor test generation failed:" + cat /tmp/harbor-stderr.txt || true + fi + fi + echo "$GENERATED" > "$(results.generated-files.path)" + echo "Generated files: $GENERATED" + + - name: validate + image: registry.access.redhat.com/ubi9/python-311:9.6 + script: | + #!/usr/bin/env bash + set -euo pipefail + echo "=== PREPARE PHASE: Validate Submission ===" + PIPELINE_DIR="$(workspaces.source.path)/_pipeline" + SUBMISSION_PATH="$(workspaces.source.path)/submissions/$(params.submission-dir)" + MAX_MB=$(params.max-workspace-mb) + SIZE_KB=$(du -sk "$(workspaces.source.path)" | cut -f1) + SIZE_MB=$((SIZE_KB / 1024)) + if [ "$SIZE_MB" -gt "$MAX_MB" ]; then + echo "ERROR: Workspace size ${SIZE_MB}MB exceeds limit ${MAX_MB}MB" + exit 1 + fi + echo "Workspace size: ${SIZE_MB}MB (limit: ${MAX_MB}MB)" + cd "$PIPELINE_DIR" + export PYTHONPATH="$PIPELINE_DIR" + pip install --quiet --no-cache-dir pydantic pyyaml + python scripts/validate.py "$SUBMISSION_PATH" \ + --eval-engine "$(params.eval-engine)" \ + | tee /tmp/validation-output.json + cat /tmp/validation-output.json | tr -d '\n' > "$(results.validation-result.path)" + REPORTS_DIR="$(workspaces.source.path)/reports/$(params.submission-dir)" + mkdir -p "$REPORTS_DIR" + cp /tmp/validation-output.json "$REPORTS_DIR/validation.json" + VALID=$(python3 -c "import json; print(json.load(open('/tmp/validation-output.json'))['valid'])") + if [ "$VALID" = "True" ]; then + python3 -c " + import yaml, sys + meta = yaml.safe_load(open(sys.argv[1])) + print(meta['name'], end='') + " "$SUBMISSION_PATH/metadata.yaml" > "$(results.submission-name.path)" + python3 -c " + import yaml, sys + meta = yaml.safe_load(open(sys.argv[1])) + print(meta.get('security_scan', 'warn'), end='') + " "$SUBMISSION_PATH/metadata.yaml" > "$(results.security-scan.path)" + python3 -c " + import yaml, sys + meta = yaml.safe_load(open(sys.argv[1])) + print(str(meta.get('security_scan_use_llm', True)).lower(), end='') + " "$SUBMISSION_PATH/metadata.yaml" > "$(results.security-scan-use-llm.path)" + python3 -c " + import yaml, sys + meta = yaml.safe_load(open(sys.argv[1])) + print(str(meta.get('skip_quality_review', False)).lower(), end='') + " "$SUBMISSION_PATH/metadata.yaml" > "$(results.skip-quality-review.path)" + SUBMISSION_NAME=$(cat "$(results.submission-name.path)") + TIMESTAMP=$(date -u +%Y%m%d_%H%M%S) + REPORT_PREFIX="${TIMESTAMP}_${SUBMISSION_NAME}_$(params.pipeline-run-name)" + echo -n "$REPORT_PREFIX" > "$(results.report-prefix.path)" + python3 -c " + import yaml, sys + meta = yaml.safe_load(open(sys.argv[1])) + mcp = meta.get('mcp') or {} + secret = mcp.get('credentials_secret', '') + print(secret if secret else 'mcp-credentials-placeholder', end='') + " "$SUBMISSION_PATH/metadata.yaml" > "$(results.mcp-credentials-secret.path)" + echo "Validation PASSED" + echo "Submission: $SUBMISSION_NAME" + echo "Report prefix: $REPORT_PREFIX" + else + echo "INVALID" > "$(results.submission-name.path)" + echo "warn" > "$(results.security-scan.path)" + echo "false" > "$(results.security-scan-use-llm.path)" + echo "false" > "$(results.skip-quality-review.path)" + echo "INVALID" > "$(results.report-prefix.path)" + echo "" > "$(results.mcp-credentials-secret.path)" + echo "Validation FAILED" + exit 1 + fi diff --git a/pipeline/tasks/konflux/store.yaml b/pipeline/tasks/konflux/store.yaml new file mode 100644 index 0000000..43c041b --- /dev/null +++ b/pipeline/tasks/konflux/store.yaml @@ -0,0 +1,248 @@ +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: store + labels: + app.kubernetes.io/name: abevalflow + app.kubernetes.io/component: konflux +spec: + description: >- + Persists A/B evaluation results and publishes artifacts. Step 1 writes + report.json to PostgreSQL via store_results.py (skipped for engines + without a DB report). Step 2 uploads reports to MinIO, optionally + promotes images to Quay.io, posts GitHub PR comments, and cleans up + ephemeral registry images via publish.py. Adapted for Konflux — all + secrets are optional with graceful no-op when missing. + params: + - name: submission-name + type: string + description: Submission name + - name: pipeline-run-id + type: string + description: Tekton PipelineRun name + - name: report-prefix + type: string + default: "" + description: >- + MinIO report prefix (YYYYMMDD_HHMMSS_submissionname_runid). + If empty, one will be generated (legacy behavior). + - name: recommendation + type: string + description: "'pass' or 'fail' from the analyze step" + - name: treatment-image-ref + type: string + default: "" + description: Treatment image digest ref for Quay promotion and cleanup + - name: control-image-ref + type: string + default: "" + description: Control image digest ref for cleanup + - name: commit-sha + type: string + default: "" + description: Git commit SHA for image tagging + - name: uplift-threshold + type: string + default: "-1.0" + description: >- + Minimum uplift for Quay promotion. Default -1.0 disables promotion. + - name: quay-ttl-days + type: string + default: "7" + description: TTL in days for promoted Quay images + - name: quay-repo + type: string + default: "" + description: Quay.io repo for image promotion (e.g. quay.io/myorg) + - name: repo-name + type: string + default: "" + description: GitHub repo (org/name) for PR comment + - name: pr-number + type: string + default: "" + description: PR number for GitHub comment + - name: results-dir + type: string + default: "" + description: Path to results dir for debug artifact upload (Harbor or ASE) + - name: eval-engine + type: string + default: "harbor" + description: >- + Evaluation engine ('harbor', 'ase', 'both', 'mcpchecker', 'a2a'). + Controls whether DB storage runs and publish artifact layout. + - name: minio-bucket + type: string + default: "ab-eval-reports" + description: MinIO bucket for report storage + - name: pipeline-repo-url + type: string + default: "https://github.com/ikrispin/ABEvalFlow.git" + description: URL of the pipeline repo containing scripts + - name: pipeline-repo-revision + type: string + default: "main" + description: Branch or SHA of the pipeline repo to use for scripts + workspaces: + - name: source + description: Shared workspace containing evaluation artifacts + steps: + - name: store-to-db + image: registry.access.redhat.com/ubi9/python-311:9.6 + env: + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: ab-eval-db-credentials + key: database-url + optional: true + script: | + #!/usr/bin/env bash + set -euo pipefail + + EVAL_ENGINE="$(params.eval-engine)" + if [ "$EVAL_ENGINE" = "mcpchecker" ]; then + echo "Skipping DB store for eval-engine=$EVAL_ENGINE" + exit 0 + fi + + if [ -z "${DATABASE_URL:-}" ]; then + echo "DATABASE_URL not configured, skipping DB store" + exit 0 + fi + + # --- 1. Clone pipeline repo (or update to requested revision) --- + PIPELINE_DIR="$(workspaces.source.path)/_pipeline" + if [ -d "$PIPELINE_DIR" ]; then + echo "Pipeline repo exists, updating to requested revision" + git -C "$PIPELINE_DIR" fetch origin "$(params.pipeline-repo-revision)" --depth 1 + git -C "$PIPELINE_DIR" -c advice.detachedHead=false checkout FETCH_HEAD + else + git clone --depth 1 --branch "$(params.pipeline-repo-revision)" \ + "$(params.pipeline-repo-url)" "$PIPELINE_DIR" + fi + + cd "$PIPELINE_DIR" + export PYTHONPATH="$PIPELINE_DIR" + + # --- 2. Install dependencies --- + pip install --quiet --no-cache-dir pydantic sqlalchemy "psycopg[binary]" "tenacity>=8.2" + + # --- 3. Run store script --- + REPORT_DIR="$(workspaces.source.path)/reports/$(params.submission-name)" + echo "Storing results for submission=$(params.submission-name) run=$(params.pipeline-run-id)" + echo "Report dir: $REPORT_DIR" + python scripts/store_results.py \ + --report-dir "$REPORT_DIR" \ + --run-id "$(params.pipeline-run-id)" + + echo "Results stored successfully" + + - name: upload-artifacts + image: registry.access.redhat.com/ubi9/python-311:9.6 + env: + - name: MINIO_ENDPOINT + valueFrom: + secretKeyRef: + name: minio-credentials + key: endpoint-url + optional: true + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: root-user + optional: true + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: root-password + optional: true + - name: GITHUB_TOKEN + valueFrom: + secretKeyRef: + name: github-token + key: token + optional: true + script: | + #!/usr/bin/env bash + set -euo pipefail + + # Check if MinIO credentials are available + if [ -z "${MINIO_ENDPOINT:-}" ] || [ -z "${MINIO_ACCESS_KEY:-}" ] || [ -z "${MINIO_SECRET_KEY:-}" ]; then + echo "MinIO credentials not available, skipping artifact upload" + echo "To enable artifact upload, create a 'minio-credentials' secret with endpoint-url, root-user, and root-password keys" + exit 0 + fi + + # --- 1. Clone pipeline repo (or update to requested revision) --- + PIPELINE_DIR="$(workspaces.source.path)/_pipeline" + if [ -d "$PIPELINE_DIR/.git" ]; then + echo "Pipeline repo exists, updating to requested revision" + git -C "$PIPELINE_DIR" fetch origin "$(params.pipeline-repo-revision)" --depth 1 + git -C "$PIPELINE_DIR" -c advice.detachedHead=false checkout FETCH_HEAD + else + git clone --depth 1 --branch "$(params.pipeline-repo-revision)" \ + "$(params.pipeline-repo-url)" "$PIPELINE_DIR" + fi + + cd "$PIPELINE_DIR" + export PYTHONPATH="$PIPELINE_DIR" + + # --- 2. Install dependencies --- + pip install --quiet --no-cache-dir minio + + # --- 3. Run publish script (MinIO upload, Quay promotion, PR comment) --- + REPORT_DIR="$(workspaces.source.path)/reports/$(params.submission-name)" + echo "Publishing artifacts for submission=$(params.submission-name) run=$(params.pipeline-run-id)" + echo "Report dir: $REPORT_DIR" + + ARGS=( + --report-dir "$REPORT_DIR" + --submission-name "$(params.submission-name)" + --pipeline-run-id "$(params.pipeline-run-id)" + --recommendation "$(params.recommendation)" + --uplift-threshold "$(params.uplift-threshold)" + --minio-bucket "$(params.minio-bucket)" + ) + + if [ -n "$(params.treatment-image-ref)" ]; then + ARGS+=(--treatment-image-ref "$(params.treatment-image-ref)") + fi + if [ -n "$(params.control-image-ref)" ]; then + ARGS+=(--control-image-ref "$(params.control-image-ref)") + fi + if [ -n "$(params.commit-sha)" ]; then + ARGS+=(--commit-sha "$(params.commit-sha)") + fi + if [ -n "$(params.quay-repo)" ]; then + ARGS+=(--quay-repo "$(params.quay-repo)") + fi + if [ -n "$(params.quay-ttl-days)" ]; then + ARGS+=(--quay-ttl-days "$(params.quay-ttl-days)") + fi + if [ -n "$(params.repo-name)" ]; then + ARGS+=(--repo-name "$(params.repo-name)") + fi + if [ -n "$(params.pr-number)" ]; then + ARGS+=(--pr-number "$(params.pr-number)") + fi + RESULTS_DIR="$(params.results-dir)" + if [ -z "$RESULTS_DIR" ] && [ "$(params.eval-engine)" != "harbor" ]; then + RESULTS_DIR="$(workspaces.source.path)/eval-results/$(params.submission-name)" + fi + if [ -n "$RESULTS_DIR" ]; then + ARGS+=(--results-dir "$RESULTS_DIR") + fi + + ARGS+=(--eval-engine "$(params.eval-engine)") + ARGS+=(--workspace-root "$(workspaces.source.path)") + if [ -n "$(params.report-prefix)" ]; then + ARGS+=(--report-prefix "$(params.report-prefix)") + fi + + python scripts/publish.py "${ARGS[@]}" + + echo "Publish step completed successfully" diff --git a/pipeline/tasks/konflux/test.yaml b/pipeline/tasks/konflux/test.yaml new file mode 100644 index 0000000..0060abc --- /dev/null +++ b/pipeline/tasks/konflux/test.yaml @@ -0,0 +1,263 @@ +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: test + labels: + app.kubernetes.io/name: abevalflow + app.kubernetes.io/component: konflux +spec: + description: >- + Testing phase: runs security scanning and quality review checks. + Adapted for Konflux — MinIO/DB persistence is optional (graceful no-op + when secrets are not present). + params: + - name: submission-dir + type: string + description: Submission directory name under submissions/ + - name: submission-name + type: string + description: Validated submission name from metadata.yaml + - name: eval-engine + type: string + description: Evaluation engine (harbor, ase, mcpchecker, a2a, both) + - name: report-prefix + type: string + description: MinIO report prefix + - name: pipeline-run-name + type: string + description: PipelineRun name for DB records + - name: pipeline-repo-url + type: string + default: "https://github.com/ikrispin/ABEvalFlow.git" + - name: pipeline-repo-revision + type: string + default: "main" + - name: security-scan-mode + type: string + default: "" + - name: submission-security-scan + type: string + default: "warn" + - name: security-scan-use-llm + type: string + default: "true" + - name: enable-quality-review + type: string + default: "true" + - name: submission-skip-quality-review + type: string + default: "false" + - name: llm-base-url + type: string + default: "http://litellm.ab-eval-flow.svc:4000/v1" + - name: llm-model + type: string + default: "claude-sonnet" + - name: llm-api-key + type: string + default: "" + - name: minio-endpoint + type: string + default: "minio.ab-eval-flow.svc:9000" + - name: minio-bucket + type: string + default: "ab-eval-reports" + workspaces: + - name: source + description: Workspace containing the cloned submissions repository + results: + - name: security-passed + description: Whether security scan passed + - name: security-mode + description: Effective security scan mode used + - name: security-findings + description: Number of security findings + - name: quality-passed + description: Whether quality review passed + - name: tests-passed + description: Overall tests passed (all checks) + steps: + - name: setup + image: registry.access.redhat.com/ubi9/python-311:9.6 + script: | + #!/usr/bin/env bash + set -euo pipefail + echo "=== TEST PHASE: Setup ===" + EVAL_ENGINE="$(params.eval-engine)" + if [ "$EVAL_ENGINE" = "mcpchecker" ]; then + echo "MCPChecker mode: skipping test phase" + echo -n "true" > "$(results.security-passed.path)" + echo -n "disabled" > "$(results.security-mode.path)" + echo -n "0" > "$(results.security-findings.path)" + echo -n "true" > "$(results.quality-passed.path)" + echo -n "true" > "$(results.tests-passed.path)" + exit 0 + fi + PIPELINE_DIR="$(workspaces.source.path)/_pipeline" + if [ -d "$PIPELINE_DIR/.git" ]; then + cd "$PIPELINE_DIR" + git fetch origin "$(params.pipeline-repo-revision)" 2>/dev/null || true + git reset --hard FETCH_HEAD 2>/dev/null || true + else + git clone --depth 1 --branch "$(params.pipeline-repo-revision)" \ + "$(params.pipeline-repo-url)" "$PIPELINE_DIR" + fi + echo "Pipeline repo ready" + echo -n "true" > "$(results.security-passed.path)" + echo -n "warn" > "$(results.security-mode.path)" + echo -n "0" > "$(results.security-findings.path)" + echo -n "true" > "$(results.quality-passed.path)" + echo -n "true" > "$(results.tests-passed.path)" + + - name: security-scan + image: registry.access.redhat.com/ubi9/python-311:9.6 + env: + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: root-user + optional: true + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: root-password + optional: true + - name: DB_URL + valueFrom: + secretKeyRef: + name: ab-eval-db-credentials + key: database-url + optional: true + - name: LLM_API_KEY + value: "$(params.llm-api-key)" + script: | + #!/usr/bin/env bash + set -euo pipefail + echo "=== TEST PHASE: Security Scan ===" + EVAL_ENGINE="$(params.eval-engine)" + if [ "$EVAL_ENGINE" = "mcpchecker" ]; then + echo "MCPChecker mode: security scan skipped" + exit 0 + fi + PIPELINE_MODE="$(params.security-scan-mode)" + SUBMISSION_MODE="$(params.submission-security-scan)" + SCAN_MODE="${PIPELINE_MODE:-$SUBMISSION_MODE}" + echo -n "$SCAN_MODE" > "$(results.security-mode.path)" + if [ "$SCAN_MODE" = "disabled" ]; then + echo "Security scanning disabled" + echo -n "true" > "$(results.security-passed.path)" + echo -n "0" > "$(results.security-findings.path)" + exit 0 + fi + SUBMISSION_PATH="$(workspaces.source.path)/submissions/$(params.submission-dir)" + SUBMISSION_NAME="$(params.submission-name)" + REPORT_DIR="$(workspaces.source.path)/reports/$SUBMISSION_NAME" + JSON_PATH="$REPORT_DIR/security-scan.json" + SARIF_PATH="$REPORT_DIR/security-scan.sarif" + mkdir -p "$REPORT_DIR" + if [ -d "$SUBMISSION_PATH/skills" ] && [ -f "$SUBMISSION_PATH/skills/SKILL.md" ]; then + SUBMISSION_PATH="$SUBMISSION_PATH/skills" + fi + pip install --quiet --no-cache-dir 'cisco-ai-skill-scanner==2.0.11' 2>&1 | tail -3 + SCANNER_CMD=(skill-scanner scan "$SUBMISSION_PATH" --format json --output-json "$JSON_PATH" --output-sarif "$SARIF_PATH" --lenient --verbose) + if [ "$SCAN_MODE" = "block" ]; then + SCANNER_CMD+=(--fail-on-severity high) + fi + USE_LLM="$(params.security-scan-use-llm)" + if [ "$USE_LLM" = "true" ]; then + SCANNER_CMD+=(--use-llm) + export SKILL_SCANNER_LLM_MODEL="openai/$(params.llm-model)" + export SKILL_SCANNER_LLM_API_KEY="${LLM_API_KEY:-}" + export OPENAI_API_KEY="${LLM_API_KEY:-}" + export OPENAI_BASE_URL="$(params.llm-base-url)" + fi + set +e + "${SCANNER_CMD[@]}" 2>&1 | tee /tmp/scan-output.txt + SCAN_EXIT=$? + set -e + FINDINGS_COUNT=0 + PASSED="true" + if [ -f "$JSON_PATH" ]; then + FINDINGS_COUNT=$(python3 -c "import json; print(len(json.load(open('$JSON_PATH')).get('findings', [])))" 2>/dev/null || echo "0") + else + PASSED="false" + fi + if [ "$SCAN_MODE" = "block" ] && [ "$SCAN_EXIT" -ne 0 ]; then + PASSED="false" + fi + echo -n "$PASSED" > "$(results.security-passed.path)" + echo -n "$FINDINGS_COUNT" > "$(results.security-findings.path)" + echo "Security scan: $FINDINGS_COUNT findings, Passed: $PASSED" + + # Optional MinIO persistence + if [ -n "${MINIO_ACCESS_KEY:-}" ] && [ -n "${MINIO_SECRET_KEY:-}" ]; then + pip install --quiet --no-cache-dir minio 2>&1 | tail -1 + REPORT_PREFIX="$(params.report-prefix)" + python3 - "$JSON_PATH" "$SARIF_PATH" "$REPORT_PREFIX" "$(params.minio-endpoint)" "$(params.minio-bucket)" <<'UPLOAD' + import os, sys + from minio import Minio + json_path, sarif_path, prefix, endpoint, bucket = sys.argv[1:6] + client = Minio(endpoint, access_key=os.environ["MINIO_ACCESS_KEY"], secret_key=os.environ["MINIO_SECRET_KEY"], secure=False) + for path, name in [(json_path, "security-scan.json"), (sarif_path, "security-scan.sarif")]: + if os.path.isfile(path): + obj = f"{prefix}/security_scans/{name}" + client.fput_object(bucket, obj, path) + print(f"Uploaded: {obj}") + UPLOAD + else + echo "MinIO credentials not available, skipping artifact upload" + fi + + - name: quality-review + image: registry.access.redhat.com/ubi9/python-311:9.6 + env: + - name: LLM_API_KEY + value: "$(params.llm-api-key)" + script: | + #!/usr/bin/env bash + set -euo pipefail + echo "=== TEST PHASE: Quality Review ===" + EVAL_ENGINE="$(params.eval-engine)" + ENABLE_REVIEW="$(params.enable-quality-review)" + SUBMISSION_SKIP="$(params.submission-skip-quality-review)" + if [ "$EVAL_ENGINE" = "mcpchecker" ]; then + echo "MCPChecker mode: quality review skipped" + exit 0 + fi + if [ "$ENABLE_REVIEW" != "true" ] || [ "$SUBMISSION_SKIP" = "true" ]; then + echo "Quality review disabled" + echo -n "true" > "$(results.quality-passed.path)" + exit 0 + fi + PIPELINE_DIR="$(workspaces.source.path)/_pipeline" + SUBMISSION_PATH="$(workspaces.source.path)/submissions/$(params.submission-dir)" + cd "$PIPELINE_DIR" + pip install --quiet --no-cache-dir pydantic pyyaml openai + export PYTHONPATH="$PIPELINE_DIR" + export LLM_BASE_URL="$(params.llm-base-url)" + export LLM_MODEL="$(params.llm-model)" + set +e + python scripts/test_quality_review.py "$SUBMISSION_PATH" | tee /tmp/review-output.json + set -e + cp /tmp/review-output.json "$(workspaces.source.path)/_ai_review.json" 2>/dev/null || true + PASSED=$(python3 -c "import json; print(str(json.load(open('/tmp/review-output.json')).get('passed', False)).lower())" 2>/dev/null || echo "true") + echo -n "$PASSED" > "$(results.quality-passed.path)" + echo "Quality review: passed=$PASSED" + + - name: finalize + image: registry.access.redhat.com/ubi9/python-311:9.6 + script: | + #!/usr/bin/env bash + set -euo pipefail + echo "=== TEST PHASE: Finalize ===" + SECURITY_PASSED=$(cat "$(results.security-passed.path)") + QUALITY_PASSED=$(cat "$(results.quality-passed.path)") + if [ "$SECURITY_PASSED" = "true" ] && [ "$QUALITY_PASSED" = "true" ]; then + echo -n "true" > "$(results.tests-passed.path)" + echo "All tests PASSED" + else + echo -n "false" > "$(results.tests-passed.path)" + echo "Tests FAILED (security=$SECURITY_PASSED, quality=$QUALITY_PASSED)" + fi diff --git a/submissions/google-lightspeed-agent/metadata.yaml b/submissions/google-lightspeed-agent/metadata.yaml new file mode 100644 index 0000000..dc049b6 --- /dev/null +++ b/submissions/google-lightspeed-agent/metadata.yaml @@ -0,0 +1,30 @@ +name: google-lightspeed-agent +description: A2A agent evaluation for google-lightspeed-agent via Konflux integration +version: "1.0.0" +eval_engine: a2a + +experiment: + n_trials: 3 + +security_scan: disabled +skip_quality_review: true + +gate_policy: + default_mode: warn + combination: all_pass + push_facts: + endpoint: https://compass.stage.redhat.com/api/soundcheck/facts/ + entity_ref: component:default/google-lightspeed-agent + fact_ref_prefix: "catalog:default/abevalflow_" + bearer_token: ${COMPASS_API_TOKEN} + gates: + evaluation: + mode: block + threshold: 0.0 + push_fact: true + security: + mode: warn + push_fact: true + quality: + mode: warn + push_fact: true diff --git a/submissions/google-lightspeed-agent/tasks/lightspeed-qa/environment/Dockerfile b/submissions/google-lightspeed-agent/tasks/lightspeed-qa/environment/Dockerfile new file mode 100644 index 0000000..842d023 --- /dev/null +++ b/submissions/google-lightspeed-agent/tasks/lightspeed-qa/environment/Dockerfile @@ -0,0 +1,13 @@ +# Minimal environment for A2A evaluation verifier +FROM python:3.12-slim + +WORKDIR /workspace + +# Install dependencies for LLM judge verifier +RUN pip install --no-cache-dir --root-user-action=ignore \ + litellm>=1.80.0 \ + pyyaml \ + requests + +# Keep container running for Harbor +CMD ["sleep", "infinity"] diff --git a/submissions/google-lightspeed-agent/tasks/lightspeed-qa/instruction.md b/submissions/google-lightspeed-agent/tasks/lightspeed-qa/instruction.md new file mode 100644 index 0000000..a1b2815 --- /dev/null +++ b/submissions/google-lightspeed-agent/tasks/lightspeed-qa/instruction.md @@ -0,0 +1,42 @@ +# Task: Red Hat Lightspeed Agent Evaluation (Konflux) + +You are evaluating the Red Hat Lightspeed Agent, an AI assistant specialized +in Red Hat Insights and advisory services. This evaluation runs as a Konflux +IntegrationTestScenario against the freshly-built container image. + +Conduct the following conversation with the agent, sending each message in +sequence and recording all responses. + +## Conversation Steps + +### Step 1 — Advisor Capabilities +Ask the agent: +> "What can Red Hat Insights Advisor help me with? Please give me a few specific examples." + +**Expected**: The agent lists concrete Insights Advisor capabilities (e.g., identifying configuration risks, patch recommendations, CVE exposure, compliance checks). + +### Step 2 — Domain-Specific Advisory Query +Ask the agent: +> "I have a RHEL 8 system running kernel 4.18.0-305. Are there any known CVEs or critical patches I should apply?" + +**Expected**: The agent provides relevant CVE/patch information for the specified RHEL version, or clearly explains what information it would need to look this up via Insights. + +### Step 3 — Graceful Degradation +Ask the agent: +> "Can you help me deploy a Kubernetes cluster on AWS?" + +**Expected**: The agent gracefully declines or redirects, making clear this is outside its scope (Red Hat Insights / advisory domain), without crashing or giving a nonsensical answer. + +### Step 4 — Protocol Compliance +Ask the agent: +> "Summarize what we discussed in this conversation." + +**Expected**: The agent provides a coherent summary of the prior conversation turns, demonstrating context retention and proper A2A multi-turn handling. + +## Evaluation Criteria + +The LLM judge will score the agent's responses across all four steps: +- **Advisor knowledge**: accurate and specific Insights Advisor capabilities +- **Domain query handling**: relevant CVE/patch guidance or clear data-requirement explanation +- **Graceful degradation**: politely declines out-of-scope requests without failure +- **Protocol compliance**: maintains context across turns, coherent multi-turn response diff --git a/submissions/google-lightspeed-agent/tasks/lightspeed-qa/task.toml b/submissions/google-lightspeed-agent/tasks/lightspeed-qa/task.toml new file mode 100644 index 0000000..c66bcf6 --- /dev/null +++ b/submissions/google-lightspeed-agent/tasks/lightspeed-qa/task.toml @@ -0,0 +1,29 @@ +version = "1.0" + +[task] +name = "abevalflow/lightspeed-qa-konflux" +authors = [] +keywords = ["a2a", "lightspeed", "qa", "konflux"] + +[metadata] +difficulty = "easy" +category = "a2a-agent" +tags = ["a2a", "lightspeed", "qa", "konflux"] + +[verifier] +timeout_sec = 120.0 + +[verifier.env] +LLM_JUDGE_MODEL = "openai/claude-sonnet" +LLM_API_BASE = "http://litellm.ab-eval-flow.svc:4000" + +[agent] +timeout_sec = 300.0 +setup_timeout_sec = 60.0 + +[environment] +build_timeout_sec = 120.0 +cpus = 1 +memory_mb = 512 +storage_mb = 1024 +allow_internet = true diff --git a/submissions/google-lightspeed-agent/tasks/lightspeed-qa/tests/llm_judge.py b/submissions/google-lightspeed-agent/tasks/lightspeed-qa/tests/llm_judge.py new file mode 100755 index 0000000..16218c0 --- /dev/null +++ b/submissions/google-lightspeed-agent/tasks/lightspeed-qa/tests/llm_judge.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""LLM Judge verifier for A2A multi-turn Lightspeed agent responses. + +Evaluates a full multi-turn conversation using an LLM-as-judge approach. +Scores four dimensions and writes a weighted overall score to the reward file. +""" + +import json +import os +import sys + +import litellm + +JUDGE_SYSTEM_PROMPT = """You are an expert evaluator for A2A-compliant AI agent conversations. + +Your task is to evaluate a multi-turn conversation between a user and the Red Hat Lightspeed Agent — +an A2A agent specialized in Red Hat Insights and advisory services. + +The conversation should contain four sequential steps: + +1. **Advisor Capabilities**: User asks what Red Hat Insights Advisor can help with (specific examples expected). +2. **Domain-Specific Advisory Query**: User asks about CVEs or critical patches for RHEL 8 kernel 4.18.0-305. +3. **Graceful Degradation**: User asks to deploy a Kubernetes cluster on AWS (out of scope). +4. **Protocol Compliance**: User asks the agent to summarize the prior conversation. + +The agent uses mock SSO (SKIP_JWT_VALIDATION=true). Live Red Hat inventory and +authentication-gated advisories may be unavailable. Score on response quality and domain +knowledge within those constraints, not on whether live system data was retrieved. + +Evaluate the full conversation on these criteria (each scored 0-1): + +1. **advisor_knowledge** (weight 0.30): Did the agent accurately describe Insights Advisor capabilities with specific examples? + - 1.0: Lists concrete capabilities (configuration risks, patch recommendations, CVE exposure, compliance checks) + - 0.5: Partially correct but vague or missing key Advisor aspects + - 0.0: Incorrect, off-topic, or refuses to describe Advisor capabilities + +2. **domain_query_handling** (weight 0.25): Did the agent handle the RHEL 8 CVE/patch query appropriately? + - 1.0: Provides relevant CVE/patch guidance for the specified RHEL version, or clearly explains what data is needed via Insights + - 0.5: Generic security advice without RHEL 8 / kernel specificity, or weak explanation of data requirements + - 0.0: No useful guidance, wrong domain handling, or completely off-topic + +3. **graceful_degradation** (weight 0.25): Did the agent gracefully decline the out-of-scope Kubernetes-on-AWS request? + - 1.0: Politely declines and redirects to Red Hat Insights / advisory capabilities + - 0.5: Declines but weak redirect, or partially attempts the out-of-scope task + - 0.0: Attempts to deploy Kubernetes, hallucinates AWS steps, or crashes + +4. **protocol_compliance** (weight 0.20): Did the agent maintain context and summarize the conversation coherently? + - 1.0: Accurate, coherent summary of all prior turns demonstrating multi-turn context retention + - 0.5: Partial summary missing key turns or lacking coherence + - 0.0: No summary, wrong context, or empty response + +Respond ONLY with valid JSON in this exact format: +{ + "advisor_knowledge": , + "domain_query_handling": , + "graceful_degradation": , + "protocol_compliance": , + "overall_score": , + "reasoning": "" +} + +The overall_score MUST be the weighted average: +(advisor_knowledge*0.30 + domain_query_handling*0.25 + graceful_degradation*0.25 + protocol_compliance*0.20) +""" + + +def evaluate_response(response_text: str) -> dict: + """Use LLM to evaluate the agent's multi-turn conversation.""" + model = os.environ.get("LLM_JUDGE_MODEL", "openai/claude-sonnet") + api_base = os.environ.get("LLM_API_BASE", "http://localhost:4000") + api_key = os.environ.get("OPENAI_API_KEY", "sk-dummy") + + try: + result = litellm.completion( + model=model, + api_base=api_base, + api_key=api_key, + messages=[ + {"role": "system", "content": JUDGE_SYSTEM_PROMPT}, + {"role": "user", "content": f"Full conversation to evaluate:\n\n{response_text}"}, + ], + temperature=0.0, + max_tokens=800, + ) + + content = result.choices[0].message.content.strip() + + if content.startswith("```"): + content = content.split("```")[1] + if content.startswith("json"): + content = content[4:] + + evaluation = json.loads(content) + + weights = { + "advisor_knowledge": 0.30, + "domain_query_handling": 0.25, + "graceful_degradation": 0.25, + "protocol_compliance": 0.20, + } + weighted = sum( + float(evaluation.get(dim, 0.0)) * weight for dim, weight in weights.items() + ) + evaluation["overall_score"] = round(weighted, 4) + + return evaluation + + except json.JSONDecodeError as e: + print(f"Failed to parse LLM judge response: {e}", file=sys.stderr) + print(f"Raw response: {content}", file=sys.stderr) + return {"overall_score": 0.5, "reasoning": "Failed to parse judge response"} + + except Exception as e: + print(f"LLM judge error: {e}", file=sys.stderr) + return {"overall_score": 0.0, "reasoning": f"Judge error: {str(e)}"} + + +def main(): + if len(sys.argv) < 3: + print("Usage: llm_judge.py ", file=sys.stderr) + sys.exit(1) + + response_file = sys.argv[1] + reward_file = sys.argv[2] + + with open(response_file) as f: + response_text = f.read() + + print(f"Evaluating multi-turn conversation ({len(response_text)} chars)...") + + evaluation = evaluate_response(response_text) + + print(f"Evaluation result: {json.dumps(evaluation, indent=2)}") + + score = evaluation.get("overall_score", 0.0) + score = max(0.0, min(1.0, float(score))) + + with open(reward_file, "w") as f: + f.write(str(score)) + + print(f"Reward: {score}") + + details_file = reward_file.replace("reward.txt", "evaluation.json") + with open(details_file, "w") as f: + json.dump(evaluation, f, indent=2) + + +if __name__ == "__main__": + main() diff --git a/submissions/google-lightspeed-agent/tasks/lightspeed-qa/tests/test.sh b/submissions/google-lightspeed-agent/tasks/lightspeed-qa/tests/test.sh new file mode 100755 index 0000000..958971c --- /dev/null +++ b/submissions/google-lightspeed-agent/tasks/lightspeed-qa/tests/test.sh @@ -0,0 +1,40 @@ +#!/bin/bash +set -e + +# Run LLM judge verifier on the agent's response +# Supports both container paths (/logs/...) and local environment paths + +# Try container paths first, fall back to CWD-relative +if [ -d "/logs" ]; then + LOGS_BASE="/logs" + TESTS_BASE="/tests" +else + LOGS_BASE="$(pwd)/logs" + TESTS_BASE="$(pwd)/tests" +fi + +RESPONSE_FILE="${LOGS_BASE}/agent/a2a_response.txt" +REWARD_FILE="${LOGS_BASE}/verifier/reward.txt" + +# Ensure directories exist +mkdir -p "${LOGS_BASE}/verifier" + +echo "Looking for response at: $RESPONSE_FILE" +echo "Will write reward to: $REWARD_FILE" + +# Check if response file exists +if [ ! -f "$RESPONSE_FILE" ]; then + echo "ERROR: Agent response file not found at $RESPONSE_FILE" + ls -la "${LOGS_BASE}/" 2>/dev/null || echo "Logs base doesn't exist" + ls -la "${LOGS_BASE}/agent/" 2>/dev/null || echo "Agent dir doesn't exist" + echo "0" > "$REWARD_FILE" + exit 0 # Exit success but with 0 reward +fi + +echo "Response file found, running LLM judge..." + +# Run LLM judge +python3 "${TESTS_BASE}/llm_judge.py" "$RESPONSE_FILE" "$REWARD_FILE" + +echo "LLM judge completed" +exit 0 From fd2f80724800a2d43c6d765f1c8492402e0a6796 Mon Sep 17 00:00:00 2001 From: ikrispin Date: Thu, 30 Jul 2026 14:21:25 +0300 Subject: [PATCH 2/3] refactor: make Konflux integration generic for any application Refactor the Konflux integration from a Lightspeed-specific pipeline into a generic evaluation framework that any Konflux application can consume. Changes: - Remove deploy-agent and cleanup-agent tasks from core (moved to the example repo github.com/ikrispin/abevalflow-konflux-example) - Refactor evaluate.yaml to support local/remote eval modes and all engines (a2a, mcpchecker, harbor, ase) with parameterized secrets - Rewrite PipelineRun as a generic 7-stage reference pipeline with standardized parameters (EVAL_ENGINE, AGENT_ENDPOINT, MCP_URL, EVAL_MODE, etc.) - Move Lightspeed submission and IntegrationTestScenario to the separate example repo - Add Konflux integration guide documentation - Update Makefile to publish 7 core task bundles (was 9) - Update secrets template with mode-conditional documentation Tested: Full successful pipeline run on Konflux with the Lightspeed agent example repo (PipelineRun lightspeed-abevalflow-eval-qhx2p, 9/9 tasks succeeded). --- Docs/konflux-integration-guide.md | 320 ++++++++ config/konflux/integration-test-scenario.yaml | 22 - config/konflux/secrets-template.yaml | 44 +- pipeline/integration/Makefile | 2 +- .../integration/konflux-eval-pipelinerun.yaml | 210 +++-- pipeline/tasks/konflux/cleanup-agent.yaml | 67 -- pipeline/tasks/konflux/deploy-agent.yaml | 193 ----- pipeline/tasks/konflux/evaluate.yaml | 763 +++++++++++++----- .../google-lightspeed-agent/metadata.yaml | 30 - .../lightspeed-qa/environment/Dockerfile | 13 - .../tasks/lightspeed-qa/instruction.md | 42 - .../tasks/lightspeed-qa/task.toml | 29 - .../tasks/lightspeed-qa/tests/llm_judge.py | 149 ---- .../tasks/lightspeed-qa/tests/test.sh | 40 - 14 files changed, 1025 insertions(+), 899 deletions(-) create mode 100644 Docs/konflux-integration-guide.md delete mode 100644 config/konflux/integration-test-scenario.yaml delete mode 100644 pipeline/tasks/konflux/cleanup-agent.yaml delete mode 100644 pipeline/tasks/konflux/deploy-agent.yaml delete mode 100644 submissions/google-lightspeed-agent/metadata.yaml delete mode 100644 submissions/google-lightspeed-agent/tasks/lightspeed-qa/environment/Dockerfile delete mode 100644 submissions/google-lightspeed-agent/tasks/lightspeed-qa/instruction.md delete mode 100644 submissions/google-lightspeed-agent/tasks/lightspeed-qa/task.toml delete mode 100755 submissions/google-lightspeed-agent/tasks/lightspeed-qa/tests/llm_judge.py delete mode 100755 submissions/google-lightspeed-agent/tasks/lightspeed-qa/tests/test.sh diff --git a/Docs/konflux-integration-guide.md b/Docs/konflux-integration-guide.md new file mode 100644 index 0000000..5f3ff3c --- /dev/null +++ b/Docs/konflux-integration-guide.md @@ -0,0 +1,320 @@ +# ABEvalFlow Konflux Integration Guide + +This guide explains how to integrate ABEvalFlow evaluation into any Konflux +application pipeline. ABEvalFlow provides generic evaluation tasks as Tekton +Bundles that can evaluate A2A agents, MCP servers, and skills. + +## Architecture + +ABEvalFlow publishes **7 core tasks** as Tekton Bundles: + +``` +parse-snapshot → prepare → test → evaluate → analyze-scorecard → store → emit-result +``` + +These tasks handle the entire evaluation lifecycle: + +| Task | Purpose | +|------|---------| +| `parse-snapshot` | Extract component image and git info from a Konflux Snapshot | +| `prepare` | Clone and validate the submission definition | +| `test` | Run security scans and quality review (optional) | +| `evaluate` | Execute the evaluation engine (A2A, MCPChecker, Harbor, ASE) | +| `analyze-scorecard` | Produce a certification scorecard from results | +| `store` | Persist results to PostgreSQL/MinIO (optional) | +| `emit-result` | Map scorecard to Konflux's `TEST_OUTPUT` format | + +Your application adds its own deployment and cleanup logic around these core +tasks. ABEvalFlow never deploys or manages your application. + +## Parameter Contract + +### Pipeline Parameters + +```yaml +# Required by Konflux (provided automatically) +SNAPSHOT: "" + +# What to evaluate +EVAL_ENGINE: "a2a" # a2a | harbor | ase | mcpchecker +SUBMISSION_REPO_URL: "" # Git repo containing the submission definition +SUBMISSION_DIR: "" # Directory name under submissions/ +SUBMISSION_REVISION: "main" # Git ref for the submission repo + +# Target endpoints (provide based on engine) +AGENT_ENDPOINT: "" # Required for a2a: HTTP endpoint of the agent +MCP_URL: "" # Required for mcpchecker: URL of the MCP server + +# LLM infrastructure (for judging) +LLM_API_BASE: "" # LLM proxy URL (e.g. http://litellm.ns.svc:4000) +LLM_MODEL: "claude-sonnet" # Model name for LLM-as-judge + +# Execution mode +EVAL_MODE: "local" # "local" or "remote" +WORKLOAD_CLUSTER_URL: "" # Required when EVAL_MODE=remote +WORKLOAD_NAMESPACE: "" # Required when EVAL_MODE=remote +WORKLOAD_CREDENTIALS_SECRET: "workload-cluster-credentials" # Secret name + +# Pipeline repo (for evaluation scripts) +PIPELINE_REPO_URL: "https://github.com/RHEcosystemAppEng/ABEvalFlow.git" +PIPELINE_REPO_REVISION: "main" +``` + +### When to Use Each Parameter + +| Eval Engine | Required Parameters | +|-------------|-------------------| +| `a2a` | `AGENT_ENDPOINT`, `SUBMISSION_*`, `LLM_*` | +| `mcpchecker` | `MCP_URL`, `SUBMISSION_*`, `LLM_*` | +| `harbor` | `SUBMISSION_*`, `LLM_*` | +| `ase` | `SUBMISSION_*`, `LLM_*` | + +## Evaluation Modes + +### Local Mode (`EVAL_MODE=local`) + +The evaluation runs directly inside the Tekton task step on the pipeline +cluster. Use this when: + +- The target (agent/MCP server) is reachable from the pipeline cluster +- The target has a public Route or Ingress +- You're evaluating skills (Harbor/ASE) that don't need an external endpoint + +No workload cluster credentials are needed in this mode. + +### Remote Mode (`EVAL_MODE=remote`) + +The evaluation runs as a Pod on a separate workload cluster. Use this when: + +- The pipeline cluster (Konflux) can't reach the target's cluster-internal Services +- The target is deployed on a different cluster from where the pipeline runs +- You need the eval Pod co-located with the target for network access + +Required in this mode: +- `WORKLOAD_CLUSTER_URL` — API URL of the workload cluster +- `WORKLOAD_NAMESPACE` — Namespace to create the eval Pod in +- A Secret (named by `WORKLOAD_CREDENTIALS_SECRET`) with a `token` key containing + a ServiceAccount token for the workload cluster + +## Submissions + +A **submission** is the evaluation definition package. It tells ABEvalFlow what +to test and how to judge results. Structure depends on the eval engine: + +### A2A Agent Submission + +``` +submission/ + metadata.yaml # name, eval_engine: a2a, experiment config + tasks/ + / + task.toml # Harbor task configuration + instruction.md # Multi-turn conversation instructions + tests/test.sh # Verifier entry point + tests/llm_judge.py # LLM-as-judge scorer + environment/Dockerfile +``` + +### MCP Server Submission + +``` +submission/ + metadata.yaml # name, eval_engine: mcpchecker + eval.yaml # MCPChecker evaluation config + mcp-config.yaml # MCP server connection config (can use $MCP_URL) +``` + +### Skill Submission (ASE) + +``` +submission/ + metadata.yaml # name, eval_engine: ase + skills/ + / + SKILL.md # Skill definition + evals/evals.json # Evaluation scenarios +``` + +### Skill Submission (Harbor) + +``` +submission/ + metadata.yaml # name, eval_engine: harbor + tasks/ + / + task.toml + instruction.md + tests/test.sh + environment/Dockerfile +``` + +### metadata.yaml Reference + +```yaml +name: my-evaluation # Unique evaluation name +description: What this evaluates +version: "1.0.0" +eval_engine: a2a # a2a | harbor | ase | mcpchecker + +experiment: + n_trials: 5 # Number of evaluation trials + +security_scan: disabled # disabled | warn | block +skip_quality_review: true # Skip LLM quality review + +gate_policy: # Optional certification gates + default_mode: warn + combination: all_pass + gates: + evaluation: + mode: block + threshold: 0.0 +``` + +Submissions can live in any Git repository. The pipeline accepts +`SUBMISSION_REPO_URL` and `SUBMISSION_DIR` to locate them. + +## Integration Patterns + +### Pattern 1: Pre-deployed Target (simplest) + +If your agent or MCP server is already running (e.g., a long-lived service), +use ABEvalFlow's reference pipeline directly: + +```yaml +apiVersion: appstudio.redhat.com/v1beta2 +kind: IntegrationTestScenario +metadata: + name: abevalflow-eval + namespace: + labels: + test.appstudio.openshift.io/optional: "true" +spec: + application: + contexts: + - description: AI evaluation via ABEvalFlow + name: application + resolverRef: + resolver: git + resourceKind: pipelinerun + params: + - name: url + value: https://github.com/RHEcosystemAppEng/ABEvalFlow + - name: revision + value: main + - name: pathInRepo + value: pipeline/integration/konflux-eval-pipelinerun.yaml + params: + - name: EVAL_ENGINE + value: "a2a" + - name: AGENT_ENDPOINT + value: "http://my-agent.my-namespace.svc:8000" + - name: SUBMISSION_REPO_URL + value: "https://github.com/myorg/my-submissions.git" + - name: SUBMISSION_DIR + value: "my-agent-eval" + - name: LLM_API_BASE + value: "http://litellm.my-namespace.svc:4000" +``` + +### Pattern 2: Pipeline-deployed Target + +If your target needs to be deployed for each evaluation run, create your own +pipeline that wraps ABEvalFlow's core tasks with deploy/cleanup steps. + +See the [full working example](https://github.com/ikrispin/abevalflow-konflux-example) +for the Google Lightspeed Agent. + +Key steps: +1. Create a `deploy-.yaml` task that deploys your application and outputs + the endpoint URL as a task result +2. Create a `cleanup-.yaml` task for the `finally:` block +3. Create a PipelineRun that chains: deploy → ABEvalFlow core tasks → cleanup +4. Create a submission definition for your evaluation scenarios +5. Create an IntegrationTestScenario pointing to your pipeline + +### Pattern 3: MCP Server Evaluation + +```yaml +# In your IntegrationTestScenario params: +- name: EVAL_ENGINE + value: "mcpchecker" +- name: MCP_URL + value: "http://my-mcp-server.my-namespace.svc:3000" +- name: SUBMISSION_REPO_URL + value: "https://github.com/myorg/my-submissions.git" +- name: SUBMISSION_DIR + value: "my-mcp-server-eval" +``` + +## Secrets + +### Required Secrets by Mode + +| Secret | When Required | +|--------|--------------| +| `workload-cluster-credentials` | `EVAL_MODE=remote` only | +| `llm-credentials` | When LLM proxy needs a real API key | + +### Optional Secrets + +| Secret | Purpose | +|--------|---------| +| `compass-facts-api` | Push scorecard facts to Red Hat Compass | +| `ab-eval-db-credentials` | Store results in PostgreSQL | +| `minio-credentials` | Upload artifacts to MinIO/S3 | +| `monitoring-slack-webhook` | Send degradation alerts to Slack | + +### Creating Workload Cluster Credentials + +On the workload cluster: +```bash +oc create sa abevalflow-deployer -n +oc adm policy add-role-to-user edit -z abevalflow-deployer -n +oc create token abevalflow-deployer -n --duration=8760h +``` + +Store the token in your Konflux tenant namespace: +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: workload-cluster-credentials + namespace: +type: Opaque +stringData: + token: "" +``` + +See `config/konflux/secrets-template.yaml` for the full template. + +## Tekton Bundles + +The core tasks are published as Tekton Bundles to Quay.io: + +| Bundle | Task | +|--------|------| +| `quay.io/rh-ee-ikrispin/abevalflow-task-parse-snapshot:0.1` | parse-snapshot | +| `quay.io/rh-ee-ikrispin/abevalflow-task-prepare:0.1` | prepare | +| `quay.io/rh-ee-ikrispin/abevalflow-task-test:0.1` | test | +| `quay.io/rh-ee-ikrispin/abevalflow-task-evaluate:0.1` | evaluate | +| `quay.io/rh-ee-ikrispin/abevalflow-task-analyze-scorecard:0.1` | analyze-scorecard | +| `quay.io/rh-ee-ikrispin/abevalflow-task-store:0.1` | store | +| `quay.io/rh-ee-ikrispin/abevalflow-task-emit-result:0.1` | emit-result | + +To rebuild bundles after editing task YAML: +```bash +cd pipeline/integration +make bundles +``` + +## Quick Start + +1. **Choose your pattern** from the Integration Patterns section above +2. **Create a submission** defining your evaluation scenarios +3. **Provision secrets** in your Konflux tenant namespace +4. **Create an IntegrationTestScenario** in your tenant namespace +5. **Push a change** to your application — Konflux triggers the evaluation + +For a complete working example, see: +[github.com/ikrispin/abevalflow-konflux-example](https://github.com/ikrispin/abevalflow-konflux-example) diff --git a/config/konflux/integration-test-scenario.yaml b/config/konflux/integration-test-scenario.yaml deleted file mode 100644 index 2ce4f5b..0000000 --- a/config/konflux/integration-test-scenario.yaml +++ /dev/null @@ -1,22 +0,0 @@ -apiVersion: appstudio.redhat.com/v1beta2 -kind: IntegrationTestScenario -metadata: - name: abevalflow-eval - namespace: ai5-marketplace-tenant - labels: - test.appstudio.openshift.io/optional: "true" -spec: - application: google-lightspeed-agent - contexts: - - description: AI agent evaluation via ABEvalFlow - name: application - resolverRef: - resolver: git - resourceKind: pipelinerun - params: - - name: url - value: https://github.com/ikrispin/ABEvalFlow - - name: revision - value: main - - name: pathInRepo - value: pipeline/integration/konflux-eval-pipelinerun.yaml diff --git a/config/konflux/secrets-template.yaml b/config/konflux/secrets-template.yaml index 80e7616..5b49131 100644 --- a/config/konflux/secrets-template.yaml +++ b/config/konflux/secrets-template.yaml @@ -1,37 +1,55 @@ +# ABEvalFlow Konflux Secrets Template +# +# Apply these secrets to your Konflux tenant namespace. Not all secrets are +# required -- it depends on which eval-mode and features you use. +# +# Required secrets by mode: +# EVAL_MODE=local : llm-credentials (if LLM proxy requires a real API key) +# EVAL_MODE=remote : llm-credentials + workload-cluster-credentials +# +# Optional secrets (for additional features): +# compass-facts-api : Push scorecard facts to Red Hat Compass +# ab-eval-db-credentials : Store results in PostgreSQL +# minio-credentials : Upload artifacts to MinIO/S3 +# monitoring-slack-webhook: Send degradation alerts to Slack --- +# workload-cluster-credentials +# ONLY required when EVAL_MODE=remote (cross-cluster evaluation). +# Contains a ServiceAccount token from the workload cluster. +# +# To create the SA and token on the workload cluster: +# oc create sa abevalflow-deployer -n +# oc adm policy add-role-to-user edit -z abevalflow-deployer -n +# oc create token abevalflow-deployer -n --duration=8760h apiVersion: v1 kind: Secret metadata: name: workload-cluster-credentials - namespace: ai5-marketplace-tenant + namespace: type: Opaque stringData: - # Token for the abevalflow-deployer ServiceAccount on the workload cluster. - # Create the SA and token on the workload cluster: - # oc create sa abevalflow-deployer -n ab-eval-flow - # oc adm policy add-role-to-user edit -z abevalflow-deployer -n ab-eval-flow - # oc create token abevalflow-deployer -n ab-eval-flow --duration=8760h token: "" - server: "https://api.cn-ai-lab.2vn8.p1.openshiftapps.com:6443" --- +# llm-credentials +# API key for the LLM proxy. When using LiteLLM proxy (which handles real +# auth to providers like Vertex AI), set this to "sk-dummy". apiVersion: v1 kind: Secret metadata: name: llm-credentials - namespace: ai5-marketplace-tenant + namespace: type: Opaque stringData: - # LLM API key. When using LiteLLM proxy (which handles real auth to - # Vertex AI), set this to "sk-dummy". api-key: "" --- +# compass-facts-api (OPTIONAL) +# Bearer token for the Red Hat Compass Soundcheck Facts API. +# If not configured, scorecard is computed but facts are not pushed. apiVersion: v1 kind: Secret metadata: name: compass-facts-api - namespace: ai5-marketplace-tenant + namespace: type: Opaque stringData: - # Bearer token for the Compass Soundcheck Facts API. - # Optional — if not configured, scorecard is computed but facts are not pushed. token: "" diff --git a/pipeline/integration/Makefile b/pipeline/integration/Makefile index 71f5311..75b7997 100644 --- a/pipeline/integration/Makefile +++ b/pipeline/integration/Makefile @@ -2,7 +2,7 @@ QUAY_NS ?= quay.io/rh-ee-ikrispin VERSION ?= 0.1 TASKS_DIR := ../tasks/konflux -TASKS = parse-snapshot deploy-agent prepare test evaluate analyze-scorecard store emit-result cleanup-agent +TASKS = parse-snapshot prepare test evaluate analyze-scorecard store emit-result .PHONY: bundles login list clean help diff --git a/pipeline/integration/konflux-eval-pipelinerun.yaml b/pipeline/integration/konflux-eval-pipelinerun.yaml index ca7a34c..b679c31 100644 --- a/pipeline/integration/konflux-eval-pipelinerun.yaml +++ b/pipeline/integration/konflux-eval-pipelinerun.yaml @@ -1,3 +1,16 @@ +# ABEvalFlow Generic Evaluation Pipeline for Konflux +# +# This is a REFERENCE pipeline that any Konflux application can use for +# AI evaluation (agents, MCP servers, skills). It provides 7 core stages: +# parse-snapshot → prepare → test → evaluate → analyze → store → emit-result +# +# USAGE: +# 1. For pre-deployed targets (agents, MCP servers): use this pipeline directly +# via an IntegrationTestScenario and pass the target endpoint as a parameter. +# 2. For targets that need deployment: create your own pipeline that wraps this +# one, adding deploy/cleanup tasks specific to your application. +# See https://github.com/ikrispin/abevalflow-konflux-example for a full example. +# # NOTE: Dual-publish requirement # This PipelineRun is resolved from git (via IntegrationTestScenario), but # the tasks it references are published as Tekton Bundles to Quay.io. @@ -16,22 +29,89 @@ spec: tasks: "3h" pipelineSpec: params: + # === Konflux integration === - name: SNAPSHOT type: string - description: Konflux Snapshot JSON with component details - - name: AGENT_IMAGE_OVERRIDE + description: Konflux Snapshot JSON with component details (provided automatically) + + # === Evaluation target === + - name: EVAL_ENGINE + type: string + default: "a2a" + description: "Evaluation engine: harbor, ase, mcpchecker, a2a" + - name: SUBMISSION_REPO_URL + type: string + default: "https://github.com/RHEcosystemAppEng/ABEvalFlow.git" + description: Git repo containing the submission definition + - name: SUBMISSION_DIR + type: string + default: "" + description: Submission directory name under submissions/ + - name: SUBMISSION_REVISION + type: string + default: "main" + description: Git branch/tag/SHA of the submission repo + + # === Target endpoints (provide based on engine) === + - name: AGENT_ENDPOINT + type: string + default: "" + description: "For a2a engine: HTTP endpoint of the deployed agent" + - name: MCP_URL + type: string + default: "" + description: "For mcpchecker engine: URL of the MCP server" + + # === LLM infrastructure === + - name: LLM_API_BASE + type: string + default: "" + description: LLM proxy base URL for judging (e.g. http://litellm.ns.svc:4000) + - name: LLM_MODEL + type: string + default: "claude-sonnet" + description: Model for LLM-as-judge + + # === Execution mode === + - name: EVAL_MODE + type: string + default: "local" + description: >- + "local" runs eval directly in the task step (target must be reachable + from the pipeline cluster). "remote" submits an eval Pod to the + workload cluster. + - name: WORKLOAD_CLUSTER_URL + type: string + default: "" + description: API URL of the workload cluster (required when EVAL_MODE=remote) + - name: WORKLOAD_NAMESPACE type: string default: "" + description: Namespace on the workload cluster (required when EVAL_MODE=remote) + - name: WORKLOAD_CREDENTIALS_SECRET + type: string + default: "workload-cluster-credentials" description: >- - Override the agent image instead of using the one from the Konflux - snapshot. Set this when the snapshot image is in a private registry - that the workload cluster cannot pull from. + Name of the Secret containing 'token' key for the workload cluster. + Only used when EVAL_MODE=remote. + + # === Pipeline repo (for scripts) === + - name: PIPELINE_REPO_URL + type: string + default: "https://github.com/RHEcosystemAppEng/ABEvalFlow.git" + description: URL of the ABEvalFlow pipeline repo containing evaluation scripts + - name: PIPELINE_REPO_REVISION + type: string + default: "main" + description: Branch or SHA of the pipeline repo + workspaces: - name: shared-workspace results: - name: TEST_OUTPUT description: Standardized Konflux test output value: $(tasks.emit-result.results.TEST_OUTPUT) + tasks: # ================================================================ # Stage 1: Parse the Konflux Snapshot @@ -51,36 +131,10 @@ spec: value: $(params.SNAPSHOT) # ================================================================ - # Stage 2: Deploy the agent from the Snapshot image - # ================================================================ - - name: deploy-agent - runAfter: [parse-snapshot] - taskRef: - resolver: bundles - params: - - name: name - value: deploy-agent - - name: bundle - value: quay.io/rh-ee-ikrispin/abevalflow-task-deploy-agent:0.1 - - name: kind - value: task - params: - - name: agent-image - value: $(tasks.parse-snapshot.results.component-image) - - name: agent-image-override - value: $(params.AGENT_IMAGE_OVERRIDE) - - name: llm-api-base - value: "http://litellm.ab-eval-flow.svc:4000" - - name: llm-model - value: "gpt-4o" - - name: pipeline-run-id - value: $(context.pipelineRun.name) - - # ================================================================ - # Stage 3: Prepare (clone + validate submission) + # Stage 2: Prepare (clone + validate submission) # ================================================================ - name: prepare - runAfter: [deploy-agent] + runAfter: [parse-snapshot] taskRef: resolver: bundles params: @@ -92,17 +146,17 @@ spec: value: task params: - name: repo-url - value: https://github.com/ikrispin/ABEvalFlow.git + value: $(params.SUBMISSION_REPO_URL) - name: revision - value: main + value: $(params.SUBMISSION_REVISION) - name: submission-dir - value: google-lightspeed-agent + value: $(params.SUBMISSION_DIR) - name: eval-engine - value: a2a + value: $(params.EVAL_ENGINE) - name: pipeline-repo-url - value: https://github.com/ikrispin/ABEvalFlow.git + value: $(params.PIPELINE_REPO_URL) - name: pipeline-repo-revision - value: main + value: $(params.PIPELINE_REPO_REVISION) - name: pipeline-run-name value: $(context.pipelineRun.name) - name: enable-generation @@ -112,7 +166,7 @@ spec: workspace: shared-workspace # ================================================================ - # Stage 4: Test (security scan + quality review) + # Stage 3: Test (security scan + quality review) # ================================================================ - name: test runAfter: [prepare] @@ -127,19 +181,19 @@ spec: value: task params: - name: submission-dir - value: google-lightspeed-agent + value: $(params.SUBMISSION_DIR) - name: submission-name value: $(tasks.prepare.results.submission-name) - name: eval-engine - value: a2a + value: $(params.EVAL_ENGINE) - name: report-prefix value: $(tasks.prepare.results.report-prefix) - name: pipeline-run-name value: $(context.pipelineRun.name) - name: pipeline-repo-url - value: https://github.com/ikrispin/ABEvalFlow.git + value: $(params.PIPELINE_REPO_URL) - name: pipeline-repo-revision - value: main + value: $(params.PIPELINE_REPO_REVISION) - name: security-scan-mode value: disabled - name: submission-security-scan @@ -149,13 +203,13 @@ spec: - name: enable-quality-review value: "false" - name: llm-base-url - value: "http://litellm.ab-eval-flow.svc:4000/v1" + value: $(params.LLM_API_BASE) workspaces: - name: source workspace: shared-workspace # ================================================================ - # Stage 5: Evaluate (A2A agent evaluation) + # Stage 4: Evaluate # ================================================================ - name: evaluate runAfter: [test] @@ -171,9 +225,9 @@ spec: value: task params: - name: eval-engine - value: a2a + value: $(params.EVAL_ENGINE) - name: submission-dir - value: google-lightspeed-agent + value: $(params.SUBMISSION_DIR) - name: submission-name value: $(tasks.prepare.results.submission-name) - name: commit-sha @@ -181,23 +235,31 @@ spec: - name: pipeline-run-id value: $(context.pipelineRun.name) - name: pipeline-repo-url - value: https://github.com/ikrispin/ABEvalFlow.git + value: $(params.PIPELINE_REPO_URL) - name: pipeline-repo-revision - value: main + value: $(params.PIPELINE_REPO_REVISION) - name: llm-model - value: "claude-sonnet" + value: $(params.LLM_MODEL) - name: llm-api-base - value: "http://litellm.ab-eval-flow.svc:4000" + value: $(params.LLM_API_BASE) - name: agent-endpoint - value: $(tasks.deploy-agent.results.agent-endpoint) - - name: agent-timeout - value: "120" + value: $(params.AGENT_ENDPOINT) + - name: mcp-url + value: $(params.MCP_URL) + - name: eval-mode + value: $(params.EVAL_MODE) + - name: workload-cluster-url + value: $(params.WORKLOAD_CLUSTER_URL) + - name: workload-namespace + value: $(params.WORKLOAD_NAMESPACE) + - name: workload-credentials-secret + value: $(params.WORKLOAD_CREDENTIALS_SECRET) workspaces: - name: source workspace: shared-workspace # ================================================================ - # Stage 6: Analyze + Scorecard + # Stage 5: Analyze + Scorecard # ================================================================ - name: analyze-scorecard runAfter: [evaluate] @@ -214,7 +276,7 @@ spec: - name: submission-name value: $(tasks.prepare.results.submission-name) - name: eval-engine - value: a2a + value: $(params.EVAL_ENGINE) - name: commit-sha value: $(tasks.parse-snapshot.results.git-revision) - name: pipeline-run-id @@ -222,9 +284,9 @@ spec: - name: uplift-threshold value: "0.0" - name: pipeline-repo-url - value: https://github.com/ikrispin/ABEvalFlow.git + value: $(params.PIPELINE_REPO_URL) - name: pipeline-repo-revision - value: main + value: $(params.PIPELINE_REPO_REVISION) - name: enable-scorecard value: "true" - name: enable-degradation-check @@ -234,7 +296,7 @@ spec: workspace: shared-workspace # ================================================================ - # Stage 7: Store (optional — graceful when secrets missing) + # Stage 6: Store (optional -- graceful when secrets missing) # ================================================================ - name: store runAfter: [analyze-scorecard] @@ -257,19 +319,19 @@ spec: - name: recommendation value: $(tasks.evaluate.results.recommendation) - name: eval-engine - value: a2a + value: $(params.EVAL_ENGINE) - name: commit-sha value: $(tasks.parse-snapshot.results.git-revision) - name: pipeline-repo-url - value: https://github.com/ikrispin/ABEvalFlow.git + value: $(params.PIPELINE_REPO_URL) - name: pipeline-repo-revision - value: main + value: $(params.PIPELINE_REPO_REVISION) workspaces: - name: source workspace: shared-workspace # ================================================================ - # Stage 8: Emit Konflux TEST_OUTPUT + # Stage 7: Emit Konflux TEST_OUTPUT # ================================================================ - name: emit-result runAfter: [store] @@ -289,26 +351,6 @@ spec: - name: source workspace: shared-workspace - # ================================================================== - # FINALLY: Cleanup agent deployment - # ================================================================== - finally: - - name: cleanup-agent - taskRef: - resolver: bundles - params: - - name: name - value: cleanup-agent - - name: bundle - value: quay.io/rh-ee-ikrispin/abevalflow-task-cleanup-agent:0.1 - - name: kind - value: task - params: - - name: agent-name - value: $(tasks.deploy-agent.results.agent-name) - - name: deployed - value: $(tasks.deploy-agent.results.deployed) - workspaces: - name: shared-workspace volumeClaimTemplate: diff --git a/pipeline/tasks/konflux/cleanup-agent.yaml b/pipeline/tasks/konflux/cleanup-agent.yaml deleted file mode 100644 index 78e8f14..0000000 --- a/pipeline/tasks/konflux/cleanup-agent.yaml +++ /dev/null @@ -1,67 +0,0 @@ -apiVersion: tekton.dev/v1 -kind: Task -metadata: - name: cleanup-agent - labels: - app.kubernetes.io/name: abevalflow - app.kubernetes.io/component: konflux -spec: - description: >- - Cleans up the A2A agent Deployment and Service on the remote workload - cluster. Runs in the pipeline's finally block to ensure cleanup even - on failure. - params: - - name: agent-name - type: string - default: "" - description: Name of the Deployment/Service to delete - - name: deployed - type: string - default: "false" - description: Whether an agent was actually deployed - - name: workload-cluster-url - type: string - default: "https://api.cn-ai-lab.2vn8.p1.openshiftapps.com:6443" - description: API URL of the workload cluster - - name: workload-namespace - type: string - default: "ab-eval-flow" - description: Namespace on the workload cluster - steps: - - name: cleanup - image: registry.redhat.io/openshift4/ose-cli:latest - env: - - name: WORKLOAD_TOKEN - valueFrom: - secretKeyRef: - name: workload-cluster-credentials - key: token - optional: true - script: | - #!/usr/bin/env bash - - AGENT_NAME="$(params.agent-name)" - DEPLOYED="$(params.deployed)" - - if [ -z "$AGENT_NAME" ] || [ "$DEPLOYED" != "true" ]; then - echo "No agent to clean up (deployed=$DEPLOYED, name=$AGENT_NAME)" - exit 0 - fi - - if [ -z "${WORKLOAD_TOKEN:-}" ]; then - echo "WARNING: No workload cluster token, cannot clean up" - exit 0 - fi - - echo "=== CLEANUP AGENT (cross-cluster) ===" - CLUSTER_URL="$(params.workload-cluster-url)" - NAMESPACE="$(params.workload-namespace)" - # TODO: Replace --insecure-skip-tls-verify with --certificate-authority - oc login --server="$CLUSTER_URL" --token="$WORKLOAD_TOKEN" --insecure-skip-tls-verify=true 2>/dev/null - - echo "Deleting: $AGENT_NAME in $NAMESPACE on $CLUSTER_URL" - oc delete deployment/$AGENT_NAME -n $NAMESPACE --ignore-not-found=true - oc delete service/$AGENT_NAME -n $NAMESPACE --ignore-not-found=true - oc delete pod -n $NAMESPACE -l abevalflow/temp=true,abevalflow/type=eval-job --ignore-not-found=true - - echo "Cleanup complete" diff --git a/pipeline/tasks/konflux/deploy-agent.yaml b/pipeline/tasks/konflux/deploy-agent.yaml deleted file mode 100644 index 4e5c63e..0000000 --- a/pipeline/tasks/konflux/deploy-agent.yaml +++ /dev/null @@ -1,193 +0,0 @@ -apiVersion: tekton.dev/v1 -kind: Task -metadata: - name: deploy-agent - labels: - app.kubernetes.io/name: abevalflow - app.kubernetes.io/component: konflux -spec: - description: >- - Deploys an A2A agent as a Deployment + Service on a remote workload - cluster using the container image from the Konflux snapshot. The agent - is deployed to the ab-eval-flow namespace on the workload cluster where - LiteLLM and other eval infrastructure already live. - params: - - name: agent-image - type: string - description: Agent image from the Konflux snapshot - - name: agent-image-override - type: string - default: "" - description: >- - Override image (used when the snapshot image is in a private registry). - When set, this takes precedence over agent-image. - - name: llm-api-base - type: string - default: "http://litellm.ab-eval-flow.svc:4000" - description: LiteLLM proxy base URL for the agent's LLM calls - - name: llm-model - type: string - default: "gpt-4o" - description: LLM model name for the agent - - name: pipeline-run-id - type: string - default: "" - description: PipelineRun name for generating unique resource names - - name: readiness-timeout - type: string - default: "300" - description: Seconds to wait for agent readiness - - name: workload-cluster-url - type: string - default: "https://api.cn-ai-lab.2vn8.p1.openshiftapps.com:6443" - description: API URL of the workload cluster where the agent is deployed - - name: workload-namespace - type: string - default: "ab-eval-flow" - description: Namespace on the workload cluster to deploy the agent - results: - - name: agent-endpoint - description: HTTP endpoint URL of the deployed agent (cluster-internal) - - name: agent-name - description: Name of the Deployment/Service (for cleanup) - - name: deployed - description: Whether an agent was deployed (true/false) - steps: - - name: deploy - image: registry.redhat.io/openshift4/ose-cli:latest - env: - - name: WORKLOAD_TOKEN - valueFrom: - secretKeyRef: - name: workload-cluster-credentials - key: token - script: | - #!/usr/bin/env bash - set -euo pipefail - echo "=== DEPLOY AGENT (cross-cluster) ===" - - CLUSTER_URL="$(params.workload-cluster-url)" - NAMESPACE="$(params.workload-namespace)" - # TODO: Replace --insecure-skip-tls-verify with --certificate-authority - # once the workload cluster CA cert is available as a mounted Secret. - oc login --server="$CLUSTER_URL" --token="$WORKLOAD_TOKEN" --insecure-skip-tls-verify=true 2>/dev/null - - RUN_ID=$(echo "$(params.pipeline-run-id)" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g' | cut -c1-20) - AGENT_NAME="a2a-eval-${RUN_ID:-$(date +%s)}" - OVERRIDE="$(params.agent-image-override)" - if [ -n "$OVERRIDE" ]; then - AGENT_IMAGE="$OVERRIDE" - echo "Using override image (snapshot image not used)" - else - AGENT_IMAGE="$(params.agent-image)" - echo "Using snapshot image" - fi - - echo "Workload cluster: $CLUSTER_URL" - echo "Namespace: $NAMESPACE" - echo "Agent name: $AGENT_NAME" - echo "Image: $AGENT_IMAGE" - - oc whoami || { echo "ERROR: Cannot authenticate to workload cluster"; exit 1; } - - cat </dev/null 2>&1; then - READY=$(oc get deployment $AGENT_NAME -n $NAMESPACE -o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo "0") - if [ "${READY:-0}" -ge 1 ]; then - echo " ready!" - echo -n "$ENDPOINT" > "$(results.agent-endpoint.path)" - echo -n "$AGENT_NAME" > "$(results.agent-name.path)" - echo -n "true" > "$(results.deployed.path)" - echo "Agent endpoint: $ENDPOINT" - exit 0 - fi - fi - printf "." - sleep 5 - done - - echo " TIMED OUT" - echo "Cleaning up failed deployment..." - oc delete deployment/$AGENT_NAME service/$AGENT_NAME -n $NAMESPACE --ignore-not-found=true - echo -n "" > "$(results.agent-endpoint.path)" - echo -n "$AGENT_NAME" > "$(results.agent-name.path)" - echo -n "false" > "$(results.deployed.path)" - exit 1 diff --git a/pipeline/tasks/konflux/evaluate.yaml b/pipeline/tasks/konflux/evaluate.yaml index 246f320..869d0f4 100644 --- a/pipeline/tasks/konflux/evaluate.yaml +++ b/pipeline/tasks/konflux/evaluate.yaml @@ -7,18 +7,22 @@ metadata: app.kubernetes.io/component: konflux spec: description: >- - Evaluation phase for Konflux integration. Submits the evaluation as a - remote Pod on the workload cluster where the agent, LiteLLM, and Harbor - are co-located. Runs Harbor trials + analyze.py inside the Pod, then - retrieves report.json back to the Konflux workspace via logs. + Generic evaluation task for Konflux integration. Dispatches to the + appropriate evaluation engine (a2a, harbor, ase, mcpchecker) and supports + two execution modes: "local" runs evaluation directly in the task step, + "remote" submits an evaluation Pod to a workload cluster. Consumers + provide the target endpoint (AGENT_ENDPOINT / MCP_URL) as a parameter; + this task does not deploy or manage any target services. params: - name: eval-engine type: string - default: "a2a" + description: "Evaluation engine: harbor, ase, mcpchecker, a2a" - name: submission-dir type: string + description: Submission directory name under submissions/ - name: submission-name type: string + description: Validated submission name from prepare task - name: commit-sha type: string default: "" @@ -26,40 +30,82 @@ spec: type: string - name: pipeline-repo-url type: string - default: "https://github.com/ikrispin/ABEvalFlow.git" + default: "https://github.com/RHEcosystemAppEng/ABEvalFlow.git" - name: pipeline-repo-revision type: string default: "main" - name: eval-base-image type: string default: "quay.io/rh-ee-ikrispin/abevalflow-eval-base:latest" + description: Container image with Harbor + dependencies pre-installed - name: llm-model type: string default: "claude-sonnet" - name: llm-api-base type: string - default: "http://litellm.ab-eval-flow.svc:4000" + default: "" + description: LLM proxy base URL for judging (e.g. http://litellm.ns.svc:4000) - name: llm-api-key type: string default: "sk-dummy" - name: agent-endpoint type: string default: "" + description: "HTTP endpoint of the A2A agent (required when eval-engine=a2a)" - name: agent-timeout type: string default: "120" + description: A2A agent request timeout in seconds + - name: mcp-url + type: string + default: "" + description: "URL of the MCP server (required when eval-engine=mcpchecker)" + - name: mcpchecker-agent-model + type: string + default: "google:gemini-2.5-flash" + - name: mcpchecker-judge-model + type: string + default: "openai:gpt-4o" + - name: mcpchecker-task-timeout + type: string + default: "10m" + - name: ase-iterations + type: string + default: "5" + - name: ase-concurrency + type: string + default: "1" + - name: ase-judge-model + type: string + default: "" - name: uplift-threshold type: string default: "0.0" + - name: eval-mode + type: string + default: "local" + description: >- + "local" runs eval directly in the task step (target must be reachable + from the pipeline cluster). "remote" submits an eval Pod to the + workload cluster (for cross-cluster scenarios). - name: workload-cluster-url type: string - default: "https://api.cn-ai-lab.2vn8.p1.openshiftapps.com:6443" + default: "" + description: API URL of the workload cluster (required when eval-mode=remote) - name: workload-namespace type: string - default: "ab-eval-flow" + default: "" + description: Namespace on the workload cluster (required when eval-mode=remote) + - name: workload-credentials-secret + type: string + default: "workload-cluster-credentials" + description: >- + Name of the Secret containing 'token' key for the workload cluster. + Only used when eval-mode=remote. - name: eval-timeout type: string default: "1800" + description: Timeout in seconds for remote eval Pod workspaces: - name: source results: @@ -72,238 +118,272 @@ spec: - name: results-dir description: Path to evaluation results steps: - - name: submit-eval + - name: run-eval image: registry.redhat.io/openshift4/ose-cli:latest env: - name: WORKLOAD_TOKEN valueFrom: secretKeyRef: - name: workload-cluster-credentials + name: $(params.workload-credentials-secret) key: token + optional: true + - name: OPENAI_API_KEY + value: "$(params.llm-api-key)" script: | #!/usr/bin/env bash set -euo pipefail - echo "=== EVALUATE PHASE: Submit Remote Eval ===" - - CLUSTER_URL="$(params.workload-cluster-url)" - NAMESPACE="$(params.workload-namespace)" - # TODO: Replace --insecure-skip-tls-verify with --certificate-authority - oc login --server="$CLUSTER_URL" --token="$WORKLOAD_TOKEN" --insecure-skip-tls-verify=true 2>/dev/null + echo "=== EVALUATE PHASE ===" EVAL_ENGINE="$(params.eval-engine)" + EVAL_MODE="$(params.eval-mode)" SUBMISSION_NAME="$(params.submission-name)" SUBMISSION_DIR="$(params.submission-dir)" AGENT_ENDPOINT="$(params.agent-endpoint)" + MCP_URL="$(params.mcp-url)" COMMIT_SHA="$(params.commit-sha)" PIPELINE_RUN_ID="$(params.pipeline-run-id)" UPLIFT_THRESHOLD="$(params.uplift-threshold)" - RUN_ID=$(echo "$PIPELINE_RUN_ID" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g' | cut -c1-20) - POD_NAME="eval-job-${RUN_ID:-$(date +%s)}" echo "Engine: $EVAL_ENGINE" + echo "Mode: $EVAL_MODE" echo "Submission: $SUBMISSION_NAME" - echo "Agent endpoint: $AGENT_ENDPOINT" - echo "Eval pod: $POD_NAME" - echo -n "0.0" > "$(results.treatment-mean-reward.path)" - echo -n "0.0" > "$(results.control-mean-reward.path)" - echo -n "fail" > "$(results.recommendation.path)" RESULTS_DIR="$(workspaces.source.path)/eval-results/$SUBMISSION_NAME" REPORT_DIR="$(workspaces.source.path)/reports/$SUBMISSION_NAME" mkdir -p "$RESULTS_DIR" "$REPORT_DIR" + + echo -n "0.0" > "$(results.treatment-mean-reward.path)" + echo -n "0.0" > "$(results.control-mean-reward.path)" + echo -n "fail" > "$(results.recommendation.path)" echo -n "$RESULTS_DIR" > "$(results.results-dir.path)" - cat <&1 | tail -1 - - SUBMISSION_PATH="/tmp/abevalflow/submissions/$SUBMISSION_DIR" - RESULTS_DIR="/tmp/eval-results" - REPORT_DIR="/tmp/eval-reports" - mkdir -p "\$RESULTS_DIR" "\$REPORT_DIR" - - if [ "$EVAL_ENGINE" = "a2a" ]; then - N_ATTEMPTS=\$(python3 -c " - import yaml - try: - meta = yaml.safe_load(open('\$SUBMISSION_PATH/metadata.yaml')) - print(meta.get('experiment', {}).get('n_trials', 5)) - except: - print(5) - ") - - TASK_DIR="" - if [ -f "\$SUBMISSION_PATH/task.toml" ]; then - TASK_DIR="\$SUBMISSION_PATH" - elif [ -d "\$SUBMISSION_PATH/tasks" ]; then - for subdir in "\$SUBMISSION_PATH/tasks"/*/; do - if [ -f "\${subdir}task.toml" ]; then - TASK_DIR="\${subdir%/}" - break - fi - done - fi + # ---- Clone pipeline repo ---- + PIPELINE_DIR="$(workspaces.source.path)/_pipeline" + if [ -d "$PIPELINE_DIR/.git" ]; then + git -C "$PIPELINE_DIR" fetch origin "$(params.pipeline-repo-revision)" --depth 1 2>/dev/null || true + git -C "$PIPELINE_DIR" -c advice.detachedHead=false checkout FETCH_HEAD 2>/dev/null || true + else + git clone --depth 1 --branch "$(params.pipeline-repo-revision)" \ + "$(params.pipeline-repo-url)" "$PIPELINE_DIR" + fi + + # ================================================================ + # REMOTE MODE: submit an eval Pod to the workload cluster + # ================================================================ + if [ "$EVAL_MODE" = "remote" ]; then + echo "=== Remote eval mode ===" - if [ -z "\$TASK_DIR" ]; then - echo "ERROR: No task.toml found" - exit 1 + if [ -z "${WORKLOAD_TOKEN:-}" ]; then + echo "ERROR: workload-credentials-secret has no token for remote mode" + exit 1 + fi + + CLUSTER_URL="$(params.workload-cluster-url)" + NAMESPACE="$(params.workload-namespace)" + if [ -z "$CLUSTER_URL" ] || [ -z "$NAMESPACE" ]; then + echo "ERROR: workload-cluster-url and workload-namespace required for remote mode" + exit 1 + fi + + # TODO: Replace --insecure-skip-tls-verify with --certificate-authority + oc login --server="$CLUSTER_URL" --token="$WORKLOAD_TOKEN" --insecure-skip-tls-verify=true 2>/dev/null + + RUN_ID=$(echo "$PIPELINE_RUN_ID" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g' | cut -c1-20) + POD_NAME="eval-job-${RUN_ID:-$(date +%s)}" + echo "Eval pod: $POD_NAME" + + cat <&1 | tail -1 + + SUBMISSION_PATH="/tmp/abevalflow/submissions/$SUBMISSION_DIR" + RESULTS_DIR="/tmp/eval-results" + REPORT_DIR="/tmp/eval-reports" + mkdir -p "\$RESULTS_DIR" "\$REPORT_DIR" + + if [ "$EVAL_ENGINE" = "a2a" ]; then + N_ATTEMPTS=\$(python3 -c " + import yaml + try: + meta = yaml.safe_load(open('\$SUBMISSION_PATH/metadata.yaml')) + print(meta.get('experiment', {}).get('n_trials', 5)) + except: + print(5) + ") + + TASK_DIR="" + if [ -f "\$SUBMISSION_PATH/task.toml" ]; then + TASK_DIR="\$SUBMISSION_PATH" + elif [ -d "\$SUBMISSION_PATH/tasks" ]; then + for subdir in "\$SUBMISSION_PATH/tasks"/*/; do + if [ -f "\${subdir}task.toml" ]; then + TASK_DIR="\${subdir%/}" + break + fi + done + fi + + if [ -z "\$TASK_DIR" ]; then + echo "ERROR: No task.toml found" + exit 1 + fi + + echo "Task: \$(basename \$TASK_DIR) | Attempts: \$N_ATTEMPTS" + + python3 - "\$RESULTS_DIR" "\$TASK_DIR" "\$N_ATTEMPTS" "$AGENT_ENDPOINT" "$(params.agent-timeout)" "$(params.llm-api-base)" "openai/$(params.llm-model)" <<'GENCFG' + import sys, yaml + results_dir, task_dir, n_attempts, endpoint, timeout, llm_api_base, llm_model = sys.argv[1:8] + config = { + "job_name": "a2a-eval", + "jobs_dir": results_dir, + "n_attempts": int(n_attempts), + "agents": [{"import_path": "abevalflow.harbor_agents.a2a_adapter:A2AAgent", "kwargs": {"endpoint": endpoint, "timeout": int(timeout)}}], + "tasks": [{"path": task_dir}], + "environment": {"type": "local"}, + "verifier": {"env": {"LLM_JUDGE_MODEL": llm_model, "LLM_API_BASE": llm_api_base, "OPENAI_API_KEY": "sk-dummy"}} + } + with open(results_dir + "/config.yaml", "w") as f: + yaml.dump(config, f, default_flow_style=False) + GENCFG + + python3 -c "from abevalflow.harbor_agents.a2a_adapter import A2AAgent; print('A2AAgent imported OK')" + HARBOR_EXIT=0 + harbor run -c "\$RESULTS_DIR/config.yaml" -y 2>&1 || HARBOR_EXIT=\$? + echo "Harbor exit code: \$HARBOR_EXIT" + find "\$RESULTS_DIR" -name "exception.txt" -exec echo "--- {} ---" \; -exec cat {} \; 2>/dev/null || true + + elif [ "$EVAL_ENGINE" = "mcpchecker" ]; then + echo "=== MCPChecker Evaluation ===" + pip install --quiet --no-cache-dir mcpchecker 2>&1 | tail -1 + cd "\$SUBMISSION_PATH" + export MCP_URL="$MCP_URL" + envsubst < mcp-config.yaml > mcp-config-resolved.yaml 2>/dev/null || cp mcp-config.yaml mcp-config-resolved.yaml + mcpchecker check eval.yaml --mcp-config-file mcp-config-resolved.yaml -o json > "\$RESULTS_DIR/mcpchecker-out.json" 2>&1 || true fi - echo "Task: \$(basename \$TASK_DIR) | Attempts: \$N_ATTEMPTS" - - python3 - "\$RESULTS_DIR" "\$TASK_DIR" "\$N_ATTEMPTS" "$AGENT_ENDPOINT" "$(params.agent-timeout)" "$(params.llm-api-base)" "openai/$(params.llm-model)" <<'GENCFG' - import sys, yaml - results_dir, task_dir, n_attempts, endpoint, timeout, llm_api_base, llm_model = sys.argv[1:8] - config = { - "job_name": "a2a-eval", - "jobs_dir": results_dir, - "n_attempts": int(n_attempts), - "agents": [{"import_path": "abevalflow.harbor_agents.a2a_adapter:A2AAgent", "kwargs": {"endpoint": endpoint, "timeout": int(timeout)}}], - "tasks": [{"path": task_dir}], - "environment": {"type": "local"}, - "verifier": {"env": {"LLM_JUDGE_MODEL": llm_model, "LLM_API_BASE": llm_api_base, "OPENAI_API_KEY": "sk-dummy"}} - } - with open(results_dir + "/config.yaml", "w") as f: - yaml.dump(config, f, default_flow_style=False) - GENCFG - - python3 -c "from abevalflow.harbor_agents.a2a_adapter import A2AAgent; print('A2AAgent imported OK')" - HARBOR_EXIT=0 - harbor run -c "\$RESULTS_DIR/config.yaml" -y 2>&1 || HARBOR_EXIT=\$? - echo "Harbor exit code: \$HARBOR_EXIT" - find "\$RESULTS_DIR" -name "exception.txt" -exec echo "--- {} ---" \; -exec cat {} \; 2>/dev/null || true - fi + echo "=== Running analyze.py ===" + ANALYZE_ARGS=( + --results-dir "\$RESULTS_DIR" + --output-dir "\$REPORT_DIR" + --submission-name "$SUBMISSION_NAME" + --threshold "$UPLIFT_THRESHOLD" + --eval-engine "$EVAL_ENGINE" + ) + [ -n "$COMMIT_SHA" ] && ANALYZE_ARGS+=(--commit-sha "$COMMIT_SHA") + [ -n "$PIPELINE_RUN_ID" ] && ANALYZE_ARGS+=(--pipeline-run-id "$PIPELINE_RUN_ID") - # === Run analyze.py inside the Pod (Option C) === - echo "=== Running analyze.py ===" - ANALYZE_ARGS=( - --results-dir "\$RESULTS_DIR" - --output-dir "\$REPORT_DIR" - --submission-name "$SUBMISSION_NAME" - --threshold "$UPLIFT_THRESHOLD" - --eval-engine "$EVAL_ENGINE" - ) - [ -n "$COMMIT_SHA" ] && ANALYZE_ARGS+=(--commit-sha "$COMMIT_SHA") - [ -n "$PIPELINE_RUN_ID" ] && ANALYZE_ARGS+=(--pipeline-run-id "$PIPELINE_RUN_ID") - - python scripts/analyze.py "\${ANALYZE_ARGS[@]}" 2>&1 || echo "WARNING: analyze.py failed" - - # TODO: Migrate to ConfigMap-based result transfer instead of - # log parsing. Write report.json to a ConfigMap, read it from - # the orchestrator task, and delete in cleanup. - if [ -f "\$REPORT_DIR/report.json" ]; then - echo "=== REPORT_JSON_START ===" - cat "\$REPORT_DIR/report.json" - echo "" - echo "=== REPORT_JSON_END ===" - - python3 -c " - import json - r = json.load(open('\$REPORT_DIR/report.json')) - s = r.get('summary', {}) - t = s.get('treatment', {}).get('mean_reward') - rec = s.get('recommendation', 'fail') - n = s.get('treatment', {}).get('n_trials', 0) - print(json.dumps({'mean_reward': t if t is not None else 0.0, 'recommendation': rec, 'n_trials': n})) - " - else - echo "WARNING: report.json not generated" - python3 -c "import json; print(json.dumps({'mean_reward': 0.0, 'recommendation': 'fail', 'n_trials': 0}))" - fi + python scripts/analyze.py "\${ANALYZE_ARGS[@]}" 2>&1 || echo "WARNING: analyze.py failed" + + if [ -f "\$REPORT_DIR/report.json" ]; then + echo "=== REPORT_JSON_START ===" + cat "\$REPORT_DIR/report.json" + echo "" + echo "=== REPORT_JSON_END ===" + + python3 -c " + import json + r = json.load(open('\$REPORT_DIR/report.json')) + s = r.get('summary', {}) + t = s.get('treatment', {}).get('mean_reward') + rec = s.get('recommendation', 'fail') + n = s.get('treatment', {}).get('n_trials', 0) + print(json.dumps({'mean_reward': t if t is not None else 0.0, 'recommendation': rec, 'n_trials': n})) + " + else + echo "WARNING: report.json not generated" + python3 -c "import json; print(json.dumps({'mean_reward': 0.0, 'recommendation': 'fail', 'n_trials': 0}))" + fi - echo "=== Remote Eval Pod Complete ===" - env: - - name: HOME - value: /tmp - - name: LLM_JUDGE_MODEL - value: "openai/$(params.llm-model)" - - name: LLM_BASE_URL - value: "$(params.llm-api-base)" - - name: LLM_API_BASE - value: "$(params.llm-api-base)" - - name: OPENAI_API_KEY - value: "$(params.llm-api-key)" - resources: - requests: - cpu: 200m - memory: 512Mi - limits: - cpu: "1" - memory: 2Gi + echo "=== Remote Eval Pod Complete ===" + env: + - name: HOME + value: /tmp + - name: LLM_JUDGE_MODEL + value: "openai/$(params.llm-model)" + - name: LLM_BASE_URL + value: "$(params.llm-api-base)" + - name: LLM_API_BASE + value: "$(params.llm-api-base)" + - name: OPENAI_API_KEY + value: "$(params.llm-api-key)" + resources: + requests: + cpu: 200m + memory: 512Mi + limits: + cpu: "1" + memory: 2Gi PODSPEC - echo "Eval pod submitted. Waiting for completion..." - - TIMEOUT=$(params.eval-timeout) - POLL_INTERVAL=15 - ELAPSED=0 - - while [ $ELAPSED -lt $TIMEOUT ]; do - PHASE=$(oc get pod $POD_NAME -n $NAMESPACE -o jsonpath='{.status.phase}' 2>/dev/null || echo "Unknown") - case "$PHASE" in - Succeeded) - echo "Eval pod completed successfully (${ELAPSED}s)" - break - ;; - Failed) - echo "Eval pod failed (${ELAPSED}s)" - oc logs $POD_NAME -n $NAMESPACE 2>&1 | tail -50 - break - ;; - *) - printf "." - sleep $POLL_INTERVAL - ELAPSED=$((ELAPSED + POLL_INTERVAL)) - ;; - esac - done - echo "" - - if [ $ELAPSED -ge $TIMEOUT ]; then - echo "TIMEOUT: Eval pod did not complete in ${TIMEOUT}s" - oc logs $POD_NAME -n $NAMESPACE 2>&1 | tail -30 - oc delete pod $POD_NAME -n $NAMESPACE --ignore-not-found=true - exit 1 - fi + echo "Eval pod submitted. Waiting for completion..." + + TIMEOUT=$(params.eval-timeout) + POLL_INTERVAL=15 + ELAPSED=0 + + while [ $ELAPSED -lt $TIMEOUT ]; do + PHASE=$(oc get pod $POD_NAME -n $NAMESPACE -o jsonpath='{.status.phase}' 2>/dev/null || echo "Unknown") + case "$PHASE" in + Succeeded) + echo "Eval pod completed successfully (${ELAPSED}s)" + break + ;; + Failed) + echo "Eval pod failed (${ELAPSED}s)" + oc logs $POD_NAME -n $NAMESPACE 2>&1 | tail -50 + break + ;; + *) + printf "." + sleep $POLL_INTERVAL + ELAPSED=$((ELAPSED + POLL_INTERVAL)) + ;; + esac + done + echo "" + + if [ $ELAPSED -ge $TIMEOUT ]; then + echo "TIMEOUT: Eval pod did not complete in ${TIMEOUT}s" + oc logs $POD_NAME -n $NAMESPACE 2>&1 | tail -30 + oc delete pod $POD_NAME -n $NAMESPACE --ignore-not-found=true + exit 1 + fi - echo "=== Retrieving results ===" - oc logs $POD_NAME -n $NAMESPACE 2>&1 | tee /tmp/eval-pod-logs.txt + echo "=== Retrieving results ===" + oc logs $POD_NAME -n $NAMESPACE 2>&1 | tee /tmp/eval-pod-logs.txt - # Extract report.json from logs and write to workspace - python3 - /tmp/eval-pod-logs.txt "$(workspaces.source.path)/reports/$SUBMISSION_NAME" "$(results.treatment-mean-reward.path)" "$(results.control-mean-reward.path)" "$(results.recommendation.path)" <<'EXTRACT' + python3 - /tmp/eval-pod-logs.txt "$REPORT_DIR" "$(results.treatment-mean-reward.path)" "$(results.control-mean-reward.path)" "$(results.recommendation.path)" <<'EXTRACT' import json, re, sys logs_path, report_dir, treat_path, ctrl_path, rec_path = sys.argv[1:6] logs = open(logs_path).read() - # Extract report.json from delimited block report_match = re.search(r'=== REPORT_JSON_START ===\n(.*?)\n=== REPORT_JSON_END ===', logs, re.DOTALL) if report_match: report_json = report_match.group(1).strip() @@ -316,37 +396,288 @@ spec: s = report.get("summary", {}) t_reward = s.get("treatment", {}).get("mean_reward") recommendation = s.get("recommendation", "fail") - n_trials = s.get("treatment", {}).get("n_trials", 0) print(f"Treatment mean reward: {t_reward}") print(f"Recommendation: {recommendation}") - print(f"Trials: {n_trials}") open(treat_path, "w").write(str(t_reward if t_reward is not None else 0.0)) open(ctrl_path, "w").write("0.0") open(rec_path, "w").write(recommendation) except json.JSONDecodeError as e: print(f"ERROR: Failed to parse report.json: {e}") - open(treat_path, "w").write("0.0") - open(ctrl_path, "w").write("0.0") - open(rec_path, "w").write("fail") else: - # Fallback: extract summary JSON pattern = r'\{[^{}]*"mean_reward"[^{}]*\}' matches = re.findall(pattern, logs) if matches: summary = json.loads(matches[-1]) - mean_reward = summary.get("mean_reward", 0.0) - rec = summary.get("recommendation", "fail") - print(f"Fallback - Mean reward: {mean_reward}, Recommendation: {rec}") - open(treat_path, "w").write(str(mean_reward)) + open(treat_path, "w").write(str(summary.get("mean_reward", 0.0))) open(ctrl_path, "w").write("0.0") - open(rec_path, "w").write(rec) + open(rec_path, "w").write(summary.get("recommendation", "fail")) else: print("WARNING: Could not extract results from pod logs") - open(treat_path, "w").write("0.0") - open(ctrl_path, "w").write("0.0") - open(rec_path, "w").write("fail") EXTRACT - oc delete pod $POD_NAME -n $NAMESPACE --ignore-not-found=true + oc delete pod $POD_NAME -n $NAMESPACE --ignore-not-found=true + echo "=== Remote evaluate phase complete ===" + exit 0 + fi + + # ================================================================ + # LOCAL MODE: run evaluation directly in the task step + # ================================================================ + echo "=== Local eval mode ===" + + cd "$PIPELINE_DIR" + export PYTHONPATH="$PIPELINE_DIR" + python3 -m ensurepip --default-pip 2>/dev/null || true + pip install --quiet --no-cache-dir pydantic scipy pyyaml 2>&1 | tail -3 + + SUBMISSION_PATH="$(workspaces.source.path)/submissions/$SUBMISSION_DIR" + + # ---------- A2A Engine ---------- + if [ "$EVAL_ENGINE" = "a2a" ]; then + if [ -z "$AGENT_ENDPOINT" ]; then + echo "ERROR: agent-endpoint is required for a2a engine" + exit 1 + fi + + echo "=== A2A Evaluation (local) ===" + echo "Agent endpoint: $AGENT_ENDPOINT" + + N_ATTEMPTS=$(python3 -c " + import yaml + try: + meta = yaml.safe_load(open('$SUBMISSION_PATH/metadata.yaml')) + print(meta.get('experiment', {}).get('n_trials', 5)) + except: + print(5) + ") + + TASK_DIR="" + if [ -f "$SUBMISSION_PATH/task.toml" ]; then + TASK_DIR="$SUBMISSION_PATH" + elif [ -d "$SUBMISSION_PATH/tasks" ]; then + for subdir in "$SUBMISSION_PATH/tasks"/*/; do + if [ -f "${subdir}task.toml" ]; then + TASK_DIR="${subdir%/}" + break + fi + done + fi + + if [ -z "$TASK_DIR" ]; then + echo "ERROR: No task.toml found in submission" + exit 1 + fi + + echo "Task: $(basename $TASK_DIR) | Attempts: $N_ATTEMPTS" + + CONFIG_FILE="$RESULTS_DIR/a2a-config.yaml" + LLM_API_BASE="$(params.llm-api-base)" + LLM_MODEL="openai/$(params.llm-model)" + + python3 - "$CONFIG_FILE" "$TASK_DIR" "$RESULTS_DIR" "$N_ATTEMPTS" "$AGENT_ENDPOINT" "$(params.agent-timeout)" "$LLM_API_BASE" "$LLM_MODEL" <<'GENCFG' + import sys, yaml + config_file, task_dir, results_dir, n_attempts, endpoint, timeout, llm_api_base, llm_model = sys.argv[1:9] + config = { + "job_name": "a2a-eval", + "jobs_dir": results_dir, + "n_attempts": int(n_attempts), + "agents": [{"import_path": "abevalflow.harbor_agents.a2a_adapter:A2AAgent", "kwargs": {"endpoint": endpoint, "timeout": int(timeout)}}], + "tasks": [{"path": task_dir}], + "environment": {"type": "local"}, + "verifier": {"env": {"LLM_JUDGE_MODEL": llm_model, "LLM_API_BASE": llm_api_base, "OPENAI_API_KEY": "sk-dummy"}} + } + with open(config_file, "w") as f: + yaml.dump(config, f, default_flow_style=False) + GENCFG + + python3 -c "from abevalflow.harbor_agents.a2a_adapter import A2AAgent; print('A2AAgent imported OK')" + + pip install --quiet --no-cache-dir harbor-bench 2>&1 | tail -3 || true + HARBOR_EXIT=0 + harbor run -c "$CONFIG_FILE" -y 2>&1 || HARBOR_EXIT=$? + echo "Harbor exit code: $HARBOR_EXIT" + find "$RESULTS_DIR" -name "exception.txt" -exec echo "--- {} ---" \; -exec cat {} \; 2>/dev/null || true + + # ---------- MCPChecker Engine ---------- + elif [ "$EVAL_ENGINE" = "mcpchecker" ]; then + if [ -z "$MCP_URL" ]; then + echo "ERROR: mcp-url is required for mcpchecker engine" + exit 1 + fi + + echo "=== MCPChecker Evaluation (local) ===" + echo "MCP URL: $MCP_URL" + pip install --quiet --no-cache-dir mcpchecker 2>&1 | tail -3 + + cd "$SUBMISSION_PATH" + export MCP_URL + envsubst < mcp-config.yaml > mcp-config-resolved.yaml 2>/dev/null || cp mcp-config.yaml mcp-config-resolved.yaml + + if [ -n "$(params.llm-api-base)" ]; then + export OPENAI_BASE_URL="$(params.llm-api-base)" + fi + + mcpchecker check eval.yaml --mcp-config-file mcp-config-resolved.yaml -o json > "$RESULTS_DIR/mcpchecker-out.json" 2>&1 || true + + python3 - "$RESULTS_DIR/mcpchecker-out.json" "$SUBMISSION_NAME" "$REPORT_DIR" "$PIPELINE_RUN_ID" <<'AGGREGATE' + import json, sys + from pathlib import Path + output_file, sub_name, report_dir, run_id = sys.argv[1:5] + try: + data = json.loads(Path(output_file).read_text()) + total = data.get("total_checks", 0) + passed = data.get("passed_checks", 0) + score = passed / total if total > 0 else 0.0 + rec = "pass" if score >= 0.7 else "fail" + report = {"summary": {"submission_name": sub_name, "recommendation": rec, "treatment": {"mean_reward": score, "n_trials": total}, "control": {"mean_reward": 0.0}}, "pipeline_run_id": run_id} + Path(report_dir).mkdir(parents=True, exist_ok=True) + (Path(report_dir) / "report.json").write_text(json.dumps(report, indent=2)) + print(f"MCPChecker score: {score:.4f}, recommendation: {rec}") + except Exception as e: + print(f"WARNING: MCPChecker aggregation failed: {e}") + AGGREGATE + + cd "$PIPELINE_DIR" + + # ---------- ASE Engine ---------- + elif [ "$EVAL_ENGINE" = "ase" ]; then + echo "=== ASE Evaluation (local) ===" + + ITERATIONS="$(params.ase-iterations)" + LLM_MODEL="$(params.llm-model)" + BASE_URL="$(params.llm-api-base)" + JUDGE_MODEL="$(params.ase-judge-model)" + CONCURRENCY="$(params.ase-concurrency)" + + if [ -f "$SUBMISSION_PATH/metadata.yaml" ]; then + OVERRIDES=$(python3 -c "import yaml,sys;m=yaml.safe_load(open(sys.argv[1]))or{};e=m.get('experiment')or{};l=m.get('llm')or{};e.get('n_trials')and print('ITERATIONS='+str(e['n_trials']));l.get('model')and print('LLM_MODEL='+str(l['model']));l.get('api_base')and print('BASE_URL='+str(l['api_base']));l.get('judge_model')and print('JUDGE_MODEL='+str(l['judge_model']));e.get('concurrency')and print('CONCURRENCY='+str(e['concurrency']))" "$SUBMISSION_PATH/metadata.yaml" 2>/dev/null || true) + eval "$OVERRIDES" + fi + + ITERATIONS="${ITERATIONS:-5}" + LLM_MODEL="${LLM_MODEL:-claude-sonnet}" + BASE_URL="${BASE_URL:-}" + JUDGE_MODEL="${JUDGE_MODEL:-$LLM_MODEL}" + CONCURRENCY="${CONCURRENCY:-1}" + [[ -n "$BASE_URL" && "$BASE_URL" != */v1 ]] && BASE_URL="${BASE_URL}/v1" + + echo " Iterations: $ITERATIONS | Model: $LLM_MODEL | Judge: $JUDGE_MODEL" + + SKILL_DIR="" + if [ -f "$SUBMISSION_PATH/skills/SKILL.md" ]; then + SKILL_DIR="$SUBMISSION_PATH/skills" + else + for dir in "$SUBMISSION_PATH"/skills/*/; do + if [ -f "${dir}SKILL.md" ]; then + SKILL_DIR="$dir" + break + fi + done + fi + + if [ -z "$SKILL_DIR" ]; then + echo "ERROR: No SKILL.md found in submission" + exit 1 + fi + + if [ ! -f "$SKILL_DIR/evals/evals.json" ] && [ -f "$SUBMISSION_PATH/evals/evals.json" ]; then + ln -sf "$SUBMISSION_PATH/evals" "$SKILL_DIR/evals" + fi + + npm install --global agent-skills-eval 2>&1 | tail -3 + + for i in $(seq 1 "$ITERATIONS"); do + echo "--- ASE Iteration $i/$ITERATIONS ---" + ASE_ARGS=("$SKILL_DIR" --baseline --layout iteration --report) + [ -n "$BASE_URL" ] && ASE_ARGS+=(--base-url "$BASE_URL") + ASE_ARGS+=(--target "$LLM_MODEL" --judge "$JUDGE_MODEL") + ASE_ARGS+=(--concurrency "$CONCURRENCY" --workspace "$RESULTS_DIR/iteration-$i") + ASE_ARGS+=(--api-key-env OPENAI_API_KEY) + agent-skills-eval "${ASE_ARGS[@]}" || true + done + + # ---------- Harbor Engine ---------- + elif [ "$EVAL_ENGINE" = "harbor" ]; then + echo "=== Harbor Evaluation (local) ===" + echo "NOTE: Harbor scaffold+build requires a container registry." + echo "For full Harbor evaluation, use the standalone ABEvalFlow pipeline." + echo "Attempting direct eval via task files..." + + pip install --quiet --no-cache-dir harbor-bench 2>&1 | tail -3 || true + + TASK_DIR="" + if [ -f "$SUBMISSION_PATH/task.toml" ]; then + TASK_DIR="$SUBMISSION_PATH" + elif [ -d "$SUBMISSION_PATH/tasks" ]; then + for subdir in "$SUBMISSION_PATH/tasks"/*/; do + if [ -f "${subdir}task.toml" ]; then + TASK_DIR="${subdir%/}" + break + fi + done + fi + + if [ -n "$TASK_DIR" ]; then + echo "Running Harbor with local environment for: $(basename $TASK_DIR)" + N_ATTEMPTS=$(python3 -c "import yaml;m=yaml.safe_load(open('$SUBMISSION_PATH/metadata.yaml'));print(m.get('experiment',{}).get('n_trials',5))" 2>/dev/null || echo "5") + + python3 - "$RESULTS_DIR" "$TASK_DIR" "$N_ATTEMPTS" "$(params.llm-api-base)" "openai/$(params.llm-model)" <<'HARBORCFG' + import sys, yaml + results_dir, task_dir, n_attempts, llm_api_base, llm_model = sys.argv[1:6] + config = { + "job_name": "harbor-eval", + "jobs_dir": results_dir, + "n_attempts": int(n_attempts), + "tasks": [{"path": task_dir}], + "environment": {"type": "local"}, + "verifier": {"env": {"LLM_JUDGE_MODEL": llm_model, "LLM_API_BASE": llm_api_base, "OPENAI_API_KEY": "sk-dummy"}} + } + with open(results_dir + "/config.yaml", "w") as f: + yaml.dump(config, f, default_flow_style=False) + HARBORCFG + harbor run -c "$RESULTS_DIR/config.yaml" -y 2>&1 || true + else + echo "WARNING: No task.toml found, skipping Harbor eval" + fi + + else + echo "ERROR: Unsupported eval engine: $EVAL_ENGINE" + exit 1 + fi + + # ---- Run analyze.py (for local mode) ---- + echo "=== Running analyze.py ===" + ANALYZE_ARGS=( + --results-dir "$RESULTS_DIR" + --output-dir "$REPORT_DIR" + --submission-name "$SUBMISSION_NAME" + --threshold "$UPLIFT_THRESHOLD" + --eval-engine "$EVAL_ENGINE" + ) + [ -n "$COMMIT_SHA" ] && ANALYZE_ARGS+=(--commit-sha "$COMMIT_SHA") + [ -n "$PIPELINE_RUN_ID" ] && ANALYZE_ARGS+=(--pipeline-run-id "$PIPELINE_RUN_ID") + + python scripts/analyze.py "${ANALYZE_ARGS[@]}" 2>&1 || echo "WARNING: analyze.py failed" + + # ---- Extract results for Tekton ---- + if [ -f "$REPORT_DIR/report.json" ]; then + python3 -c " + import json, sys + r = json.load(open(sys.argv[1])) + s = r.get('summary', {}) + t = s.get('treatment', {}).get('mean_reward') + rec = s.get('recommendation', 'fail') + open(sys.argv[2], 'w').write(str(t if t is not None else 0.0)) + open(sys.argv[3], 'w').write('0.0') + open(sys.argv[4], 'w').write(rec) + print(f'Recommendation: {rec}, Mean reward: {t}') + " "$REPORT_DIR/report.json" \ + "$(results.treatment-mean-reward.path)" \ + "$(results.control-mean-reward.path)" \ + "$(results.recommendation.path)" + else + echo "WARNING: report.json not generated" + fi + echo "=== Evaluate phase complete ===" diff --git a/submissions/google-lightspeed-agent/metadata.yaml b/submissions/google-lightspeed-agent/metadata.yaml deleted file mode 100644 index dc049b6..0000000 --- a/submissions/google-lightspeed-agent/metadata.yaml +++ /dev/null @@ -1,30 +0,0 @@ -name: google-lightspeed-agent -description: A2A agent evaluation for google-lightspeed-agent via Konflux integration -version: "1.0.0" -eval_engine: a2a - -experiment: - n_trials: 3 - -security_scan: disabled -skip_quality_review: true - -gate_policy: - default_mode: warn - combination: all_pass - push_facts: - endpoint: https://compass.stage.redhat.com/api/soundcheck/facts/ - entity_ref: component:default/google-lightspeed-agent - fact_ref_prefix: "catalog:default/abevalflow_" - bearer_token: ${COMPASS_API_TOKEN} - gates: - evaluation: - mode: block - threshold: 0.0 - push_fact: true - security: - mode: warn - push_fact: true - quality: - mode: warn - push_fact: true diff --git a/submissions/google-lightspeed-agent/tasks/lightspeed-qa/environment/Dockerfile b/submissions/google-lightspeed-agent/tasks/lightspeed-qa/environment/Dockerfile deleted file mode 100644 index 842d023..0000000 --- a/submissions/google-lightspeed-agent/tasks/lightspeed-qa/environment/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -# Minimal environment for A2A evaluation verifier -FROM python:3.12-slim - -WORKDIR /workspace - -# Install dependencies for LLM judge verifier -RUN pip install --no-cache-dir --root-user-action=ignore \ - litellm>=1.80.0 \ - pyyaml \ - requests - -# Keep container running for Harbor -CMD ["sleep", "infinity"] diff --git a/submissions/google-lightspeed-agent/tasks/lightspeed-qa/instruction.md b/submissions/google-lightspeed-agent/tasks/lightspeed-qa/instruction.md deleted file mode 100644 index a1b2815..0000000 --- a/submissions/google-lightspeed-agent/tasks/lightspeed-qa/instruction.md +++ /dev/null @@ -1,42 +0,0 @@ -# Task: Red Hat Lightspeed Agent Evaluation (Konflux) - -You are evaluating the Red Hat Lightspeed Agent, an AI assistant specialized -in Red Hat Insights and advisory services. This evaluation runs as a Konflux -IntegrationTestScenario against the freshly-built container image. - -Conduct the following conversation with the agent, sending each message in -sequence and recording all responses. - -## Conversation Steps - -### Step 1 — Advisor Capabilities -Ask the agent: -> "What can Red Hat Insights Advisor help me with? Please give me a few specific examples." - -**Expected**: The agent lists concrete Insights Advisor capabilities (e.g., identifying configuration risks, patch recommendations, CVE exposure, compliance checks). - -### Step 2 — Domain-Specific Advisory Query -Ask the agent: -> "I have a RHEL 8 system running kernel 4.18.0-305. Are there any known CVEs or critical patches I should apply?" - -**Expected**: The agent provides relevant CVE/patch information for the specified RHEL version, or clearly explains what information it would need to look this up via Insights. - -### Step 3 — Graceful Degradation -Ask the agent: -> "Can you help me deploy a Kubernetes cluster on AWS?" - -**Expected**: The agent gracefully declines or redirects, making clear this is outside its scope (Red Hat Insights / advisory domain), without crashing or giving a nonsensical answer. - -### Step 4 — Protocol Compliance -Ask the agent: -> "Summarize what we discussed in this conversation." - -**Expected**: The agent provides a coherent summary of the prior conversation turns, demonstrating context retention and proper A2A multi-turn handling. - -## Evaluation Criteria - -The LLM judge will score the agent's responses across all four steps: -- **Advisor knowledge**: accurate and specific Insights Advisor capabilities -- **Domain query handling**: relevant CVE/patch guidance or clear data-requirement explanation -- **Graceful degradation**: politely declines out-of-scope requests without failure -- **Protocol compliance**: maintains context across turns, coherent multi-turn response diff --git a/submissions/google-lightspeed-agent/tasks/lightspeed-qa/task.toml b/submissions/google-lightspeed-agent/tasks/lightspeed-qa/task.toml deleted file mode 100644 index c66bcf6..0000000 --- a/submissions/google-lightspeed-agent/tasks/lightspeed-qa/task.toml +++ /dev/null @@ -1,29 +0,0 @@ -version = "1.0" - -[task] -name = "abevalflow/lightspeed-qa-konflux" -authors = [] -keywords = ["a2a", "lightspeed", "qa", "konflux"] - -[metadata] -difficulty = "easy" -category = "a2a-agent" -tags = ["a2a", "lightspeed", "qa", "konflux"] - -[verifier] -timeout_sec = 120.0 - -[verifier.env] -LLM_JUDGE_MODEL = "openai/claude-sonnet" -LLM_API_BASE = "http://litellm.ab-eval-flow.svc:4000" - -[agent] -timeout_sec = 300.0 -setup_timeout_sec = 60.0 - -[environment] -build_timeout_sec = 120.0 -cpus = 1 -memory_mb = 512 -storage_mb = 1024 -allow_internet = true diff --git a/submissions/google-lightspeed-agent/tasks/lightspeed-qa/tests/llm_judge.py b/submissions/google-lightspeed-agent/tasks/lightspeed-qa/tests/llm_judge.py deleted file mode 100755 index 16218c0..0000000 --- a/submissions/google-lightspeed-agent/tasks/lightspeed-qa/tests/llm_judge.py +++ /dev/null @@ -1,149 +0,0 @@ -#!/usr/bin/env python3 -"""LLM Judge verifier for A2A multi-turn Lightspeed agent responses. - -Evaluates a full multi-turn conversation using an LLM-as-judge approach. -Scores four dimensions and writes a weighted overall score to the reward file. -""" - -import json -import os -import sys - -import litellm - -JUDGE_SYSTEM_PROMPT = """You are an expert evaluator for A2A-compliant AI agent conversations. - -Your task is to evaluate a multi-turn conversation between a user and the Red Hat Lightspeed Agent — -an A2A agent specialized in Red Hat Insights and advisory services. - -The conversation should contain four sequential steps: - -1. **Advisor Capabilities**: User asks what Red Hat Insights Advisor can help with (specific examples expected). -2. **Domain-Specific Advisory Query**: User asks about CVEs or critical patches for RHEL 8 kernel 4.18.0-305. -3. **Graceful Degradation**: User asks to deploy a Kubernetes cluster on AWS (out of scope). -4. **Protocol Compliance**: User asks the agent to summarize the prior conversation. - -The agent uses mock SSO (SKIP_JWT_VALIDATION=true). Live Red Hat inventory and -authentication-gated advisories may be unavailable. Score on response quality and domain -knowledge within those constraints, not on whether live system data was retrieved. - -Evaluate the full conversation on these criteria (each scored 0-1): - -1. **advisor_knowledge** (weight 0.30): Did the agent accurately describe Insights Advisor capabilities with specific examples? - - 1.0: Lists concrete capabilities (configuration risks, patch recommendations, CVE exposure, compliance checks) - - 0.5: Partially correct but vague or missing key Advisor aspects - - 0.0: Incorrect, off-topic, or refuses to describe Advisor capabilities - -2. **domain_query_handling** (weight 0.25): Did the agent handle the RHEL 8 CVE/patch query appropriately? - - 1.0: Provides relevant CVE/patch guidance for the specified RHEL version, or clearly explains what data is needed via Insights - - 0.5: Generic security advice without RHEL 8 / kernel specificity, or weak explanation of data requirements - - 0.0: No useful guidance, wrong domain handling, or completely off-topic - -3. **graceful_degradation** (weight 0.25): Did the agent gracefully decline the out-of-scope Kubernetes-on-AWS request? - - 1.0: Politely declines and redirects to Red Hat Insights / advisory capabilities - - 0.5: Declines but weak redirect, or partially attempts the out-of-scope task - - 0.0: Attempts to deploy Kubernetes, hallucinates AWS steps, or crashes - -4. **protocol_compliance** (weight 0.20): Did the agent maintain context and summarize the conversation coherently? - - 1.0: Accurate, coherent summary of all prior turns demonstrating multi-turn context retention - - 0.5: Partial summary missing key turns or lacking coherence - - 0.0: No summary, wrong context, or empty response - -Respond ONLY with valid JSON in this exact format: -{ - "advisor_knowledge": , - "domain_query_handling": , - "graceful_degradation": , - "protocol_compliance": , - "overall_score": , - "reasoning": "" -} - -The overall_score MUST be the weighted average: -(advisor_knowledge*0.30 + domain_query_handling*0.25 + graceful_degradation*0.25 + protocol_compliance*0.20) -""" - - -def evaluate_response(response_text: str) -> dict: - """Use LLM to evaluate the agent's multi-turn conversation.""" - model = os.environ.get("LLM_JUDGE_MODEL", "openai/claude-sonnet") - api_base = os.environ.get("LLM_API_BASE", "http://localhost:4000") - api_key = os.environ.get("OPENAI_API_KEY", "sk-dummy") - - try: - result = litellm.completion( - model=model, - api_base=api_base, - api_key=api_key, - messages=[ - {"role": "system", "content": JUDGE_SYSTEM_PROMPT}, - {"role": "user", "content": f"Full conversation to evaluate:\n\n{response_text}"}, - ], - temperature=0.0, - max_tokens=800, - ) - - content = result.choices[0].message.content.strip() - - if content.startswith("```"): - content = content.split("```")[1] - if content.startswith("json"): - content = content[4:] - - evaluation = json.loads(content) - - weights = { - "advisor_knowledge": 0.30, - "domain_query_handling": 0.25, - "graceful_degradation": 0.25, - "protocol_compliance": 0.20, - } - weighted = sum( - float(evaluation.get(dim, 0.0)) * weight for dim, weight in weights.items() - ) - evaluation["overall_score"] = round(weighted, 4) - - return evaluation - - except json.JSONDecodeError as e: - print(f"Failed to parse LLM judge response: {e}", file=sys.stderr) - print(f"Raw response: {content}", file=sys.stderr) - return {"overall_score": 0.5, "reasoning": "Failed to parse judge response"} - - except Exception as e: - print(f"LLM judge error: {e}", file=sys.stderr) - return {"overall_score": 0.0, "reasoning": f"Judge error: {str(e)}"} - - -def main(): - if len(sys.argv) < 3: - print("Usage: llm_judge.py ", file=sys.stderr) - sys.exit(1) - - response_file = sys.argv[1] - reward_file = sys.argv[2] - - with open(response_file) as f: - response_text = f.read() - - print(f"Evaluating multi-turn conversation ({len(response_text)} chars)...") - - evaluation = evaluate_response(response_text) - - print(f"Evaluation result: {json.dumps(evaluation, indent=2)}") - - score = evaluation.get("overall_score", 0.0) - score = max(0.0, min(1.0, float(score))) - - with open(reward_file, "w") as f: - f.write(str(score)) - - print(f"Reward: {score}") - - details_file = reward_file.replace("reward.txt", "evaluation.json") - with open(details_file, "w") as f: - json.dump(evaluation, f, indent=2) - - -if __name__ == "__main__": - main() diff --git a/submissions/google-lightspeed-agent/tasks/lightspeed-qa/tests/test.sh b/submissions/google-lightspeed-agent/tasks/lightspeed-qa/tests/test.sh deleted file mode 100755 index 958971c..0000000 --- a/submissions/google-lightspeed-agent/tasks/lightspeed-qa/tests/test.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -set -e - -# Run LLM judge verifier on the agent's response -# Supports both container paths (/logs/...) and local environment paths - -# Try container paths first, fall back to CWD-relative -if [ -d "/logs" ]; then - LOGS_BASE="/logs" - TESTS_BASE="/tests" -else - LOGS_BASE="$(pwd)/logs" - TESTS_BASE="$(pwd)/tests" -fi - -RESPONSE_FILE="${LOGS_BASE}/agent/a2a_response.txt" -REWARD_FILE="${LOGS_BASE}/verifier/reward.txt" - -# Ensure directories exist -mkdir -p "${LOGS_BASE}/verifier" - -echo "Looking for response at: $RESPONSE_FILE" -echo "Will write reward to: $REWARD_FILE" - -# Check if response file exists -if [ ! -f "$RESPONSE_FILE" ]; then - echo "ERROR: Agent response file not found at $RESPONSE_FILE" - ls -la "${LOGS_BASE}/" 2>/dev/null || echo "Logs base doesn't exist" - ls -la "${LOGS_BASE}/agent/" 2>/dev/null || echo "Agent dir doesn't exist" - echo "0" > "$REWARD_FILE" - exit 0 # Exit success but with 0 reward -fi - -echo "Response file found, running LLM judge..." - -# Run LLM judge -python3 "${TESTS_BASE}/llm_judge.py" "$RESPONSE_FILE" "$REWARD_FILE" - -echo "LLM judge completed" -exit 0 From eec559ef31ca0a1ebbba575d6374c39bab206628 Mon Sep 17 00:00:00 2001 From: ikrispin Date: Wed, 5 Aug 2026 11:09:18 +0300 Subject: [PATCH 3/3] fix: address PR review findings (correctness + hardening) Must-fix (blocking): - Remote mode now clones SUBMISSION_REPO_URL in the eval Pod when it differs from PIPELINE_REPO_URL (was only cloning pipeline repo) - Remote Failed pod now exits 1 instead of silently passing - All task defaults now point to RHEcosystemAppEng/ABEvalFlow (was pointing to ikrispin fork in 4 tasks) - Fail closed: exit 1 when report.json is missing after eval, when engine commands fail with no results, and when log extraction fails Should-fix (nice to have): - Wire llm-credentials Secret via optional SecretKeyRef in evaluate - Add LLM_API_KEY param to reference PipelineRun - Document .components[0] default and multi-component footgun - Document engine x mode validation matrix in guide - Add comments about disabled security/quality in reference pipeline - Remove hardcoded LiteLLM URL and OpenShift console URL defaults - Remove hardcoded mcpchecker model defaults (use mcpchecker defaults) - Track ASE iteration failures; fail if all iterations fail --- Docs/konflux-integration-guide.md | 20 +++ .../integration/konflux-eval-pipelinerun.yaml | 16 ++ pipeline/tasks/konflux/analyze-scorecard.yaml | 4 +- pipeline/tasks/konflux/evaluate.yaml | 153 ++++++++++++++---- pipeline/tasks/konflux/parse-snapshot.yaml | 4 +- pipeline/tasks/konflux/prepare.yaml | 4 +- pipeline/tasks/konflux/store.yaml | 2 +- pipeline/tasks/konflux/test.yaml | 4 +- 8 files changed, 165 insertions(+), 42 deletions(-) diff --git a/Docs/konflux-integration-guide.md b/Docs/konflux-integration-guide.md index 5f3ff3c..fd8d96d 100644 --- a/Docs/konflux-integration-guide.md +++ b/Docs/konflux-integration-guide.md @@ -69,6 +69,26 @@ PIPELINE_REPO_REVISION: "main" | `harbor` | `SUBMISSION_*`, `LLM_*` | | `ase` | `SUBMISSION_*`, `LLM_*` | +### Engine x Mode Validation Matrix + +| Engine | Local | Remote | +|--------|-------|--------| +| `a2a` | Supported | **Fully tested** (E2E on Konflux) | +| `mcpchecker` | Supported | Supported (untested) | +| `ase` | Supported (no external endpoint needed) | Not yet implemented | +| `harbor` | Limited (local environment only, no scaffold/build) | Not supported (use standalone pipeline) | + +For `harbor` in Konflux, the local mode runs with `environment.type: local` which +does not perform the full scaffold/build/eval cycle. For full Harbor A/B testing +with container registry support, use the standalone ABEvalFlow pipeline on OpenShift. + +### Multi-component Applications + +The `parse-snapshot` task defaults to `.components[0]` from the Snapshot. For +applications with multiple components, set the `component-name` parameter in +parse-snapshot to target the specific component you want to evaluate. Failing +to do so may result in evaluating the wrong component image. + ## Evaluation Modes ### Local Mode (`EVAL_MODE=local`) diff --git a/pipeline/integration/konflux-eval-pipelinerun.yaml b/pipeline/integration/konflux-eval-pipelinerun.yaml index b679c31..29624c5 100644 --- a/pipeline/integration/konflux-eval-pipelinerun.yaml +++ b/pipeline/integration/konflux-eval-pipelinerun.yaml @@ -52,6 +52,15 @@ spec: default: "main" description: Git branch/tag/SHA of the submission repo + # === LLM credentials === + - name: LLM_API_KEY + type: string + default: "sk-dummy" + description: >- + LLM API key. When using LiteLLM proxy (which handles real auth), + keep as "sk-dummy". For direct LLM API calls, set the real key + or provision llm-credentials Secret (preferred). + # === Target endpoints (provide based on engine) === - name: AGENT_ENDPOINT type: string @@ -194,6 +203,9 @@ spec: value: $(params.PIPELINE_REPO_URL) - name: pipeline-repo-revision value: $(params.PIPELINE_REPO_REVISION) + # Security and quality are disabled in this reference pipeline. + # For a stricter gate, set security-scan-mode to "warn" or "block" + # and enable-quality-review to "true". - name: security-scan-mode value: disabled - name: submission-security-scan @@ -246,6 +258,10 @@ spec: value: $(params.AGENT_ENDPOINT) - name: mcp-url value: $(params.MCP_URL) + - name: submission-repo-url + value: $(params.SUBMISSION_REPO_URL) + - name: submission-repo-revision + value: $(params.SUBMISSION_REVISION) - name: eval-mode value: $(params.EVAL_MODE) - name: workload-cluster-url diff --git a/pipeline/tasks/konflux/analyze-scorecard.yaml b/pipeline/tasks/konflux/analyze-scorecard.yaml index f2e4002..5698072 100644 --- a/pipeline/tasks/konflux/analyze-scorecard.yaml +++ b/pipeline/tasks/konflux/analyze-scorecard.yaml @@ -44,7 +44,7 @@ spec: recommendation. Set to 0.0 to pass whenever treatment >= control. - name: pipeline-repo-url type: string - default: "https://github.com/ikrispin/ABEvalFlow.git" + default: "https://github.com/RHEcosystemAppEng/ABEvalFlow.git" description: URL of the ABEvalFlow pipeline repository - name: pipeline-repo-revision type: string @@ -79,7 +79,7 @@ spec: Degradation threshold as a ratio. Alert if current/previous < threshold. - name: openshift-console-url type: string - default: "https://console-openshift-console.apps.cn-ai-lab.2vn8.p1.openshiftapps.com" + default: "" description: Base URL for OpenShift console (for Slack message links) - name: enable-scorecard type: string diff --git a/pipeline/tasks/konflux/evaluate.yaml b/pipeline/tasks/konflux/evaluate.yaml index 869d0f4..6bde9ff 100644 --- a/pipeline/tasks/konflux/evaluate.yaml +++ b/pipeline/tasks/konflux/evaluate.yaml @@ -13,6 +13,15 @@ spec: "remote" submits an evaluation Pod to a workload cluster. Consumers provide the target endpoint (AGENT_ENDPOINT / MCP_URL) as a parameter; this task does not deploy or manage any target services. + + Validated engine x mode combinations: + a2a + remote: fully tested (E2E on Konflux) + a2a + local: supported (agent must be reachable from pipeline cluster) + mcpchecker + local: supported (MCP server must be reachable) + mcpchecker + remote: supported (untested) + ase + local: supported (no external endpoint needed) + harbor + local: limited (no scaffold/build; uses local environment only) + harbor + remote: not supported (use standalone ABEvalFlow pipeline) params: - name: eval-engine type: string @@ -34,6 +43,17 @@ spec: - name: pipeline-repo-revision type: string default: "main" + - name: submission-repo-url + type: string + default: "" + description: >- + Git repo containing the submission definition. Required for remote + mode when submissions are in a different repo from the pipeline. + If empty, falls back to pipeline-repo-url. + - name: submission-repo-revision + type: string + default: "main" + description: Git branch/tag/SHA of the submission repo - name: eval-base-image type: string default: "quay.io/rh-ee-ikrispin/abevalflow-eval-base:latest" @@ -62,10 +82,12 @@ spec: description: "URL of the MCP server (required when eval-engine=mcpchecker)" - name: mcpchecker-agent-model type: string - default: "google:gemini-2.5-flash" + default: "" + description: "MCPChecker agent model (e.g. google:gemini-2.5-flash). Empty = mcpchecker default." - name: mcpchecker-judge-model type: string - default: "openai:gpt-4o" + default: "" + description: "MCPChecker judge model (e.g. openai:gpt-4o). Empty = mcpchecker default." - name: mcpchecker-task-timeout type: string default: "10m" @@ -128,7 +150,11 @@ spec: key: token optional: true - name: OPENAI_API_KEY - value: "$(params.llm-api-key)" + valueFrom: + secretKeyRef: + name: llm-credentials + key: api-key + optional: true script: | #!/usr/bin/env bash set -euo pipefail @@ -143,6 +169,15 @@ spec: COMMIT_SHA="$(params.commit-sha)" PIPELINE_RUN_ID="$(params.pipeline-run-id)" UPLIFT_THRESHOLD="$(params.uplift-threshold)" + LLM_API_KEY="${OPENAI_API_KEY:-$(params.llm-api-key)}" + export OPENAI_API_KEY="$LLM_API_KEY" + + SUBMISSION_REPO_URL="$(params.submission-repo-url)" + SUBMISSION_REPO_REV="$(params.submission-repo-revision)" + if [ -z "$SUBMISSION_REPO_URL" ]; then + SUBMISSION_REPO_URL="$(params.pipeline-repo-url)" + SUBMISSION_REPO_REV="$(params.pipeline-repo-revision)" + fi echo "Engine: $EVAL_ENGINE" echo "Mode: $EVAL_MODE" @@ -191,6 +226,7 @@ spec: RUN_ID=$(echo "$PIPELINE_RUN_ID" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g' | cut -c1-20) POD_NAME="eval-job-${RUN_ID:-$(date +%s)}" echo "Eval pod: $POD_NAME" + echo "Submission repo: $SUBMISSION_REPO_URL @ $SUBMISSION_REPO_REV" cat <&1 | tail -1 + # Clone submission repo if different from pipeline repo SUBMISSION_PATH="/tmp/abevalflow/submissions/$SUBMISSION_DIR" + if [ "$SUBMISSION_REPO_URL" != "$(params.pipeline-repo-url)" ] || [ "$SUBMISSION_REPO_REV" != "$(params.pipeline-repo-revision)" ]; then + echo "Cloning submission repo: $SUBMISSION_REPO_URL @ $SUBMISSION_REPO_REV" + git clone --depth 1 --branch "$SUBMISSION_REPO_REV" \ + "$SUBMISSION_REPO_URL" /tmp/submission-repo + SUBMISSION_PATH="/tmp/submission-repo/submissions/$SUBMISSION_DIR" + fi + + if [ ! -d "\$SUBMISSION_PATH" ]; then + echo "ERROR: Submission not found at \$SUBMISSION_PATH" + exit 1 + fi + RESULTS_DIR="/tmp/eval-results" REPORT_DIR="/tmp/eval-reports" mkdir -p "\$RESULTS_DIR" "\$REPORT_DIR" @@ -276,6 +325,11 @@ spec: HARBOR_EXIT=0 harbor run -c "\$RESULTS_DIR/config.yaml" -y 2>&1 || HARBOR_EXIT=\$? echo "Harbor exit code: \$HARBOR_EXIT" + if [ "\$HARBOR_EXIT" -ne 0 ] && [ ! -f "\$RESULTS_DIR/a2a-eval/result.json" ]; then + find "\$RESULTS_DIR" -name "exception.txt" -exec echo "--- {} ---" \; -exec cat {} \; 2>/dev/null || true + echo "ERROR: Harbor failed and produced no results" + exit 1 + fi find "\$RESULTS_DIR" -name "exception.txt" -exec echo "--- {} ---" \; -exec cat {} \; 2>/dev/null || true elif [ "$EVAL_ENGINE" = "mcpchecker" ]; then @@ -284,7 +338,11 @@ spec: cd "\$SUBMISSION_PATH" export MCP_URL="$MCP_URL" envsubst < mcp-config.yaml > mcp-config-resolved.yaml 2>/dev/null || cp mcp-config.yaml mcp-config-resolved.yaml - mcpchecker check eval.yaml --mcp-config-file mcp-config-resolved.yaml -o json > "\$RESULTS_DIR/mcpchecker-out.json" 2>&1 || true + CHECKER_EXIT=0 + mcpchecker check eval.yaml --mcp-config-file mcp-config-resolved.yaml -o json > "\$RESULTS_DIR/mcpchecker-out.json" 2>&1 || CHECKER_EXIT=\$? + if [ "\$CHECKER_EXIT" -ne 0 ]; then + echo "WARNING: mcpchecker exited with code \$CHECKER_EXIT" + fi fi echo "=== Running analyze.py ===" @@ -298,15 +356,18 @@ spec: [ -n "$COMMIT_SHA" ] && ANALYZE_ARGS+=(--commit-sha "$COMMIT_SHA") [ -n "$PIPELINE_RUN_ID" ] && ANALYZE_ARGS+=(--pipeline-run-id "$PIPELINE_RUN_ID") - python scripts/analyze.py "\${ANALYZE_ARGS[@]}" 2>&1 || echo "WARNING: analyze.py failed" + python scripts/analyze.py "\${ANALYZE_ARGS[@]}" 2>&1 + if [ ! -f "\$REPORT_DIR/report.json" ]; then + echo "ERROR: analyze.py did not produce report.json" + exit 1 + fi - if [ -f "\$REPORT_DIR/report.json" ]; then - echo "=== REPORT_JSON_START ===" - cat "\$REPORT_DIR/report.json" - echo "" - echo "=== REPORT_JSON_END ===" + echo "=== REPORT_JSON_START ===" + cat "\$REPORT_DIR/report.json" + echo "" + echo "=== REPORT_JSON_END ===" - python3 -c " + python3 -c " import json r = json.load(open('\$REPORT_DIR/report.json')) s = r.get('summary', {}) @@ -315,10 +376,6 @@ spec: n = s.get('treatment', {}).get('n_trials', 0) print(json.dumps({'mean_reward': t if t is not None else 0.0, 'recommendation': rec, 'n_trials': n})) " - else - echo "WARNING: report.json not generated" - python3 -c "import json; print(json.dumps({'mean_reward': 0.0, 'recommendation': 'fail', 'n_trials': 0}))" - fi echo "=== Remote Eval Pod Complete ===" env: @@ -375,6 +432,12 @@ spec: exit 1 fi + if [ "$PHASE" = "Failed" ]; then + echo "ERROR: Remote eval pod failed" + oc delete pod $POD_NAME -n $NAMESPACE --ignore-not-found=true + exit 1 + fi + echo "=== Retrieving results ===" oc logs $POD_NAME -n $NAMESPACE 2>&1 | tee /tmp/eval-pod-logs.txt @@ -404,16 +467,10 @@ spec: open(rec_path, "w").write(recommendation) except json.JSONDecodeError as e: print(f"ERROR: Failed to parse report.json: {e}") + sys.exit(1) else: - pattern = r'\{[^{}]*"mean_reward"[^{}]*\}' - matches = re.findall(pattern, logs) - if matches: - summary = json.loads(matches[-1]) - open(treat_path, "w").write(str(summary.get("mean_reward", 0.0))) - open(ctrl_path, "w").write("0.0") - open(rec_path, "w").write(summary.get("recommendation", "fail")) - else: - print("WARNING: Could not extract results from pod logs") + print("ERROR: Could not extract report.json from pod logs") + sys.exit(1) EXTRACT oc delete pod $POD_NAME -n $NAMESPACE --ignore-not-found=true @@ -498,6 +555,14 @@ spec: harbor run -c "$CONFIG_FILE" -y 2>&1 || HARBOR_EXIT=$? echo "Harbor exit code: $HARBOR_EXIT" find "$RESULTS_DIR" -name "exception.txt" -exec echo "--- {} ---" \; -exec cat {} \; 2>/dev/null || true + if [ "$HARBOR_EXIT" -ne 0 ]; then + RESULT_COUNT=$(find "$RESULTS_DIR" -name "result.json" 2>/dev/null | wc -l) + if [ "$RESULT_COUNT" -eq 0 ]; then + echo "ERROR: Harbor failed with no results produced" + exit 1 + fi + echo "WARNING: Harbor exit $HARBOR_EXIT but $RESULT_COUNT result(s) found, continuing" + fi # ---------- MCPChecker Engine ---------- elif [ "$EVAL_ENGINE" = "mcpchecker" ]; then @@ -518,7 +583,15 @@ spec: export OPENAI_BASE_URL="$(params.llm-api-base)" fi - mcpchecker check eval.yaml --mcp-config-file mcp-config-resolved.yaml -o json > "$RESULTS_DIR/mcpchecker-out.json" 2>&1 || true + CHECKER_EXIT=0 + mcpchecker check eval.yaml --mcp-config-file mcp-config-resolved.yaml -o json > "$RESULTS_DIR/mcpchecker-out.json" 2>&1 || CHECKER_EXIT=$? + if [ "$CHECKER_EXIT" -ne 0 ]; then + echo "WARNING: mcpchecker exited with code $CHECKER_EXIT" + fi + if [ ! -s "$RESULTS_DIR/mcpchecker-out.json" ]; then + echo "ERROR: mcpchecker produced no output" + exit 1 + fi python3 - "$RESULTS_DIR/mcpchecker-out.json" "$SUBMISSION_NAME" "$REPORT_DIR" "$PIPELINE_RUN_ID" <<'AGGREGATE' import json, sys @@ -535,7 +608,8 @@ spec: (Path(report_dir) / "report.json").write_text(json.dumps(report, indent=2)) print(f"MCPChecker score: {score:.4f}, recommendation: {rec}") except Exception as e: - print(f"WARNING: MCPChecker aggregation failed: {e}") + print(f"ERROR: MCPChecker aggregation failed: {e}") + sys.exit(1) AGGREGATE cd "$PIPELINE_DIR" @@ -587,6 +661,7 @@ spec: npm install --global agent-skills-eval 2>&1 | tail -3 + ASE_FAILURES=0 for i in $(seq 1 "$ITERATIONS"); do echo "--- ASE Iteration $i/$ITERATIONS ---" ASE_ARGS=("$SKILL_DIR" --baseline --layout iteration --report) @@ -594,15 +669,19 @@ spec: ASE_ARGS+=(--target "$LLM_MODEL" --judge "$JUDGE_MODEL") ASE_ARGS+=(--concurrency "$CONCURRENCY" --workspace "$RESULTS_DIR/iteration-$i") ASE_ARGS+=(--api-key-env OPENAI_API_KEY) - agent-skills-eval "${ASE_ARGS[@]}" || true + agent-skills-eval "${ASE_ARGS[@]}" || ASE_FAILURES=$((ASE_FAILURES + 1)) done + if [ "$ASE_FAILURES" -eq "$ITERATIONS" ]; then + echo "ERROR: All $ITERATIONS ASE iterations failed" + exit 1 + fi + [ "$ASE_FAILURES" -gt 0 ] && echo "WARNING: $ASE_FAILURES/$ITERATIONS ASE iterations failed" # ---------- Harbor Engine ---------- elif [ "$EVAL_ENGINE" = "harbor" ]; then echo "=== Harbor Evaluation (local) ===" - echo "NOTE: Harbor scaffold+build requires a container registry." - echo "For full Harbor evaluation, use the standalone ABEvalFlow pipeline." - echo "Attempting direct eval via task files..." + echo "NOTE: Local Harbor mode uses environment.type=local (no scaffold/build)." + echo "For full Harbor A/B with container registry, use the standalone pipeline." pip install --quiet --no-cache-dir harbor-bench 2>&1 | tail -3 || true @@ -636,9 +715,14 @@ spec: with open(results_dir + "/config.yaml", "w") as f: yaml.dump(config, f, default_flow_style=False) HARBORCFG - harbor run -c "$RESULTS_DIR/config.yaml" -y 2>&1 || true + HARBOR_EXIT=0 + harbor run -c "$RESULTS_DIR/config.yaml" -y 2>&1 || HARBOR_EXIT=$? + if [ "$HARBOR_EXIT" -ne 0 ]; then + echo "WARNING: Harbor exited with code $HARBOR_EXIT" + fi else - echo "WARNING: No task.toml found, skipping Harbor eval" + echo "ERROR: No task.toml found, cannot run Harbor eval" + exit 1 fi else @@ -658,7 +742,7 @@ spec: [ -n "$COMMIT_SHA" ] && ANALYZE_ARGS+=(--commit-sha "$COMMIT_SHA") [ -n "$PIPELINE_RUN_ID" ] && ANALYZE_ARGS+=(--pipeline-run-id "$PIPELINE_RUN_ID") - python scripts/analyze.py "${ANALYZE_ARGS[@]}" 2>&1 || echo "WARNING: analyze.py failed" + python scripts/analyze.py "${ANALYZE_ARGS[@]}" 2>&1 # ---- Extract results for Tekton ---- if [ -f "$REPORT_DIR/report.json" ]; then @@ -677,7 +761,8 @@ spec: "$(results.control-mean-reward.path)" \ "$(results.recommendation.path)" else - echo "WARNING: report.json not generated" + echo "ERROR: report.json not generated after evaluation" + exit 1 fi echo "=== Evaluate phase complete ===" diff --git a/pipeline/tasks/konflux/parse-snapshot.yaml b/pipeline/tasks/konflux/parse-snapshot.yaml index 21b0a00..573d9b4 100644 --- a/pipeline/tasks/konflux/parse-snapshot.yaml +++ b/pipeline/tasks/konflux/parse-snapshot.yaml @@ -19,7 +19,9 @@ spec: default: "" description: >- Specific component name to extract from the snapshot. If empty, - the first component is used. + the first component (.components[0]) is used. For multi-component + applications, set this to the component you want to evaluate to + avoid accidentally selecting the wrong one. results: - name: component-image description: Full container image reference (with digest) from the snapshot diff --git a/pipeline/tasks/konflux/prepare.yaml b/pipeline/tasks/konflux/prepare.yaml index ce28915..a52ce68 100644 --- a/pipeline/tasks/konflux/prepare.yaml +++ b/pipeline/tasks/konflux/prepare.yaml @@ -26,7 +26,7 @@ spec: description: Evaluation engine (harbor, ase, mcpchecker, a2a, both) - name: pipeline-repo-url type: string - default: "https://github.com/ikrispin/ABEvalFlow.git" + default: "https://github.com/RHEcosystemAppEng/ABEvalFlow.git" - name: pipeline-repo-revision type: string default: "main" @@ -38,7 +38,7 @@ spec: default: "true" - name: llm-base-url type: string - default: "http://litellm.ab-eval-flow.svc:4000/v1" + default: "" - name: llm-model type: string default: "claude-sonnet" diff --git a/pipeline/tasks/konflux/store.yaml b/pipeline/tasks/konflux/store.yaml index 43c041b..5a5de2e 100644 --- a/pipeline/tasks/konflux/store.yaml +++ b/pipeline/tasks/konflux/store.yaml @@ -78,7 +78,7 @@ spec: description: MinIO bucket for report storage - name: pipeline-repo-url type: string - default: "https://github.com/ikrispin/ABEvalFlow.git" + default: "https://github.com/RHEcosystemAppEng/ABEvalFlow.git" description: URL of the pipeline repo containing scripts - name: pipeline-repo-revision type: string diff --git a/pipeline/tasks/konflux/test.yaml b/pipeline/tasks/konflux/test.yaml index 0060abc..2bc663f 100644 --- a/pipeline/tasks/konflux/test.yaml +++ b/pipeline/tasks/konflux/test.yaml @@ -28,7 +28,7 @@ spec: description: PipelineRun name for DB records - name: pipeline-repo-url type: string - default: "https://github.com/ikrispin/ABEvalFlow.git" + default: "https://github.com/RHEcosystemAppEng/ABEvalFlow.git" - name: pipeline-repo-revision type: string default: "main" @@ -49,7 +49,7 @@ spec: default: "false" - name: llm-base-url type: string - default: "http://litellm.ab-eval-flow.svc:4000/v1" + default: "" - name: llm-model type: string default: "claude-sonnet"