} */
+ async _fetch() {
+ try {
+ const resp = await fetch("summary.json");
+ if (!resp.ok) {
+ throw new Error(`${resp.status}`);
+ }
+ this.data = await resp.json();
+ this.error = null;
+ } catch (e) {
+ this.error = /** @type {Error} */ (e).message;
+ }
+ }
+
+ /** @override */
+ render() {
+ if (!this.data) {
+ return this.error ? html`${this.error}
` : html`loading...
`;
+ }
+ const data = this.data;
+ const now = this._now;
+
+ /**
+ * @param {string} iso_string
+ * @returns {string}
+ */
+ function format_age(iso_string) {
+ const ms = now - new Date(iso_string).getTime();
+ const mins = Math.floor(ms / 60000);
+ if (mins < 60) {
+ return `${mins}m`;
+ }
+ return `${Math.floor(mins / 60)}h${mins % 60}m`;
+ }
+
+ /** @param {Job} job */
+ function job_instances(job) {
+ return (job.observed_instances || []).filter((iid) => data.instances[iid]);
+ }
+
+ /**
+ * @param {string} slug
+ * @param {Job} job
+ */
+ function render_job(slug, job) {
+ const instances = job_instances(job).sort((a, b) =>
+ data.instances[a].launch_time.localeCompare(data.instances[b].launch_time),
+ );
+
+ const label = job.human || slug;
+ const slugCell = job.logs_visible ? html`${label}` : label;
+
+ return html`
+
+ | ${slugCell} |
+
+ ${
+ instances.length
+ ? instances.map((iid) => {
+ const inst = data.instances[iid];
+ return html`
+
+ | ${iid} |
+ ${inst.state} |
+ ${inst.ip || ""} |
+ ${format_age(inst.launch_time)} |
+
+ `;
+ })
+ : html`
+
+ |
+ queued |
+ |
+ |
+
+ `
+ }
+ `;
+ }
+
+ /** @param {Job} job */
+ function is_job_terminated(job) {
+ const iids = job_instances(job);
+ return (
+ iids.length > 0 &&
+ iids.every((iid) => ["terminated", "shutting-down"].includes(data.instances[iid].state))
+ );
+ }
+
+ /** @param {Job} job */
+ function newest_launch(job) {
+ const iids = job_instances(job);
+ if (!iids.length) {
+ return Infinity;
+ }
+ return Math.max(...iids.map((iid) => new Date(data.instances[iid].launch_time).getTime()));
+ }
+
+ return html`
+ cockpit CI
+ ${this.error ? html`fetch error: ${this.error}
` : ""}
+
+
+
+
+
+
+ | job |
+ state |
+ ip |
+ age |
+
+
+
+ ${Object.entries(data.jobs)
+ .filter(([, job]) => this._showTerminated || !is_job_terminated(job))
+ .sort(([, a], [, b]) => newest_launch(b) - newest_launch(a))
+ .map(([slug, job]) => render_job(slug, job))}
+
+
+ `;
+ }
+}
+
+customElements.define("ci-dashboard", CiDashboard);
diff --git a/lib/html/tsconfig.json b/lib/html/tsconfig.json
new file mode 100644
index 0000000000..bf0fde3cee
--- /dev/null
+++ b/lib/html/tsconfig.json
@@ -0,0 +1,13 @@
+{
+ "compilerOptions": {
+ "checkJs": true,
+ "strict": true,
+ "noEmit": true,
+ "noImplicitOverride": true,
+ "target": "ES2022",
+ "module": "ES2022",
+ "moduleResolution": "bundler"
+ },
+ "include": ["*/*.js"],
+ "files": ["types/lit-cdn.d.ts"]
+}
diff --git a/lib/html/types/lit-cdn.d.ts b/lib/html/types/lit-cdn.d.ts
new file mode 100644
index 0000000000..e5b18b9365
--- /dev/null
+++ b/lib/html/types/lit-cdn.d.ts
@@ -0,0 +1,27 @@
+declare module "https://cdn.jsdelivr.net/gh/lit/dist@3/all/lit-all.min.js" {
+ type PropertyDeclaration = {
+ type?: typeof String | typeof Number | typeof Boolean | typeof Array | typeof Object;
+ state?: boolean;
+ };
+
+ export type CSSResult = { cssText: string };
+
+ export class LitElement extends HTMLElement {
+ static properties: Record;
+ static styles: CSSResult | CSSResult[];
+ connectedCallback(): void;
+ disconnectedCallback(): void;
+ createRenderRoot(): HTMLElement | DocumentFragment;
+ requestUpdate(): void;
+ render(): unknown;
+ }
+
+ export function css(strings: TemplateStringsArray, ...values: unknown[]): CSSResult;
+ export function html(strings: TemplateStringsArray, ...values: unknown[]): unknown;
+ export const nothing: symbol;
+ export function repeat(
+ items: Iterable,
+ keyFn: (item: T) => unknown,
+ template: (item: T) => unknown,
+ ): unknown;
+}
diff --git a/lib/s3.py b/lib/s3.py
index fd063b71bf..1fa949b4a1 100644
--- a/lib/s3.py
+++ b/lib/s3.py
@@ -39,6 +39,9 @@ class S3Key(NamedTuple):
secret: str
token: str | None = None
+ def __str__(self) -> str:
+ return ' '.join(v for v in self if v is not None)
+
SHA256_NIL = hashlib.sha256(b'').hexdigest()
diff --git a/lib/stores.py b/lib/stores.py
index fe38ff6e13..55d292d4ea 100644
--- a/lib/stores.py
+++ b/lib/stores.py
@@ -17,12 +17,12 @@
from collections.abc import Sequence
+from lib.aws.account import CI_IMAGES_BUCKETS, LOGS_URL
from lib.directories import xdg_config_home
# hosted on the public internet, requires a private token for some/all images
IMAGE_STORES: Sequence[str] = (
- "https://cockpit-ci-images-fra.s3.eu-central-1.amazonaws.com/",
- "https://cockpit-ci-images.s3.us-east-1.amazonaws.com/",
+ *(f'https://{name}.s3.{region}.amazonaws.com/' for name, region in CI_IMAGES_BUCKETS.items()),
)
# locally configured stores in ~/.config/cockpit-dev/image-stores or $COCKPIT_IMAGE_STORES_FILE
@@ -35,4 +35,4 @@
LOCAL_STORES: Sequence[str] = data.splitlines()
-LOG_STORE = "https://cockpit-ci-logs.s3.us-east-1.amazonaws.com/"
+LOG_STORE = LOGS_URL
diff --git a/lib/stubs/pika/__init__.pyi b/lib/stubs/pika/__init__.pyi
new file mode 100644
index 0000000000..d56681b2b8
--- /dev/null
+++ b/lib/stubs/pika/__init__.pyi
@@ -0,0 +1,33 @@
+import ssl
+from typing import Any
+
+from .channel import Channel
+from .credentials import ExternalCredentials, PlainCredentials
+from .spec import Basic
+
+class SSLOptions:
+ def __init__(self, context: ssl.SSLContext, *, server_hostname: str = ...) -> None: ...
+
+class ConnectionParameters:
+ def __init__(
+ self,
+ host: str = ...,
+ port: int = ...,
+ *,
+ ssl_options: SSLOptions = ...,
+ credentials: PlainCredentials | ExternalCredentials = ...,
+ ) -> None: ...
+
+class BasicProperties:
+ def __init__(self, *, priority: int = ..., **kwargs: Any) -> None: ...
+
+class BlockingChannel(Channel):
+ def basic_get(
+ self, queue: str, auto_ack: bool = ...
+ ) -> tuple[Basic.GetOk, BasicProperties, bytes] | tuple[None, None, None]: ...
+
+class BlockingConnection:
+ def __init__(self, parameters: ConnectionParameters = ...) -> None: ...
+ def channel(self) -> BlockingChannel: ...
+ def close(self) -> None: ...
+ def process_data_events(self, time_limit: float = ...) -> None: ...
diff --git a/lib/stubs/pika/adapters/__init__.pyi b/lib/stubs/pika/adapters/__init__.pyi
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/lib/stubs/pika/adapters/asyncio_connection.pyi b/lib/stubs/pika/adapters/asyncio_connection.pyi
new file mode 100644
index 0000000000..aed0ad7037
--- /dev/null
+++ b/lib/stubs/pika/adapters/asyncio_connection.pyi
@@ -0,0 +1,18 @@
+from collections.abc import Callable
+
+from pika import ConnectionParameters
+from pika.channel import Channel
+
+class AsyncioConnection:
+ def __init__(
+ self,
+ parameters: ConnectionParameters | None = ...,
+ *,
+ on_open_callback: Callable[[AsyncioConnection], object] | None = ...,
+ on_open_error_callback: Callable[[AsyncioConnection, Exception], object] | None = ...,
+ on_close_callback: Callable[[AsyncioConnection, Exception], object] | None = ...,
+ ) -> None: ...
+
+ def channel(self, *, on_open_callback: Callable[[Channel], object]) -> None: ...
+
+ def close(self) -> None: ...
diff --git a/lib/stubs/pika/channel.pyi b/lib/stubs/pika/channel.pyi
new file mode 100644
index 0000000000..7b35cc1a94
--- /dev/null
+++ b/lib/stubs/pika/channel.pyi
@@ -0,0 +1,51 @@
+from collections.abc import Callable
+from typing import Any
+
+from pika import BasicProperties
+from pika.frame import Method
+from pika.spec import Basic
+
+class Channel:
+ def queue_declare(
+ self,
+ queue: str,
+ *,
+ durable: bool = ...,
+ passive: bool = ...,
+ arguments: dict[str, Any] | None = ...,
+ ) -> Method: ...
+
+ def basic_publish(
+ self,
+ exchange: str,
+ routing_key: str,
+ body: str | bytes,
+ *,
+ properties: BasicProperties = ...,
+ ) -> None: ...
+
+ def basic_qos(
+ self,
+ *,
+ prefetch_size: int = ...,
+ prefetch_count: int = ...,
+ global_qos: bool = ...,
+ callback: Callable[..., object] | None = ...,
+ ) -> None: ...
+
+ def basic_consume(
+ self,
+ queue: str,
+ *,
+ on_message_callback: Callable[[Channel, Basic.Deliver, BasicProperties | None, bytes], object] = ...,
+ auto_ack: bool = ...,
+ arguments: dict[str, Any] | None = ...,
+ ) -> str: ...
+
+ def basic_cancel(self, consumer_tag: str) -> None: ...
+
+ def basic_ack(self, delivery_tag: int) -> None: ...
+
+ def basic_reject(self, delivery_tag: int, *, requeue: bool = ...) -> None: ...
+
+ def add_on_close_callback(self, callback: Callable[[Channel, Exception], object]) -> None: ...
diff --git a/lib/stubs/pika/credentials.pyi b/lib/stubs/pika/credentials.pyi
new file mode 100644
index 0000000000..20449e73f3
--- /dev/null
+++ b/lib/stubs/pika/credentials.pyi
@@ -0,0 +1,5 @@
+class PlainCredentials:
+ def __init__(self, username: str, password: str) -> None: ...
+
+class ExternalCredentials:
+ def __init__(self) -> None: ...
diff --git a/lib/stubs/pika/exceptions.pyi b/lib/stubs/pika/exceptions.pyi
new file mode 100644
index 0000000000..63b41262ec
--- /dev/null
+++ b/lib/stubs/pika/exceptions.pyi
@@ -0,0 +1,6 @@
+class AMQPError(Exception): ...
+class AMQPConnectionError(AMQPError): ...
+class ChannelClosedByBroker(AMQPError):
+ reply_code: int
+ reply_text: str
+ def __init__(self, reply_code: int, reply_text: str) -> None: ...
diff --git a/lib/stubs/pika/frame.pyi b/lib/stubs/pika/frame.pyi
new file mode 100644
index 0000000000..624d2284fe
--- /dev/null
+++ b/lib/stubs/pika/frame.pyi
@@ -0,0 +1,4 @@
+from pika.spec import Queue
+
+class Method:
+ method: Queue.DeclareOk
diff --git a/lib/stubs/pika/spec.pyi b/lib/stubs/pika/spec.pyi
new file mode 100644
index 0000000000..4b457a97ce
--- /dev/null
+++ b/lib/stubs/pika/spec.pyi
@@ -0,0 +1,10 @@
+class Basic:
+ class Deliver:
+ delivery_tag: int
+
+ class GetOk:
+ delivery_tag: int
+
+class Queue:
+ class DeclareOk:
+ message_count: int | None
diff --git a/pyproject.toml b/pyproject.toml
index ac1d1bee8b..14775cf0b0 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -2,20 +2,32 @@
name = "bots"
requires-python = ">= 3.14"
+[tool.pyright]
+stubPath = "lib/stubs"
+
[tool.mypy]
python_version = "3.14"
strict = true
follow_imports = 'silent' # https://github.com/python-lsp/pylsp-mypy/issues/81
scripts_are_modules = true # allow checking all scripts in one invocation
+mypy_path = 'lib/stubs'
warn_return_any = false
[[tool.mypy.overrides]]
# things which may be unavailable when running checks
module = [
+ 'boto3',
+ 'botocore',
+ 'botocore.*',
'libvirt',
'libvirt_qemu',
'nacl',
- 'pika.*',
+ 'types_boto3_autoscaling.*',
+ 'types_boto3_ec2',
+ 'types_boto3_ec2.*',
+ 'types_boto3_s3.*',
+ 'types_boto3_ssm.*',
+ 'types_boto3_sts',
]
ignore_missing_imports = true
diff --git a/saml-login b/saml-login
index 4b9bc9832d..485ad2220e 100755
--- a/saml-login
+++ b/saml-login
@@ -11,53 +11,99 @@ for use by image-download.
"""
import argparse
+import json
import logging
+import shlex
import sys
+from datetime import timedelta
+from typing import Literal
import gi # type: ignore [import-untyped]
from lib.ansi import GREEN, RESET
+from lib.aws.account import (
+ ACCOUNT_ID,
+ REDHAT_SSO_IDP_URL,
+ REDHAT_SSO_IMAGE_DOWNLOAD_MAX_SESSION,
+ REDHAT_SSO_IMAGE_DOWNLOAD_ROLE,
+ REDHAT_SSO_SAML_PROVIDER_ARN,
+)
from lib.gssapi_saml_sts import (
- TARGETS,
AuthError,
aws_sts_assume_role,
cache_path,
+ load_from_cache,
save_to_cache,
)
+from lib.s3 import S3Key
logger = logging.getLogger(__name__)
+def print_credentials(key: S3Key, output: Literal['awscreds', 'env']) -> None:
+ match output:
+ case 'env':
+ print(f'export AWS_ACCESS_KEY_ID={shlex.quote(key.access)}')
+ print(f'export AWS_SECRET_ACCESS_KEY={shlex.quote(key.secret)}')
+ if key.token:
+ print(f'export AWS_SESSION_TOKEN={shlex.quote(key.token)}')
+ case 'awscreds':
+ print(json.dumps({
+ 'Version': 1,
+ 'AccessKeyId': key.access,
+ 'SecretAccessKey': key.secret,
+ 'SessionToken': key.token,
+ }))
+
+
def main() -> None:
parser = argparse.ArgumentParser(description='Log in to AWS via SAML using a web browser')
+ # fmt: off
parser.add_argument('-d', '--debug', action='store_true', help='Enable debug logging')
+ parser.add_argument('role', nargs='?', default=REDHAT_SSO_IMAGE_DOWNLOAD_ROLE,
+ help='IAM role name to assume (default: %(default)s)')
+ parser.add_argument('--duration', type=lambda s: timedelta(hours=float(s)),
+ default=REDHAT_SSO_IMAGE_DOWNLOAD_MAX_SESSION,
+ help='session duration in hours (default: %(default)s)')
+ parser.add_argument('--output', choices=['env', 'awscreds'],
+ help='output credentials in the given format')
+ # fmt: on
args = parser.parse_args()
if args.debug:
logging.basicConfig(level=logging.DEBUG, format='%(name)s: %(message)s')
+ cached = load_from_cache(args.role)
+ if cached is not None:
+ if args.output:
+ print_credentials(cached, args.output)
+ else:
+ print(f'{GREEN}Cached credentials for {args.role} still valid{RESET}')
+ return
+
gi.require_version('Gtk', '4.0')
gi.require_version('JavaScriptCore', '6.0')
gi.require_version('WebKit', '6.0')
from gi.repository import GLib, Gtk, JavaScriptCore, WebKit # type: ignore [import-untyped]
- target, = TARGETS.values() # only one for now
loop = GLib.MainLoop()
def on_saml_message_received(_manager: WebKit.UserContentManager, message: JavaScriptCore.Value) -> None:
saml_assertion = message.to_string()
logger.debug('captured SAML assertion (%d bytes)', len(saml_assertion))
- xml = aws_sts_assume_role(target.role_arn, target.provider_arn, saml_assertion)
+ xml = aws_sts_assume_role(ACCOUNT_ID, args.role, REDHAT_SSO_SAML_PROVIDER_ARN, saml_assertion, args.duration)
- role_name = target.role_arn.rsplit('/', 1)[-1]
try:
- _key, expiration = save_to_cache(role_name, xml)
+ key, expiration = save_to_cache(args.role, xml)
except AuthError as exc:
sys.exit(str(exc))
- local = expiration.astimezone()
- print(f'\n{GREEN}STS credentials saved to {cache_path(role_name)}\nValid until {local:%c}{RESET}\n')
+ if args.output:
+ print_credentials(key, args.output)
+ else:
+ local = expiration.astimezone()
+ print(f'\n{GREEN}STS credentials saved to {cache_path(args.role)}\nValid until {local:%c}{RESET}\n')
loop.quit()
content_manager = WebKit.UserContentManager()
@@ -88,7 +134,7 @@ def main() -> None:
)
webview = WebKit.WebView(user_content_manager=content_manager)
- webview.load_uri(target.idp_url)
+ webview.load_uri(REDHAT_SSO_IDP_URL)
win = Gtk.Window(
title='SAML Login',
diff --git a/test/generate_test_jobs.py b/test/generate_test_jobs.py
new file mode 100644
index 0000000000..fec8e64be6
--- /dev/null
+++ b/test/generate_test_jobs.py
@@ -0,0 +1,241 @@
+# Copyright (C) 2026 Red Hat, Inc.
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import argparse
+import json
+import logging
+import time
+from collections.abc import Sequence
+
+import pika
+
+from lib import distributed_queue, github, testmap
+from lib.aio.jsonutil import get_dict, get_str
+from lib.jobqueue import QueueEntry
+
+logger = logging.getLogger(__name__)
+
+
+def resolve_ref(api: github.GitHub, ref: str) -> tuple[str, str | None]:
+ """Resolve a ref to (sha, branch).
+
+ ref can be 'pr:NUMBER', a branch name, or a SHA.
+ For PRs, branch is the base branch. For branches, branch is the ref itself.
+ For bare SHAs, branch is None.
+ """
+ if ref.startswith('pr:'):
+ pr_nr = int(ref[3:])
+ pull = api.get_obj(f"pulls/{pr_nr}")
+ sha = get_str(get_dict(pull, "head"), "sha")
+ branch = get_str(get_dict(pull, "base"), "ref")
+ logger.info("pr %d → sha %s, base branch %r", pr_nr, sha, branch)
+ return sha, branch
+
+ result = api.get_obj(f"commits/{ref}", None)
+ if result is None:
+ raise SystemExit(f"ref {ref!r} not found on github.com/{api.repo}")
+
+ sha = get_str(result, "sha")
+ logger.debug("resolved %r to %r", ref, sha)
+
+ branches = testmap.tests_for_project(api.repo)
+ if ref in branches:
+ return sha, ref
+
+ return sha, None
+
+
+def filter_contexts(
+ contexts: Sequence[str],
+ os_filter: Sequence[str],
+ scenario_filter: Sequence[str],
+) -> Sequence[str]:
+ result: list[str] = []
+ for context in contexts:
+ # skip cross-repo contexts
+ if '@' in context:
+ continue
+
+ image, _, scenario = context.partition('/')
+
+ if os_filter and image not in os_filter:
+ continue
+ if scenario_filter and not any(s in scenario for s in scenario_filter):
+ continue
+
+ result.append(context)
+
+ return result
+
+
+def generate_entries(
+ repo: str,
+ sha: str,
+ branch: str | None,
+ contexts: Sequence[str],
+ bots_ref: str | None,
+ extra_env: Sequence[str],
+ pull: int | None,
+) -> Sequence[QueueEntry]:
+ entries: list[QueueEntry] = []
+ timestamp = time.strftime('%Y%m%d-%H%M%S')
+
+ for context in contexts:
+ image, _, scenario = context.partition('/')
+
+ slug_suffix = context.replace('/', '-').replace('@', '-')
+ slug = f"ec2-full-{timestamp}-{repo.replace('/', '-')}-{sha[:8]}-{slug_suffix}"
+
+ env: dict[str, str] = {
+ "TEST_OS": image,
+ "TEST_REVISION": sha,
+ }
+ if scenario:
+ env["TEST_SCENARIO"] = scenario
+ if branch:
+ env["BASE_BRANCH"] = branch
+ if bots_ref:
+ env["COCKPIT_BOTS_REF"] = bots_ref
+ if pull is not None:
+ env["TEST_PULL"] = str(pull)
+
+ for item in extra_env:
+ key, _, value = item.partition('=')
+ env[key] = value
+
+ secrets: list[str] = ["github-token", "image-download"]
+ if repo == "rhinstaller/anaconda-webui":
+ secrets.extend(["fedora-wiki", "fedora-wiki-staging"])
+
+ entries.append({
+ "job": {
+ "repo": repo,
+ "sha": sha,
+ "context": context,
+ "pull": pull,
+ "command_subject": None,
+ "slug": slug,
+ "env": env,
+ "secrets": secrets,
+ },
+ "human": f"{context}@{repo}#{sha[:12]}",
+ })
+
+ return entries
+
+
+def publish(entries: Sequence[QueueEntry]) -> None:
+ queue = 'public'
+ with distributed_queue.DistributedQueue('localhost', [queue]) as dq:
+ for entry in entries:
+ context = entry['job']['context']
+ priority = distributed_queue.MAX_PRIORITY if '/devel' in context else distributed_queue.BASELINE_PRIORITY
+ properties = pika.BasicProperties(priority=priority)
+ dq.channel.basic_publish('', queue, json.dumps(entry), properties=properties)
+ logger.info("published %s", entry['human'])
+ logger.info("published %d entries", len(entries))
+
+
+def collect_entries(
+ repo: str,
+ branches: dict[str, Sequence[str]],
+ ref: str | None,
+ os_filter: Sequence[str],
+ scenario_filter: Sequence[str],
+ bots_ref: str | None,
+ extra_env: Sequence[str],
+) -> Sequence[QueueEntry]:
+ api = github.GitHub(repo=repo)
+ entries: list[QueueEntry] = []
+
+ pull: int | None = None
+ if ref is not None:
+ sha, resolved_branch = resolve_ref(api, ref)
+ if ref.startswith('pr:'):
+ pull = int(ref[3:])
+
+ if resolved_branch and resolved_branch in branches:
+ branch_list = {resolved_branch: branches[resolved_branch]}
+ elif resolved_branch is None:
+ # bare SHA: run against default branch contexts
+ default = testmap.get_default_branch(repo)
+ if default in branches:
+ branch_list = {default: branches[default]}
+ else:
+ branch_list = {}
+ else:
+ logger.warning("branch %r not in testmap for %r", resolved_branch, repo)
+ branch_list = {}
+
+ for branch, contexts in branch_list.items():
+ filtered = filter_contexts(contexts, os_filter, scenario_filter)
+ entries.extend(generate_entries(repo, sha, branch, filtered, bots_ref, extra_env, pull))
+ else:
+ for branch, contexts in branches.items():
+ if branch.startswith('_'):
+ continue
+ sha, _ = resolve_ref(api, branch)
+ filtered = filter_contexts(contexts, os_filter, scenario_filter)
+ entries.extend(generate_entries(repo, sha, branch, filtered, bots_ref, extra_env, None))
+
+ return entries
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(
+ description='Generate test jobs from the testmap',
+ epilog='With no arguments, generates jobs for all repos and branches in the testmap.',
+ )
+ parser.add_argument('repo', nargs='?',
+ help='repository (e.g. cockpit-project/cockpit)')
+ parser.add_argument('ref', nargs='?',
+ help='branch, SHA, or pr:NUMBER')
+ parser.add_argument('--os', action='append', default=[], dest='os_filter',
+ help='filter by OS image (repeatable)')
+ parser.add_argument('--scenario', action='append', default=[], dest='scenario_filter',
+ help='filter by scenario (repeatable)')
+ parser.add_argument('--bots-ref',
+ help='bots ref to set as COCKPIT_BOTS_REF')
+ parser.add_argument('--env', action='append', default=[], dest='extra_env',
+ help='extra env var as KEY=VALUE (repeatable)')
+ parser.add_argument('--amqp', action='store_true',
+ help='publish to AMQP on localhost instead of printing JSONL')
+ args = parser.parse_args()
+
+ logging.basicConfig(level=logging.INFO)
+
+ all_entries: list[QueueEntry] = []
+
+ if args.repo:
+ branches = dict(testmap.tests_for_project(args.repo))
+ if not branches:
+ raise SystemExit(f"repo {args.repo!r} not found in testmap")
+ all_entries.extend(collect_entries(
+ args.repo, branches, args.ref, args.os_filter, args.scenario_filter, args.bots_ref, args.extra_env,
+ ))
+ else:
+ for repo, branch_map in testmap.REPO_BRANCH_CONTEXT.items():
+ if ':' in repo:
+ logger.debug("skipping non-github repo %r", repo)
+ continue
+ branches = dict(branch_map)
+ all_entries.extend(collect_entries(
+ repo, branches, args.ref, args.os_filter, args.scenario_filter, args.bots_ref, args.extra_env,
+ ))
+
+ if not all_entries:
+ raise SystemExit("no matching test contexts found")
+
+ logger.info("%d entries", len(all_entries))
+
+ if args.amqp:
+ publish(all_entries)
+ for entry in all_entries:
+ print(entry['job']['slug'])
+ else:
+ for entry in all_entries:
+ print(json.dumps(entry))
+
+
+if __name__ == '__main__':
+ main()
diff --git a/test/run b/test/run
index 97fc396462..677cf1ec92 100755
--- a/test/run
+++ b/test/run
@@ -20,3 +20,11 @@ find_python_files() {
find_python_files | xargs -0 ruff check --quiet
find_python_files | xargs -0 mypy --no-error-summary
pytest -vv
+
+if command -v biome >/dev/null 2>&1 && [ -f biome.jsonc ]; then
+ biome check
+fi
+
+if command -v tsc >/dev/null 2>&1; then
+ tsc -p lib/html/tsconfig.json
+fi
diff --git a/test/test_aio.py b/test/test_aio.py
index 9a45028077..dd6bd78598 100644
--- a/test/test_aio.py
+++ b/test/test_aio.py
@@ -14,7 +14,7 @@
from lib.aio.base import SubjectSpecification
from lib.aio.github import GitHub
from lib.aio.jobcontext import JobContext
-from lib.aio.jsonutil import JsonObject, JsonValue, json_merge_patch
+from lib.aio.jsonutil import JsonError, JsonObject, JsonValue, json_merge_patch
from lib.aio.s3 import S3LogDriver
from lib.aio.util import LRUCache
from lib.s3 import S3Key
@@ -327,10 +327,10 @@ async def test_secrets_expansion(tmp_path: Path) -> None:
home = Path.home()
async with JobContext(config_file) as context:
- assert context.secrets_args == {
- 's3-keys': (f'--volume={home}/.config/s3-keys:/run/secrets/s3-keys:ro',),
- 'github-token': ('--env=GITHUB_TOKEN_FILE=/etc/github-token',),
- }
+ assert context.prepare_secrets(['s3-keys', 'github-token'], tmp_path / 'secrets') == [
+ f'--volume={home}/.config/s3-keys:/run/secrets/s3-keys:ro',
+ '--env=GITHUB_TOKEN_FILE=/etc/github-token',
+ ]
async def test_secrets_undefined_error(tmp_path: Path) -> None:
@@ -349,9 +349,9 @@ async def test_secrets_undefined_error(tmp_path: Path) -> None:
local.directory = '/tmp/logs'
''')
- with pytest.raises(SystemExit, match=r"undefined secret '%\{undefined\}'"):
- async with JobContext(config_file):
- pass
+ async with JobContext(config_file) as ctx:
+ with pytest.raises(LookupError, match='undefined'):
+ ctx.prepare_secrets(['bad'], tmp_path / 'secrets')
async def test_inline_secrets(tmp_path: Path) -> None:
@@ -375,22 +375,20 @@ async def test_inline_secrets(tmp_path: Path) -> None:
local.directory = '/tmp/logs'
''')
+ secrets_dir = tmp_path / 'secrets'
async with JobContext(config_file) as ctx:
+ result = ctx.prepare_secrets(['github-token', 's3-keys'], secrets_dir)
+
# Check github-token
- assert ctx.secrets_args['github-token'][0] == '-e=COCKPIT_GITHUB_TOKEN_FILE=/x'
- token_path = Path(ctx.secrets_args['github-token'][1].removeprefix('-v=').removesuffix(':/x'))
+ assert result[0] == '-e=COCKPIT_GITHUB_TOKEN_FILE=/x'
+ token_path = Path(result[1].removeprefix('-v=').removesuffix(':/x'))
assert token_path.read_text() == 'ghp_secret123'
# Check s3-keys
- assert ctx.secrets_args['s3-keys'][0] == '-e=COCKPIT_S3_KEY_DIR=/y'
- s3_path = Path(ctx.secrets_args['s3-keys'][1].removeprefix('-v=').removesuffix(':/y'))
+ assert result[2] == '-e=COCKPIT_S3_KEY_DIR=/y'
+ s3_path = Path(result[3].removeprefix('-v=').removesuffix(':/y'))
assert (s3_path / 's3.example.com').read_text() == 'ABCD Zx2xPa'
- # After context closes, temp files should be cleaned up
- assert not token_path.exists()
- assert not s3_path.exists()
- assert not token_path.parent.exists()
-
async def test_inline_path_nested_error(tmp_path: Path) -> None:
config_file = tmp_path / 'config.toml'
@@ -404,19 +402,19 @@ async def test_inline_path_nested_error(tmp_path: Path) -> None:
default-image = 'ghcr.io/test:latest'
[container.secrets]
+ mydir = ['--volume=%{mydir}:/mnt']
[logs]
driver = 'local'
local.directory = '/tmp/logs'
''')
- with pytest.raises(
- SystemExit,
- match=r"attribute 'secrets': attribute 'inline': attribute 'mydir': "
- r"attribute 'subdir': invalid filename: '\.\./escape'",
- ):
- async with JobContext(config_file):
- pass
+ async with JobContext(config_file) as ctx:
+ with pytest.raises(
+ JsonError,
+ match=r"attribute 'mydir': attribute 'subdir': invalid filename: '\.\./escape'",
+ ):
+ ctx.prepare_secrets(['mydir'], tmp_path / 'secrets')
async def test_inline_path_invalid_type(tmp_path: Path) -> None:
@@ -431,17 +429,18 @@ async def test_inline_path_invalid_type(tmp_path: Path) -> None:
default-image = 'ghcr.io/test:latest'
[container.secrets]
+ badvalue = ['--env=X=%{badvalue}']
[logs]
driver = 'local'
local.directory = '/tmp/logs'
''')
- with pytest.raises(
- SystemExit, match=r"attribute 'secrets': attribute 'inline': attribute 'badvalue': must be string or object"
- ):
- async with JobContext(config_file):
- pass
+ async with JobContext(config_file) as ctx:
+ with pytest.raises(
+ JsonError, match=r"attribute 'badvalue': must be string or object"
+ ):
+ ctx.prepare_secrets(['badvalue'], tmp_path / 'secrets')
async def test_secrets_conflict_error(tmp_path: Path) -> None:
@@ -514,5 +513,5 @@ async def test_serialize_roundtrip(tmp_path: Path, monkeypatch: pytest.MonkeyPat
# Load from serialized config (simulating remote execution)
monkeypatch.setenv('JOB_RUNNER_CONFIG_JSON', json.dumps(serialized))
async with JobContext() as ctx2:
- # Secrets should work the same way
- assert ctx2.secrets_args['github-token'][0].endswith('/github-token')
+ result = ctx2.prepare_secrets(['github-token'], tmp_path / 'secrets2')
+ assert result[0].endswith('/github-token')
diff --git a/test/test_job.py b/test/test_job.py
index 5d31f7cf8f..a3c3765600 100644
--- a/test/test_job.py
+++ b/test/test_job.py
@@ -130,7 +130,7 @@ async def mock_job_context(tmp_path: Path) -> AsyncGenerator[Mock, None]:
mock_ctx.default_image = 'registry.fedoraproject.org/fedora:latest'
mock_ctx.container_cmd = ['podman']
mock_ctx.container_run_args = ['--pull=newer']
- mock_ctx.secrets_args = {}
+ mock_ctx.prepare_secrets = Mock(return_value=[])
mock_ctx.resolve_subject = AsyncMock(wraps=forge.resolve_subject)
mock_ctx.server = server
@@ -272,6 +272,33 @@ async def test_run_job_failure(
assert get_str(failure_data, 'description').startswith('Container exited with code 1')
assert failure_data['target_url'] == 'http://localhost:9000/test-job/log.html'
+ @pytest.mark.parametrize('side_effect', [
+ LookupError('no container.secrets.github-token entry'),
+ LookupError('secret %{github-token} is not configured'),
+ ])
+ async def test_run_job_missing_secret(
+ self,
+ side_effect: LookupError,
+ log_streamer_mocks: LogStreamerMocks,
+ mock_job_context: Mock,
+ ) -> None:
+ job = Job({
+ 'repo': 'cockpit-project/cockpit',
+ 'sha': 'abc123',
+ 'context': 'verify/rhel-9',
+ 'secrets': ['github-token'],
+ })
+ mock_job_context.prepare_secrets = Mock(side_effect=side_effect)
+
+ await run_job(job, mock_job_context)
+
+ post_calls = mock_job_context.server.drain_queue()
+ assert len(post_calls) == 2
+ failure_path, failure_data = post_calls[1]
+ assert failure_path == '/repos/cockpit-project/cockpit/statuses/abc123'
+ assert failure_data['state'] == 'failure'
+ assert str(side_effect) in get_str(failure_data, 'description')
+
@patch('lib.aio.job.run_container')
async def test_run_job_failure_with_report(
self,