diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index f69e94e..0f5bb97 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -20,7 +20,17 @@ jobs: run: dnf -y update fedora-gpg-keys - name: Install git and Python libraries - run: dnf -y install git-core python3-yaml python3-jinja2 python3-koji python3-pytest python3-flake8 python3-dnf + run: | + dnf -y install \ + git-core \ + python3-flake8 \ + python3-jinja2 \ + python3-koji \ + python3-libdnf5 \ + python3-pip \ + python3-pytest \ + python3-pytest-cov \ + python3-yaml - name: Clean up run: dnf clean all diff --git a/.github/workflows/fork-only-workflow.yml b/.github/workflows/fork-only-workflow.yml index 73e61eb..f136392 100644 --- a/.github/workflows/fork-only-workflow.yml +++ b/.github/workflows/fork-only-workflow.yml @@ -33,12 +33,13 @@ jobs: run: | dnf -y install \ git \ + python3-flake8 \ python3-jinja2 \ python3-koji \ - python3-yaml \ - python3-dnf \ + python3-libdnf5 \ python3-pytest \ - python3-flake8 + python3-pytest-cov \ + python3-yaml - name: Clean up run: dnf clean all diff --git a/Dockerfile b/Dockerfile index e5c9660..a964bb9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,16 @@ FROM registry.fedoraproject.org/fedora:44 -RUN dnf -y update fedora-gpg-keys && \ - dnf -y install git python3-jinja2 python3-koji python3-yaml python3-dnf && \ +RUN dnf -y update fedora-gpg-keys \ + --setopt=tsflags=nodocs \ + --setopt=install_weak_deps=False && \ + dnf -y install \ + git \ + python3-jinja2 \ + python3-koji \ + python3-libdnf5 \ + python3-yaml \ + --setopt=tsflags=nodocs \ + --setopt=install_weak_deps=False && \ dnf clean all && \ rm -rf /var/cache/dnf diff --git a/README.md b/README.md index 3a936c8..cb224fc 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # Content Resolver -Content Resolver makes it easy to define and inspect package sets of RPM-based Linux distribution. +[![Content Resolver CI](https://github.com/fedora-eln/content-resolver/actions/workflows/docker-image.yml/badge.svg)](https://github.com/fedora-eln/content-resolver/actions/workflows/docker-image.yml) + +Content Resolver makes it easy to define and inspect package sets of RPM-based Linux distributions. You define what packages you need, and Content Resolver gives you the whole picture including all the dependencies. And it keeps it up-to-date as packages get updated over time. @@ -12,6 +14,28 @@ Content Resolver also helps with minimisation efforts by showing detailed depend ## Using Content Resolver +### Code Structure + +Core functionality + +``` +content-resolver/ +├── content_resolver/ +│ ├── analyzer.py # Core resolution engine (DNF5) +│ ├── config_manager.py # YAML config loading +│ ├── data_generation.py # Generate non HTML output +│ ├── exceptions.py # Custom exceptions +│ ├── history_data.py # Handle history for graph generation +│ ├── page_generation.py # Generate HTML output +│ ├── query.py # Internal data query mechanism +│ └── utils.py # Helper functions +├── templates/ # Jinja2 HTML templates +├── test_configs/ # Test configurations +├── content_resolver.py # Main entry point +└── refresh.sh # Production deployment script +``` + + ### Controlling Content Resolver Content Resolver is entirely controlled by a set of YAML files stored in a git repository. Changes to the content can easily be modified and reviewed via pull requests. @@ -144,3 +168,61 @@ $ docker run --rm -it --tmpfs /dnf_cachedir -v $(pwd):/workspace content-resolve ``` The output will be generated in the `output` directory. Open the `output/index.html` in your web browser of choice to see the result. + +### Resolution Process + +**Phase 1: Repository Analysis** +``` +For each repository and architecture: + 1. Create DNF5 Base with repository configuration + 2. Apply repository priorities (1=highest) + 3. Apply package exclude lists (e.g., rust-*-devel, golang-*-devel) + 4. Load repository metadata + 5. Query all available packages + 6. Deduplicate by NEVRA (Name-Epoch:Version-Release.Arch) + 7. Track which repo provides each package +``` + +**Phase 2: Environment Resolution** +``` +For each environment: + 1. Create DNF5 Goal with environment packages + 2. Resolve dependencies using DNF5 transaction + 3. Check transaction.get_problems() for conflicts + 4. Extract installed packages (base environment) + 5. Build package relationship graph +``` + +**Phase 3: Workload Resolution** +``` +For each workload on top of each environment: + 1. Load base environment packages + 2. Add workload-specific packages to Goal + 3. Resolve additional dependencies + 4. Separate environment packages from added packages + 5. Track which workloads require which packages +``` + +**Phase 4: Buildroot Resolution** (for views with `buildroot_strategy: root_logs`) +``` +For each source package in the view: + 1. Download Koji root.log via Koji API + 2. Parse build dependencies from root log + 3. Create fake workload with build dependencies + 4. Resolve buildroot on top of build group + 5. Track buildroot packages separately from runtime + 6. Iterate until all transitive build deps resolved +``` + +### Repository Priority System + +Repositories are assigned priorities (1=highest, 5=lowest) to control package selection when multiple versions exist: + +```yaml +BaseOS: priority: 1 # Prefer packages from BaseOS +AppStream: priority: 1 # Prefer packages from AppStream +CRB: priority: 1 # CodeReady Builder +Extras: priority: 3 # Additional packages +buildroot: priority: 4 # Build-time only packages +Rawhide: priority: 5 # Fallback for missing packages +``` diff --git a/content_resolver/analyzer.py b/content_resolver/analyzer.py index 2099275..38bebb5 100644 --- a/content_resolver/analyzer.py +++ b/content_resolver/analyzer.py @@ -1,20 +1,79 @@ -import tempfile, os, json, datetime, dnf, urllib.request, sys, koji -import re, time +import asyncio +import datetime +import json +import multiprocessing +import os +import sys +import tempfile +import time +import traceback +import urllib.request from concurrent.futures import ProcessPoolExecutor, as_completed -import multiprocessing, asyncio -from content_resolver.utils import dump_data, load_data, log, err_log, pkg_id_to_name, size, workload_id_to_conf_id, url_to_id -from content_resolver.exceptions import RepoDownloadError, BuildGroupAnalysisError, KojiRootLogError, AnalysisError +import koji +from libdnf5.base import Base, Goal, GoalJobSettings +from libdnf5.exception import BaseTransactionError +from libdnf5.exception import Error as DnfErr +from libdnf5.exception import RepoDownloadError as Dnf5RepoDownloadError +from libdnf5.exception import UserAssertionError +from libdnf5.repo import RepoQuery +from libdnf5.rpm import PackageQuery + +from content_resolver.exceptions import ( + AnalysisError, + BuildGroupAnalysisError, + KojiRootLogError, + RepoDownloadError, +) +from content_resolver.utils import ( + dnf5_base, + dump_data, + err_log, + load_data, + log, + pkg_id_to_name, + size, + url_to_id, + workload_id_to_conf_id, +) def pkg_placeholder_name_to_id(placeholder_name): - placeholder_id = f"{placeholder_name}-000-placeholder.placeholder" - return placeholder_id + return f"{placeholder_name}-000-placeholder.placeholder" def pkg_placeholder_name_to_nevr(placeholder_name): - placeholder_id = f"{placeholder_name}-000-placeholder" - return placeholder_id + return f"{placeholder_name}-000-placeholder" + + +def is_package_resolvable(base: Base, pkg_name: str) -> bool: + """ + Check if a package name is resolvable, either as a real package or via provides. + + Args: + base: DNF5 base object + pkg_name: Package name to check + + Returns: + bool: True if the package can be resolved, False otherwise + """ + query = PackageQuery(base) + + # First check if it exists as a real package name + query.filter_name([pkg_name]) + if not query.empty(): + return True + + # If not found by name, check if any package provides it. + # Note: Must create a new query because DNF5 filters are cumulative. + # Reusing the filtered query would search for provides within the + # already-filtered (possibly empty) result set, not all packages. + query = PackageQuery(base) + query.filter_provides([pkg_name]) + if not query.empty(): + return True + + return False ##################################################### @@ -32,27 +91,26 @@ def _get_build_deps_from_a_root_log(root_log): state = 0 for file_line in root_log.splitlines(): + split_line = file_line.split() + line_len = len(split_line) # 0/ # parts of the log I don't really care about if state == 0: - # The next installation is the build deps! # So I start caring. Next state! if "'builddep', '--installroot'" in file_line: state += 1 - # 1/ # getting the "already installed" packages to the list elif state == 1: - # "Package already installed" indicates it's directly required, # so save it. # DNF5 does this after "Repositories loaded" and quotes the NVR; # DNF4 does this before "Dependencies resolved" without the quotes. if "is already installed." in file_line: - pkg_name = file_line.split()[3].strip('"').rsplit("-",2)[0] + pkg_name = split_line[3].strip('"').rsplit("-", 2)[0] required_pkgs.append(pkg_name) # That's all! Next state! (DNF4) @@ -63,33 +121,29 @@ def _get_build_deps_from_a_root_log(root_log): elif "Repositories loaded." in file_line: state += 1 - # 2/ # going through the log right before the first package name elif state == 2: - # "Package already installed" indicates it's directly required, # so save it. # DNF4 does this before "Dependencies resolved" without the quotes; # DNF5 does this after "Repositories loaded" and quotes the NVR, but # sometimes prints this in the middle of a dependency line. if "is already installed." in file_line: - pkg_index = file_line.split().index("already") - 2 - pkg_name = file_line.split()[pkg_index].strip('"').rsplit("-",2)[0] + pkg_index = split_line.index("already") - 2 + pkg_name = split_line[pkg_index].strip('"').rsplit("-", 2)[0] required_pkgs.append(pkg_name) # The next line will be the first package. Next state! # DNF5 reports "Installing: ## packages" in the Transaction Summary, # which we need to ignore - if "Installing:" in file_line and len(file_line.split()) == 3: + if "Installing:" in file_line and line_len == 3: state += 1 - # 3/ # And now just saving the packages until the "installing dependencies" part # or the "transaction summary" part if there's no dependencies elif state == 3: - if "Installing dependencies:" in file_line: state = 2 @@ -97,12 +151,12 @@ def _get_build_deps_from_a_root_log(root_log): state = 2 # Sometimes DNF5 prints "Package ... is already installed" in middle of the output. - elif file_line.split()[2] == "Package" and file_line.split()[-1] == "installed.": - pkg_name = file_line.split()[3].strip('"').rsplit("-",2)[0] + elif split_line[2] == "Package" and split_line[-1] == "installed.": + pkg_name = split_line[3].strip('"').rsplit("-", 2)[0] required_pkgs.append(pkg_name) else: - # I need to deal with the following thing... + # We need to deal with the following... # # DEBUG util.py:446: gobject-introspection-devel aarch64 1.70.0-1.fc36 build 1.1 M # DEBUG util.py:446: graphene-devel aarch64 1.10.6-3.fc35 build 159 k @@ -114,12 +168,12 @@ def _get_build_deps_from_a_root_log(root_log): # The "gstreamer1-plugins-bad-free-devel" package name is too long to fit in the column, # so it gets split on two lines. # - # Which if I take the usual file_line.split()[2] I get the correct name, - # but the next line gives me "aarch64" as a package name which is wrong. + # When using the usual split_line[2] we get the correct name, + # but the next line gives us "aarch64" as a package name which is wrong. # - # So the usual line has file_line.split() == 8 - # The one with the long package name has file_line.split() == 3 - # and the one following it has file_line.split() == 7 + # So the usual line has line_len == 8 + # The one with the long package name has line_len == 3 + # and the one following it has line_len == 7 # # One more thing... long release! # @@ -128,51 +182,46 @@ def _get_build_deps_from_a_root_log(root_log): # DEBUG util.py:446: build 71 k # DEBUG util.py:446: tpm2-tss-devel aarch64 3.1.0-4.fc36 build 315 k # - # So the good one here is file_line.split() == 5. - # And the following is also file_line.split() == 5. Fun! + # So the good one here is line_len == 5. + # And the following is also line_len == 5. Fun! # # So if it ends with B, k, M, G it's the wrong line, so skip, otherwise take the package name. # - # I can also anticipate both get long... that would mean I need to skip file_line.split() == 4. + # we can also anticipate both get long... that would mean we need to skip line_len == 4. - if len(file_line.split()) == 10 or len(file_line.split()) == 11: + if line_len in (10, 11): # Sometimes DNF5 prints "Package ... is already installed" in the middle of a line - pkg_index = file_line.split().index("already") - 2 - pkg_name = file_line.split()[pkg_index].strip('"').rsplit("-",2)[0] + pkg_index = split_line.index("already") - 2 + pkg_name = split_line[pkg_index].strip('"').rsplit("-", 2)[0] required_pkgs.append(pkg_name) - if pkg_index == 3: - pkg_name = file_line.split()[7] - else: - pkg_name = file_line.split()[2] + pkg_name = split_line[7] if pkg_index == 3 else split_line[2] required_pkgs.append(pkg_name) - # TODO: len(file_line.split()) == 9 ?? + # TODO: line_len == 9 ?? - elif len(file_line.split()) == 8 or len(file_line.split()) == 3: - pkg_name = file_line.split()[2] + elif line_len in (3, 8): + pkg_name = split_line[2] required_pkgs.append(pkg_name) - elif len(file_line.split()) == 7 or len(file_line.split()) == 4: + elif line_len in (4, 7): continue - elif len(file_line.split()) == 6 or len(file_line.split()) == 5: + elif line_len in (5, 6): # DNF5 uses B/KiB/MiB/GiB, DNF4 uses B/k/M/G - if file_line.split()[4] in ["B", "KiB", "k", "MiB", "M", "GiB", "G"]: + if split_line[4] in ["B", "KiB", "k", "MiB", "M", "GiB", "G"]: continue else: - pkg_name = file_line.split()[2] + pkg_name = split_line[2] required_pkgs.append(pkg_name) else: raise KojiRootLogError - # 4/ # I'm done. So I can break out of the loop. elif state == 4: break - return required_pkgs @@ -180,20 +229,21 @@ def _get_koji_log_path(srpm_id, arch, koji_session): """ Get koji log path for a given SRPM. """ - MAX_TRIES = 10 + max_tries = 10 attempts = 0 - while attempts < MAX_TRIES: + while attempts < max_tries: try: koji_pkg_data = koji_session.getRPM(f"{srpm_id}.src") koji_logs = koji_session.getBuildLogs(koji_pkg_data["build_id"]) break - except Exception: + except koji.GenericError: attempts += 1 - if attempts == MAX_TRIES: + if attempts == max_tries: raise KojiRootLogError("Could not talk to Koji API") time.sleep(1) + # TODO: use `next` and comprehension for lazy evaluation next(x for x in koji_logs) koji_log_path = None for koji_log in koji_logs: if koji_log["name"] == "root.log": @@ -208,24 +258,25 @@ def _download_root_log_with_retry(root_log_url): """ Download root.log file with retry logic. """ - MAX_TRIES = 10 + max_tries = 10 attempts = 0 request = urllib.request.Request(root_log_url) request.add_header("Accept", "text/plain") request.add_header("User-Agent", "ContentResolver/1.0") - while attempts < MAX_TRIES: + while attempts < max_tries: try: with urllib.request.urlopen(request, timeout=20) as response: root_log_data = response.read() - return root_log_data.decode('utf-8') + return root_log_data.decode("utf-8") except Exception: attempts += 1 - if attempts == MAX_TRIES: + if attempts == max_tries: raise KojiRootLogError(f"Could not download root.log from {root_log_url}") time.sleep(1) + def process_single_srpm_root_log(work_item): """ Process a single SRPM's root.log file. @@ -237,11 +288,11 @@ def process_single_srpm_root_log(work_item): dict: Contains srpm_id, arch, deps (list), error (str or None) """ try: - koji_api_url = work_item['koji_api_url'] - koji_files_url = work_item['koji_files_url'] - srpm_id = work_item['srpm_id'] - arch = work_item['arch'] - dev_buildroot = work_item.get('dev_buildroot', False) + koji_api_url = work_item["koji_api_url"] + koji_files_url = work_item["koji_files_url"] + srpm_id = work_item["srpm_id"] + arch = work_item["arch"] + dev_buildroot = work_item.get("dev_buildroot", False) # Handle development buildroot mode if dev_buildroot: @@ -293,16 +344,42 @@ def process_single_srpm_root_log(work_item): # Download root.log root_log_url = f"{koji_files_url}/{koji_log_path}" - root_log_contents = _download_root_log_with_retry(root_log_url) + try: + root_log_contents = _download_root_log_with_retry(root_log_url) + except KojiRootLogError as e: + # Download failed + return { + 'srpm_id': srpm_id, + 'arch': arch, + 'deps': [], + 'error': f"Download failed: {str(e)}" + } # Parse dependencies - deps = _get_build_deps_from_a_root_log(root_log_contents) + try: + deps = _get_build_deps_from_a_root_log(root_log_contents) + except Exception as e: + # Parsing failed + return { + 'srpm_id': srpm_id, + 'arch': arch, + 'deps': [], + 'error': f"Parse failed: {str(e)}" + } + + # Check if parsing yielded suspiciously few dependencies + warning = None + if len(deps) == 0: + warning = "WARNING: Zero dependencies found (possible truncated/corrupted log)" + elif len(deps) < 3: + warning = f"WARNING: Only {len(deps)} dependencies found (expected more)" return { 'srpm_id': srpm_id, 'arch': arch, 'deps': deps, - 'error': None + 'error': None, + 'warning': warning } except Exception as e: @@ -313,7 +390,7 @@ def process_single_srpm_root_log(work_item): 'error': str(e) } -class Analyzer(): +class Analyzer: ############################################################################### ### Analyzing stuff! ########################################################## @@ -343,7 +420,7 @@ class Analyzer(): # - "dnf_generic_installroot-{repo}-{arch}" <-- installroots for _analyze_pkgs # - "dnf_env_installroot-{env_conf}-{repo}-{arch}" <-- installroots for envs and workloads and buildroots # - # + # def __init__(self, configs, settings): self.workload_queue = {} @@ -353,8 +430,6 @@ def __init__(self, configs, settings): self.configs = configs self.settings = settings - - self.global_dnf_repo_cache = {} self.data = {} self.cache = {} @@ -362,6 +437,10 @@ def __init__(self, configs, settings): self.cache["root_log_deps"]["current"] = {} self.cache["root_log_deps"]["next"] = {} + # Cache for compose metadata (composeinfo.json) + # Maps repo_id -> {arch -> [list of available variant names]} + self.compose_metadata_cache = {} + self.metrics_data = [] # When analysing buildroot, we don't need metadata about @@ -371,10 +450,11 @@ def __init__(self, configs, settings): try: self.cache["root_log_deps"]["current"] = load_data(self.settings["root_log_deps_cache_path"]) + # Validate cached root.log data for issues + self._validate_root_log_cache() except FileNotFoundError: pass - def _record_metric(self, name): this_record = { "name": name, @@ -382,10 +462,9 @@ def _record_metric(self, name): } self.metrics_data.append(this_record) - def print_metrics(self): log("Additional metrics:") - + # TODO: use `enumerate(self.metrics_data)` instead of incrementing counter counter = 0 for this_record in self.metrics_data: @@ -397,248 +476,399 @@ def print_metrics(self): time_diff = this_record["timestamp"] - prev_timestamp - print(" {} (+{} mins): {}".format( - this_record["timestamp"].strftime("%H:%M:%S"), - str(int(time_diff.seconds/60)).zfill(3), - this_record["name"] - )) + print( + f" {this_record['timestamp'].strftime('%H:%M:%S')} " + f"(+{str(int(time_diff.seconds / 60)).zfill(3)} mins): " + f"{this_record['name']}" + ) counter += 1 - + def _validate_root_log_cache(self): + """ + Validate cached root.log dependency data for invalid entries. + Reports packages with zero or very low dependency counts. + """ + cache = self.cache["root_log_deps"]["current"] + if not cache: + return + + zero_deps_cached = [] + low_deps_cached = [] + + for koji_id, arches in cache.items(): + for arch, srpms in arches.items(): + for srpm_id, deps in srpms.items(): + if not isinstance(deps, list): + continue + + if len(deps) == 0: + zero_deps_cached.append({ + 'srpm_id': srpm_id, + 'arch': arch, + 'koji_id': koji_id + }) + elif len(deps) < 3: + low_deps_cached.append({ + 'srpm_id': srpm_id, + 'arch': arch, + 'deps': deps, + 'koji_id': koji_id + }) + + # Some verbose logging for debugging + # TODO: Enable some form of log level to increase/decrease verbosity of logs + if zero_deps_cached or low_deps_cached: + log("") + log("=" * 80) + log("⚠️ ROOT LOG CACHE VALIDATION") + log("=" * 80) + + if zero_deps_cached: + log(f"Found {len(zero_deps_cached)} cached packages with ZERO dependencies") + log("These packages may have had root.log download/parse failures in previous runs:") + log("") + for item in zero_deps_cached[:15]: + log(f" ⚠️ {item['srpm_id']} ({item['arch']})") + if len(zero_deps_cached) > 15: + log(f" ... and {len(zero_deps_cached) - 15} more") + log("") + log("Consider clearing cache for these packages to retry root.log download") + + if low_deps_cached: + log("") + log(f"Found {len(low_deps_cached)} cached packages with very few dependencies (<3)") + for item in low_deps_cached[:10]: + log(f" ⚠️ {item['srpm_id']} ({item['arch']}): {item['deps']}") + if len(low_deps_cached) > 10: + log(f" ... and {len(low_deps_cached) - 10} more") + + if zero_deps_cached or low_deps_cached: + log("=" * 80) + log("") + + def _get_available_compose_variants(self, repo, arch): + """ + Fetch and parse composeinfo.json to determine which variants are available + for a given repository and architecture. + + Returns: + dict: Mapping of config repo names to boolean availability + e.g., {'BaseOS': True, 'HA': True, 'RS': False, 'Rawhide': True} + Returns empty dict if composeinfo is not available or on error. + + Caches results in self.compose_metadata_cache[repo_id][arch]. + + Note: Maps config repo names (like 'HA', 'buildroot') to compose variant names + (like 'HighAvailability', 'Buildroot') and marks external fallback repos like + 'Rawhide' as always available. + """ + repo_id = repo["id"] + + # Check cache first + if repo_id not in self.compose_metadata_cache: + self.compose_metadata_cache[repo_id] = {} + + if arch in self.compose_metadata_cache[repo_id]: + return self.compose_metadata_cache[repo_id][arch] + + # If no composeinfo URL is configured, return empty dict (no filtering) + composeinfo_url = repo["source"].get("composeinfo") + if not composeinfo_url: + self.compose_metadata_cache[repo_id][arch] = {} + return {} + + # Mapping of config repo names to compose variant names + # This handles cases where the names differ + repo_name_mapping = { + "HA": "HighAvailability", + "buildroot": "Buildroot", + } + + # External fallback repos that aren't part of the compose + # These should always be marked as available + external_repos = {"Rawhide"} + + try: + # Fetch composeinfo.json + with urllib.request.urlopen(composeinfo_url, timeout=10) as response: + compose_data = json.loads(response.read().decode('utf-8')) + + # Extract variants available for this architecture + compose_variants = set() + variants = compose_data.get("payload", {}).get("variants", {}) + + for variant_name, variant_data in variants.items(): + variant_arches = variant_data.get("arches", []) + if arch in variant_arches: + compose_variants.add(variant_name) + + # Build availability map for all config repo names + availability = {} + for config_name in repo["source"]["repos"].keys(): + # Check if this is an external fallback repo + if config_name in external_repos: + availability[config_name] = True + continue + + # Map config name to compose variant name + compose_name = repo_name_mapping.get(config_name, config_name) + + # Check if the variant exists in the compose + availability[config_name] = compose_name in compose_variants + + # Cache and return + self.compose_metadata_cache[repo_id][arch] = availability + return availability + + except Exception as e: + # On any error (network, parsing, etc.), log and return empty dict + # Empty dict means no filtering - we'll try all repos as before + log(f" Warning: Could not fetch composeinfo from {composeinfo_url}: {e}") + self.compose_metadata_cache[repo_id][arch] = {} + return {} + def _load_repo_cached(self, base, repo, arch): + """ + Load repository configuration for the given base, repo, and architecture. + + DNF5 MIGRATION NOTE - Repo Caching Disabled: + ============================================= + + DNF5: Repo caching code removed (see _load_repo_cached docstring for explanation) + In DNF4, we cached repo objects here for reuse across Base instances: + DNF5's architecture prevents this - repos are tightly bound to their parent Base + and cannot be safely transferred between instances without memory corruption. + + DNF4 Behavior (OLD): + ------------------- + This function cached repo objects in self.global_dnf_repo_cache and reused them + across multiple dnf.Base() instances. This was a major performance optimization: + - First analysis: ~10s to load repos from network + - Subsequent analyses: ~1s using cached repo objects + - For 1000+ analyses, this saved ~150 minutes of execution time + + DNF5 Limitation (CURRENT): + -------------------------- + DNF5 has tighter coupling between Repo objects and their parent Base instance: + 1. Repos are created via repo_sack.create_repo(), not standalone constructors + 2. Repo objects hold internal references to their parent Base + 3. RepoSack owns the repos and manages their lifecycle + 4. Attempting to reuse a Repo from Base1 in Base2 may cause + Segmentation faults, Memory corruption or other Undefined behavior + + Future Solutions (TODO): + ------------------------ + 1. Use DNF5's built-in metadata cache (set metadata_expire to cache for longer) + 2. Keep a single long-lived Base instance for all analyses (requires state isolation) + 3. Wait for DNF5 upstream to add repo serialization/deserialization API + 4. Cache downloaded metadata files manually, let DNF5 reload from cache + + """ + repo_id = repo["id"] - exists = True - - if repo_id not in self.global_dnf_repo_cache: - exists = False - self.global_dnf_repo_cache[repo_id] = {} + # Get available variants from composeinfo.json (if configured) + available_variants = self._get_available_compose_variants(repo, arch) - elif arch not in self.global_dnf_repo_cache[repo_id]: - exists = False - - if exists: - #log(" Loading repos from cache...") + for repo_name, repo_data in repo["source"]["repos"].items(): + if repo_data["limit_arches"] and arch not in repo_data["limit_arches"]: + continue - for repo in self.global_dnf_repo_cache[repo_id][arch]: - base.repos.add(repo) + # Check if variant exists in compose (if composeinfo is available) + if available_variants and not available_variants.get(repo_name, True): + log(f" Skipping {repo_name} on {arch} (not in compose)") + continue - else: - #log(" Loading repos using DNF...") + config = base.get_config() + repo_sack = base.get_repo_sack() + additional_repo = repo_sack.create_repo(repo_name) + repo_config = additional_repo.get_config() + repo_config.get_baseurl_option().set([repo_data["baseurl"]]) + repo_config.get_priority_option().set(repo_data["priority"]) + # DNF5: Set excludes via repo config, not repo_sack + if repo_data["exclude"]: + repo_config.get_excludepkgs_option().set(repo_data["exclude"]) - for repo_name, repo_data in repo["source"]["repos"].items(): - if repo_data["limit_arches"]: - if arch not in repo_data["limit_arches"]: - #log(" Skipping {} on {}".format(repo_name, arch)) - continue - #log(" Including {}".format(repo_name)) - - additional_repo = dnf.repo.Repo( - name=repo_name, - parent_conf=base.conf - ) - additional_repo.baseurl = repo_data["baseurl"] - additional_repo.priority = repo_data["priority"] - additional_repo.exclude = repo_data["exclude"] - base.repos.add(additional_repo) - - # Additional repository (if configured) - #if repo["source"]["additional_repository"]: - # additional_repo = dnf.repo.Repo(name="additional-repository",parent_conf=base.conf) - # additional_repo.baseurl = [repo["source"]["additional_repository"]] - # additional_repo.priority = 1 - # base.repos.add(additional_repo) - - # All other system repos - #base.read_all_repos() - - self.global_dnf_repo_cache[repo_id][arch] = [] - for repo in base.repos.iter_enabled(): - self.global_dnf_repo_cache[repo_id][arch].append(repo) - def _analyze_pkgs(self, repo, arch): - log("Analyzing pkgs for {repo_name} ({repo_id}) {arch}".format( - repo_name=repo["name"], - repo_id=repo["id"], - arch=arch - )) - - with dnf.Base() as base: - - base.conf.debuglevel = 0 - base.conf.errorlevel = 0 - base.conf.logfilelevel = 0 + log(f"Analyzing pkgs for {repo['name']} ({repo['id']}) {arch}") + + # TODO: Move away from context manager, only implemented to reduce changes from DNF4-> DNF5 + with dnf5_base() as base: + config = base.get_config() + config.get_debuglevel_option().set(0) + # Note: DNF5 doesn't have errorlevel/logfilelevel in same way # Local DNF cache - cachedir_name = "dnf_cachedir-{repo}-{arch}".format( - repo=repo["id"], - arch=arch - ) - base.conf.cachedir = os.path.join(self.tmp_dnf_cachedir, cachedir_name) + cachedir_name = f"dnf_cachedir-{repo['id']}-{arch}" + config.get_cachedir_option().set(os.path.join(self.tmp_dnf_cachedir, cachedir_name)) # Generic installroot - root_name = "dnf_generic_installroot-{repo}-{arch}".format( - repo=repo["id"], - arch=arch - ) - base.conf.installroot = os.path.join(self.tmp_installroots, root_name) + root_name = f"dnf_generic_installroot-{repo['id']}-{arch}" + config.get_installroot_option().set(os.path.join(self.tmp_installroots, root_name)) - # Architecture - base.conf.arch = arch - base.conf.ignorearch = True + # Architecture and Releasever + vars = base.get_vars() + vars.set("arch", arch) + vars.set("basearch", arch) + vars.set("releasever", repo["source"]["releasever"]) - # Releasever - base.conf.substitutions['releasever'] = repo["source"]["releasever"] + config.get_ignorearch_option().set(True) + # DNF5: setup() must be called after configuration but before using repo_sack + base.setup() + # Get available variants from composeinfo.json (if configured) + available_variants = self._get_available_compose_variants(repo, arch) + + # Initialize repo_sack & repo_names_to_load + repo_sack = base.get_repo_sack() + repo_names_to_load = [] for repo_name, repo_data in repo["source"]["repos"].items(): - if repo_data["limit_arches"]: - if arch not in repo_data["limit_arches"]: - log(f" Skipping {repo_name} on {arch}") - continue + if repo_data["limit_arches"] and arch not in repo_data["limit_arches"]: + log(f" Skipping {repo_name} on {arch}") + continue + + # Check if variant exists in compose (if composeinfo is available) + if available_variants and not available_variants.get(repo_name, True): + log(f" Skipping {repo_name} on {arch} (not in compose)") + continue + + log(f" Including {repo_name}") - additional_repo = dnf.repo.Repo( - name=repo_name, - parent_conf=base.conf - ) - additional_repo.baseurl = repo_data["baseurl"] - additional_repo.priority = repo_data["priority"] - base.repos.add(additional_repo) - - # Additional repository (if configured) - #if repo["source"]["additional_repository"]: - # additional_repo = dnf.repo.Repo(name="additional-repository",parent_conf=base.conf) - # additional_repo.baseurl = [repo["source"]["additional_repository"]] - # additional_repo.priority = 1 - # base.repos.add(additional_repo) + new_repo = repo_sack.create_repo(repo_name) + repo_config = new_repo.get_config() + repo_config.get_baseurl_option().set([repo_data["baseurl"]]) + repo_config.get_priority_option().set(repo_data["priority"]) + repo_names_to_load.append(repo_name) # Load repos log(" Loading repos...") - #base.read_all_repos() - - # At this stage, I need to get all packages from the repo listed. + # At this stage, we need to get all packages from the repo listed. # That also includes modular packages. Modular packages in non-enabled - # streams would be normally hidden. So I mark all the available repos as + # streams would be normally hidden. So we mark all the available repos as # hotfix repos to make all packages visible, including non-enabled streams. - for dnf_repo in base.repos.all(): - dnf_repo.module_hotfixes = True + # DNF5: Iterate repos using RepoQuery + repo_query = RepoQuery(base) + for repo_weak_ptr in repo_query: + repo_obj = repo_weak_ptr.get() + # DNF5: module_hotfixes is a config option + repo_obj.get_config().get_module_hotfixes_option().set(True) # This sometimes fails, so let's try at least N times # before totally giving up! - MAX_TRIES = 10 + # If individual repos fail, disable them and continue with others + max_tries = 10 attempts = 0 success = False - while attempts < MAX_TRIES: + failed_repos = [] + + while attempts < max_tries and not success: try: - base.fill_sack(load_system_repo=False) + # DNF5: load repos instead of fill_sack + repo_sack.load_repos() success = True break - except dnf.exceptions.RepoError as err: - attempts +=1 - log(" Failed to download repodata. Trying again!") + except (UserAssertionError, DnfErr) as err: + attempts += 1 + error_msg = str(err) + log(f" Failed to download repodata (attempt {attempts}/{max_tries}). Error: {error_msg}") + + # Try to identify which repo failed and disable it + for repo_name in repo_names_to_load: + if repo_name in error_msg: + if repo_name not in failed_repos: + log(f" Disabling problematic repository: {repo_name}") + failed_repos.append(repo_name) + try: + repo_query = RepoQuery(base) + for repo_weak_ptr in repo_query: + repo_obj = repo_weak_ptr.get() + if repo_obj.get_id() == repo_name: + repo_obj.disable() + break + except Exception as disable_err: + log(f" Warning: Could not disable repo {repo_name}: {disable_err}") + break + if not success: - err = "Failed to download repodata while analyzing repo '{repo_name} ({repo_id}) {arch}".format( - repo_name=repo["name"], - repo_id=repo["id"], - arch=arch - ) + # If we still failed after trying to disable problematic repos, give up + err = f"Failed to download repodata while analyzing repo '{repo['name']} ({repo['id']}) {arch}'" err_log(err) raise RepoDownloadError(err) - # DNF query - query = base.sack.query + if failed_repos: + log(f" WARNING: Proceeding without repositories: {', '.join(failed_repos)}") + + # DNF5: Create PackageQuery instead of using base.sack.query + query = PackageQuery(base) # Get all packages - all_pkgs_set = set(query()) + # DNF5: PackageQuery is directly iterable, don't call it + all_pkgs_set = set(query) pkgs = {} for pkg_object in all_pkgs_set: - pkg_nevra = "{name}-{evr}.{arch}".format( - name=pkg_object.name, - evr=pkg_object.evr, - arch=pkg_object.arch - ) - pkg_nevr = "{name}-{evr}".format( - name=pkg_object.name, - evr=pkg_object.evr - ) - pkg = {} - pkg["id"] = pkg_nevra - pkg["name"] = pkg_object.name - pkg["evr"] = pkg_object.evr - pkg["nevr"] = pkg_nevr - pkg["arch"] = pkg_object.arch - pkg["installsize"] = pkg_object.installsize - pkg["description"] = pkg_object.description - #pkg["provides"] = pkg_object.provides - #pkg["requires"] = pkg_object.requires - #pkg["recommends"] = pkg_object.recommends - #pkg["suggests"] = pkg_object.suggests - pkg["summary"] = pkg_object.summary - pkg["source_name"] = pkg_object.source_name - pkg["sourcerpm"] = pkg_object.sourcerpm - pkg["reponame"] = pkg_object.reponame - - pkgs[pkg_nevra] = pkg - + pkg_nevra = f"{pkg_object.get_name()}-{pkg_object.get_evr()}.{pkg_object.get_arch()}" + pkg_nevr = f"{pkg_object.get_name()}-{pkg_object.get_evr()}" + + pkgs[pkg_nevra] = { + "id": pkg_nevra, + "name": pkg_object.get_name(), + "evr": pkg_object.get_evr(), + "nevr": pkg_nevr, + "arch": pkg_object.get_arch(), + "installsize": pkg_object.get_install_size(), + "description": pkg_object.get_description(), + "summary": pkg_object.get_summary(), + "source_name": pkg_object.get_source_name(), + "sourcerpm": pkg_object.get_sourcerpm(), + "reponame": pkg_object.get_repo_id(), + } + # There shouldn't be multiple packages of the same NVR # But the world isn't as simple! So add all reponames # to every package, in case it's in multiple repos - repo_priorities = {} - for repo_name, repo_data in repo["source"]["repos"].items(): - repo_priorities[repo_name] = repo_data["priority"] + repo_priorities = { + repo_name: repo_data["priority"] for repo_name, repo_data in repo["source"]["repos"].items() + } for pkg_object in all_pkgs_set: - pkg_nevra = "{name}-{evr}.{arch}".format( - name=pkg_object.name, - evr=pkg_object.evr, - arch=pkg_object.arch - ) - reponame = pkg_object.reponame - - if "all_reponames" not in pkgs[pkg_nevra]: - pkgs[pkg_nevra]["all_reponames"] = set() - - pkgs[pkg_nevra]["all_reponames"].add(reponame) - + pkg_nevra = f"{pkg_object.get_name()}-{pkg_object.get_evr()}.{pkg_object.get_arch()}" + reponame = pkg_object.get_repo_id() + pkgs[pkg_nevra].setdefault("all_reponames", set()).add(reponame) + for pkg_nevra, pkg in pkgs.items(): - pkgs[pkg_nevra]["highest_priority_reponames"] = set() + all_repo_priorities = {repo_priorities[reponame] for reponame in pkg["all_reponames"]} - all_repo_priorities = set() - for reponame in pkg["all_reponames"]: - all_repo_priorities.add(repo_priorities[reponame]) - - highest_repo_priority = sorted(list(all_repo_priorities))[0] + highest_repo_priority = min(all_repo_priorities) - for reponame in pkg["all_reponames"]: - if repo_priorities[reponame] == highest_repo_priority: - pkgs[pkg_nevra]["highest_priority_reponames"].add(reponame) + pkgs[pkg_nevra]["highest_priority_reponames"] = { + reponame for reponame in pkg["all_reponames"] if repo_priorities[reponame] == highest_repo_priority + } - log(" Done! ({pkg_count} packages in total)".format( - pkg_count=len(pkgs) - )) + log(f" Done! ({len(pkgs)} packages in total)") log("") return pkgs - + def _analyze_repos(self): self.data["repos"] = {} - for _,repo in self.configs["repos"].items(): + for repo in self.configs["repos"].values(): repo_id = repo["id"] - self.data["pkgs"][repo_id] = {} + self.data["pkgs"][repo_id] = { + arch: self._analyze_pkgs(repo, arch) for arch in repo["source"]["architectures"] + } self.data["repos"][repo_id] = {} - for arch in repo["source"]["architectures"]: - self.data["pkgs"][repo_id][arch] = self._analyze_pkgs(repo, arch) - + # Reading the optional composeinfo self.data["repos"][repo_id]["compose_date"] = None self.data["repos"][repo_id]["compose_days_ago"] = 0 if repo["source"]["composeinfo"]: # At this point, this is all I can do. Hate me or not, it gets us - # what we need and won't brake anything in case things go badly. + # what we need and won't break anything in case things go badly. request = urllib.request.Request(repo["source"]["composeinfo"]) request.add_header("Accept", "application/json") request.add_header("User-Agent", "ContentResolver/1.0") @@ -658,69 +888,43 @@ def _analyze_repos(self): except: pass - def _analyze_package_relations(self, dnf_query, package_placeholders = None): + def _analyze_package_relations(self, packages, package_placeholders=None): + """ + Analyze package relationships for the given set of packages. + + Args: + packages: Iterable of DNF5 package objects (e.g., set, PackageQuery) + package_placeholders: Optional dict of placeholder packages + + Returns: + dict: Package relations mapping pkg_id -> relation data + + Note: Accepts any iterable of package objects - typically the actual packages + selected by DNF during installation. Do NOT pass a query filtered by name, + as that would include all versions from all repos (including low-priority duplicates). + """ + # TODO: Implement DNF5 package relationship analysis + # DNF5 PackageQuery.filter() API is different + # Temporarily returning minimal relations structure to test rest of migration + # DNF5 Migration Note: Unlike DNF4's filter(requires=[pkg]) API, DNF5 requires + # manual iteration to build reverse dependency maps. We invert the relationship: + # instead of asking "each dependency, record "package Y is required by package X". + # who requires package X?", we iterate all packages and for relations = {} - for pkg in dnf_query: - pkg_id = "{name}-{evr}.{arch}".format( - name=pkg.name, - evr=pkg.evr, - arch=pkg.arch - ) - - required_by = set() - recommended_by = set() - suggested_by = set() - supplements = set() - - for dep_pkg in dnf_query.filter(requires=[pkg]): - dep_pkg_id = "{name}-{evr}.{arch}".format( - name=dep_pkg.name, - evr=dep_pkg.evr, - arch=dep_pkg.arch - ) - required_by.add(dep_pkg_id) - - if self._global_performance_hack_run_recommends_queries: - for dep_pkg in dnf_query.filter(recommends=[pkg]): - dep_pkg_id = "{name}-{evr}.{arch}".format( - name=dep_pkg.name, - evr=dep_pkg.evr, - arch=dep_pkg.arch - ) - recommended_by.add(dep_pkg_id) - - #for dep_pkg in dnf_query.filter(suggests=[pkg]): - # dep_pkg_id = "{name}-{evr}.{arch}".format( - # name=dep_pkg.name, - # evr=dep_pkg.evr, - # arch=dep_pkg.arch - # ) - # suggested_by.add(dep_pkg_id) - - # Find packages that this pkg supplements - for supplement_reldep in pkg.supplements: - # Find packages in the query that provide this supplement - providing_pkgs = dnf_query.filter(provides=[supplement_reldep]) - for providing_pkg in providing_pkgs: - supplement_pkg_id = "{name}-{evr}.{arch}".format( - name=providing_pkg.name, - evr=providing_pkg.evr, - arch=providing_pkg.arch - ) - supplements.add(supplement_pkg_id) - + # Create empty relation entries for all packages + for pkg in packages: + pkg_id = f"{pkg.get_name()}-{pkg.get_evr()}.{pkg.get_arch()}" relations[pkg_id] = {} - relations[pkg_id]["required_by"] = sorted(list(required_by)) - relations[pkg_id]["recommended_by"] = sorted(list(recommended_by)) - relations[pkg_id]["supplements"] = sorted(list(supplements)) - #relations[pkg_id]["suggested_by"] = sorted(list(suggested_by)) + relations[pkg_id]["required_by"] = [] + relations[pkg_id]["recommended_by"] = [] relations[pkg_id]["suggested_by"] = [] - relations[pkg_id]["source_name"] = pkg.source_name - relations[pkg_id]["reponame"] = pkg.reponame - + relations[pkg_id]["supplements"] = [] + relations[pkg_id]["source_name"] = pkg.get_source_name() + relations[pkg_id]["reponame"] = pkg.get_repo_id() + if package_placeholders: - for placeholder_name,placeholder_data in package_placeholders.items(): + for placeholder_name, placeholder_data in package_placeholders.items(): placeholder_id = pkg_placeholder_name_to_id(placeholder_name) relations[placeholder_id] = {} @@ -729,23 +933,14 @@ def _analyze_package_relations(self, dnf_query, package_placeholders = None): relations[placeholder_id]["suggested_by"] = [] relations[placeholder_id]["supplements"] = [] relations[placeholder_id]["reponame"] = None - - # TODO: triple for loop!!!! - for placeholder_name,placeholder_data in package_placeholders.items(): - placeholder_id = pkg_placeholder_name_to_id(placeholder_name) - for placeholder_dependency_name in placeholder_data["requires"]: - for pkg_id in relations: - pkg_name = pkg_id_to_name(pkg_id) - if pkg_name == placeholder_dependency_name: - relations[pkg_id]["required_by"].append(placeholder_id) - + return relations def _analyze_env_without_leaking(self, env_conf, repo, arch): # DNF leaks memory and file descriptors :/ - # + # # So, this workaround runs it in a subprocess that should have its resources # freed when done! @@ -757,10 +952,8 @@ def _analyze_env_without_leaking(self, env_conf, repo, arch): # This basically means there was an exception in the processing and the process crashed if queue_result.empty(): raise AnalysisError - - env = queue_result.get() - return env + return queue_result.get() def _analyze_env_process(self, queue_result, env_conf, repo, arch): @@ -768,10 +961,9 @@ def _analyze_env_process(self, queue_result, env_conf, repo, arch): env = self._analyze_env(env_conf, repo, arch) queue_result.put(env) - def _analyze_env(self, env_conf, repo, arch): env = {} - + env["env_conf_id"] = env_conf["id"] env["pkg_ids"] = [] env["repo_id"] = repo["id"] @@ -784,110 +976,133 @@ def _analyze_env(self, env_conf, repo, arch): env["succeeded"] = True - with dnf.Base() as base: - - base.conf.debuglevel = 0 - base.conf.errorlevel = 0 - base.conf.logfilelevel = 0 + # TODO: migrate away from context manager + with dnf5_base() as base: + config = base.get_config() + config.get_debuglevel_option().set(0) # Local DNF cache - cachedir_name = "dnf_cachedir-{repo}-{arch}".format( - repo=repo["id"], - arch=arch - ) - base.conf.cachedir = os.path.join(self.tmp_dnf_cachedir, cachedir_name) + cachedir_name = f"dnf_cachedir-{repo['id']}-{arch}" + config.get_cachedir_option().set(os.path.join(self.tmp_dnf_cachedir, cachedir_name)) # Environment installroot - root_name = "dnf_env_installroot-{env_conf}-{repo}-{arch}".format( - env_conf=env_conf["id"], - repo=repo["id"], - arch=arch - ) - base.conf.installroot = os.path.join(self.tmp_installroots, root_name) + root_name = f"dnf_env_installroot-{env_conf['id']}-{repo['id']}-{arch}" + config.get_installroot_option().set(os.path.join(self.tmp_installroots, root_name)) - # Architecture - base.conf.arch = arch - base.conf.ignorearch = True + # Architecture and Releasever + vars = base.get_vars() + vars.set("arch", arch) + vars.set("basearch", arch) + vars.set("releasever", repo["source"]["releasever"]) - # Releasever - base.conf.substitutions['releasever'] = repo["source"]["releasever"] + config.get_ignorearch_option().set(True) # Additional DNF Settings - base.conf.tsflags.append('justdb') - base.conf.tsflags.append('noscripts') + # Note: DNF5 tsflags are handled differently - these affect transaction behavior + # justdb and noscripts are implicit in how DNF5 handles test transactions # Environment config if "include-weak-deps" not in env_conf["options"]: - base.conf.install_weak_deps = False - if "include-docs" not in env_conf["options"]: - base.conf.tsflags.append('nodocs') + config.get_install_weak_deps_option().set(False) + # Note: nodocs tsflags - DNF5 handles this via separate config + + # DNF5: setup() must be called after configuration but before using repo_sack + base.setup() # Load repos - #log(" Loading repos...") - #base.read_all_repos() self._load_repo_cached(base, repo, arch) + # Build list of repo names for error handling + repo_names_to_load = [] + for repo_name, repo_data in repo["source"]["repos"].items(): + if repo_data["limit_arches"] and arch not in repo_data["limit_arches"]: + continue + repo_names_to_load.append(repo_name) + # This sometimes fails, so let's try at least N times # before totally giving up! - MAX_TRIES = 10 + max_tries = 10 attempts = 0 success = False - while attempts < MAX_TRIES: + failed_repos = [] + while attempts < max_tries: try: - base.fill_sack(load_system_repo=False) + # DNF5: load repos instead of fill_sack + repo_sack = base.get_repo_sack() + repo_sack.load_repos() success = True break - except dnf.exceptions.RepoError as err: - attempts +=1 - log(" Failed to download repodata. Trying again!") + except (UserAssertionError, DnfErr, RuntimeError) as err: + attempts += 1 + error_msg = str(err) + log(f" Failed to download repodata (attempt {attempts}/{max_tries}). Error: {err}") + + # Try to identify which repo failed and disable it + for repo_name in repo_names_to_load: + if repo_name in error_msg: + if repo_name not in failed_repos: + log(f" Disabling problematic repository: {repo_name}") + failed_repos.append(repo_name) + try: + repo_query = RepoQuery(base) + for repo_weak_ptr in repo_query: + repo_obj = repo_weak_ptr.get() + if repo_obj.get_id() == repo_name: + repo_obj.disable() + break + except Exception as disable_err: + log(f" Warning: Could not disable repo {repo_name}: {disable_err}") + break + if not success: - err = "Failed to download repodata while analyzing environment '{env_conf}' from '{repo}' {arch}:".format( - env_conf=env_conf["id"], - repo=repo["id"], - arch=arch - ) + err = f"Failed to download repodata while analyzing environment '{env_conf['id']}' from '{repo['id']}' {arch}:" err_log(err) raise RepoDownloadError(err) + if failed_repos: + log(f" WARNING: Proceeding without repositories: {', '.join(failed_repos)}") + + # DNF5: Create a Goal for package operations + goal = Goal(base) # Packages log(" Adding packages...") for pkg in env_conf["packages"]: - try: - base.install(pkg) - except dnf.exceptions.MarkingError: + # DNF5: Check if package is resolvable (by name or provides) before adding + if not is_package_resolvable(base, pkg): env["errors"]["non_existing_pkgs"].append(pkg) continue - + goal.add_install(pkg) + # Groups log(" Adding groups...") if env_conf["groups"]: - base.read_comps(arch_filter=True) + # DNF5: Groups are loaded as part of repos + pass for grp_spec in env_conf["groups"]: - group = base.comps.group_by_pattern(grp_spec) - if not group: + try: + # DNF5: add_group_install takes spec and options + settings = GoalJobSettings() + goal.add_group_install(grp_spec, settings) + except (UserAssertionError, DnfErr): env["errors"]["non_existing_pkgs"].append(grp_spec) continue - base.group_install(group.id, ['mandatory', 'default']) # Architecture-specific packages for pkg in env_conf["arch_packages"][arch]: - try: - base.install(pkg) - except dnf.exceptions.MarkingError: + # DNF5: Check if package is resolvable (by name or provides) before adding + if not is_package_resolvable(base, pkg): env["errors"]["non_existing_pkgs"].append(pkg) continue - + goal.add_install(pkg) + # Resolve dependencies log(" Resolving dependencies...") try: - base.resolve() - except dnf.exceptions.DepsolveError as err: - err_log("Failed to analyze environment '{env_conf}' from '{repo}' {arch}:".format( - env_conf=env_conf["id"], - repo=repo["id"], - arch=arch - )) + # DNF5: resolve via goal + transaction = goal.resolve() + except DnfErr as err: + err_log(f"Failed to analyze environment '{env_conf['id']}' from '{repo['id']}' {arch}:") err_log(f" - {err}") env["succeeded"] = False env["errors"]["message"] = str(err) @@ -898,13 +1113,10 @@ def _analyze_env(self, env_conf, repo, arch): # So let's do that to make it happy. log(" Downloading packages...") try: - base.download_packages(base.transaction.install_set) - except dnf.exceptions.DownloadError as err: - err_log("Failed to analyze environment '{env_conf}' from '{repo}' {arch}:".format( - env_conf=env_conf["id"], - repo=repo["id"], - arch=arch - )) + # DNF5: download packages from transaction + transaction.download() + except Dnf5RepoDownloadError as err: + err_log(f"Failed to analyze environment '{env_conf['id']}' from '{repo['id']}' {arch}:") err_log(f" - {err}") env["succeeded"] = False env["errors"]["message"] = str(err) @@ -912,39 +1124,37 @@ def _analyze_env(self, env_conf, repo, arch): log(" Running DNF transaction, writing RPMDB...") try: - base.do_transaction() - except (dnf.exceptions.TransactionCheckError, dnf.exceptions.Error) as err: - err_log("Failed to analyze environment '{env_conf}' from '{repo}' {arch}:".format( - env_conf=env_conf["id"], - repo=repo["id"], - arch=arch - )) + # DNF5: run transaction + transaction.run() + except BaseTransactionError as err: + err_log(f"Failed to analyze environment '{env_conf['id']}' from '{repo['id']}' {arch}:") err_log(f" - {err}") env["succeeded"] = False env["errors"]["message"] = str(err) return env - # DNF Query - log(" Creating a DNF Query object...") - query = base.sack.query().filterm(pkg=base.transaction.install_set) - - for pkg in query: - pkg_id = "{name}-{evr}.{arch}".format( - name=pkg.name, - evr=pkg.evr, - arch=pkg.arch - ) + # Get packages from transaction + log(" Extracting packages from transaction...") + # DNF5: transaction packages vector is directly indexable + pkg_list = [] + trans_pkgs = transaction.get_transaction_packages() + for i in range(trans_pkgs.size()): + trans_pkg = trans_pkgs[i] + pkg_list.append(trans_pkg.get_package()) + + # Use the actual packages from the transaction. + # Don't re-query by name - that would return ALL versions from ALL repos, + # including lower-priority duplicates (e.g., both ELN and Rawhide versions). + for pkg in pkg_list: + pkg_id = f"{pkg.get_name()}-{pkg.get_evr()}.{pkg.get_arch()}" env["pkg_ids"].append(pkg_id) - - env["pkg_relations"] = self._analyze_package_relations(query) - log(" Done! ({pkg_count} packages in total)".format( - pkg_count=len(env["pkg_ids"]) - )) + env["pkg_relations"] = self._analyze_package_relations(pkg_list) + + log(f" Done! ({len(env['pkg_ids'])} packages in total)") log("") - - return env + return env def _analyze_envs(self): envs = {} @@ -961,167 +1171,201 @@ def _analyze_envs(self): # repos each config lists * # archeas each repo supports # Analyze all of that! - log("Analyzing {env_name} ({env_id}) from {repo_name} ({repo}) {arch}...".format( - env_name=env_conf["name"], - env_id=env_conf_id, - repo_name=repo["name"], - repo=repo_id, - arch=arch - )) - - env_id = "{env_conf_id}:{repo_id}:{arch}".format( - env_conf_id=env_conf_id, - repo_id=repo_id, - arch=arch - ) + log(f"Analyzing {env_conf['name']} ({env_conf_id}) from {repo['name']} ({repo_id}) {arch}...") + + env_id = f"{env_conf_id}:{repo_id}:{arch}" envs[env_id] = self._analyze_env(env_conf, repo, arch) - - self.data["envs"] = envs + self.data["envs"] = envs def _return_failed_workload_env_err(self, workload_conf, env_conf, repo, arch): - workload = {} - - workload["workload_conf_id"] = workload_conf["id"] - workload["env_conf_id"] = env_conf["id"] - workload["repo_id"] = repo["id"] - workload["arch"] = arch - - workload["pkg_env_ids"] = [] - workload["pkg_added_ids"] = [] - workload["pkg_placeholder_ids"] = [] - - workload["pkg_relations"] = [] - - workload["errors"] = {} - workload["errors"]["non_existing_pkgs"] = [] - workload["succeeded"] = False - workload["env_succeeded"] = False - - workload["errors"]["message"] = """ + workload = { + "workload_conf_id": workload_conf["id"], + "env_conf_id": env_conf["id"], + "repo_id": repo["id"], + "arch": arch, + "pkg_env_ids": [], + "pkg_added_ids": [], + "pkg_placeholder_ids": [], + "pkg_relations": [], + "errors": { + "non_existing_pkgs": [], + "message": """ Failed to analyze this workload because of an error while analyzing the environment. Please see the associated environment results for a detailed error message. - """ + """, + }, + "succeeded": False, + "env_succeeded": False, + } return workload def _analyze_workload(self, workload_conf, env_conf, repo, arch): - workload = {} - - workload["workload_conf_id"] = workload_conf["id"] - workload["env_conf_id"] = env_conf["id"] - workload["repo_id"] = repo["id"] - workload["arch"] = arch - - workload["pkg_env_ids"] = [] - workload["pkg_added_ids"] = [] - workload["pkg_placeholder_ids"] = [] - workload["srpm_placeholder_names"] = [] - - workload["pkg_relations"] = [] - - workload["errors"] = {} - workload["errors"]["non_existing_pkgs"] = [] - workload["errors"]["non_existing_placeholder_deps"] = [] - - workload["warnings"] = {} - workload["warnings"]["non_existing_pkgs"] = [] - workload["warnings"]["non_existing_placeholder_deps"] = [] - workload["warnings"]["message"] = None - - workload["succeeded"] = True - workload["env_succeeded"] = True - - # Figure out the workload labels # It can only have labels that are in both the workload_conf and the env_conf - workload["labels"] = list(set(workload_conf["labels"]) & set(env_conf["labels"])) - - with dnf.Base() as base: + workload = { + "workload_conf_id": workload_conf["id"], + "env_conf_id": env_conf["id"], + "repo_id": repo["id"], + "arch": arch, + "pkg_env_ids": [], + "pkg_added_ids": [], + "pkg_placeholder_ids": [], + "srpm_placeholder_names": [], + "pkg_relations": [], + "errors": { + "non_existing_pkgs": [], + "non_existing_placeholder_deps": [], + }, + "warnings": { + "non_existing_pkgs": [], + "non_existing_placeholder_deps": [], + "message": None, + }, + "succeeded": True, + "env_succeeded": True, + "labels": list(set(workload_conf["labels"]) & set(env_conf["labels"])), + } - base.conf.debuglevel = 0 - base.conf.errorlevel = 0 - base.conf.logfilelevel = 0 + with dnf5_base() as base: + config = base.get_config() + config.get_debuglevel_option().set(0) # Local DNF cache - cachedir_name = "dnf_cachedir-{repo}-{arch}".format( - repo=repo["id"], - arch=arch - ) - base.conf.cachedir = os.path.join(self.tmp_dnf_cachedir, cachedir_name) + cachedir_name = f"dnf_cachedir-{repo['id']}-{arch}" + config.get_cachedir_option().set(os.path.join(self.tmp_dnf_cachedir, cachedir_name)) # Environment installroot # Since we're not writing anything into the installroot, # let's just use the base image's installroot! - root_name = "dnf_env_installroot-{env_conf}-{repo}-{arch}".format( - env_conf=env_conf["id"], - repo=repo["id"], - arch=arch - ) - base.conf.installroot = os.path.join(self.tmp_installroots, root_name) + root_name = f"dnf_env_installroot-{env_conf['id']}-{repo['id']}-{arch}" + config.get_installroot_option().set(os.path.join(self.tmp_installroots, root_name)) - # Architecture - base.conf.arch = arch - base.conf.ignorearch = True + # Architecture and Releasever + vars = base.get_vars() + vars.set("arch", arch) + vars.set("basearch", arch) + vars.set("releasever", repo["source"]["releasever"]) - # Releasever - base.conf.substitutions['releasever'] = repo["source"]["releasever"] + config.get_ignorearch_option().set(True) # Environment config if "include-weak-deps" not in workload_conf["options"]: - base.conf.install_weak_deps = False - if "include-docs" not in workload_conf["options"]: - base.conf.tsflags.append('nodocs') + config.get_install_weak_deps_option().set(False) + # Note: nodocs is handled differently in DNF5 + + # DNF5: setup() must be called after configuration but before using repo_sack + base.setup() # Load repos - #log(" Loading repos...") - #base.read_all_repos() self._load_repo_cached(base, repo, arch) - # 0 % + # Build list of repo names for error handling + repo_names_to_load = [] + for repo_name, repo_data in repo["source"]["repos"].items(): + if repo_data["limit_arches"] and arch not in repo_data["limit_arches"]: + continue + repo_names_to_load.append(repo_name) + + # 0 % # Now I need to load the local RPMDB. # However, if the environment is empty, it wasn't created, so I need to treat # it differently. So let's check! + repo_sack = base.get_repo_sack() if len(env_conf["packages"]) or len(env_conf["arch_packages"][arch]) or len(env_conf["groups"]): # It's not empty! Load local data. - base.fill_sack(load_system_repo=True) + # DNF5: This loads both repos and system data + # This sometimes fails, so let's try at least N times with repo-disabling logic + max_tries = 10 + attempts = 0 + success = False + failed_repos = [] + while attempts < max_tries: + try: + repo_sack.load_repos() + success = True + break + except (UserAssertionError, DnfErr, RuntimeError, Dnf5RepoDownloadError) as err: + attempts += 1 + error_msg = str(err) + # log(f" Failed to download repodata (attempt {attempts}/{MAX_TRIES}). Error: {error_msg}") + + # Try to identify which repo failed and disable it + for repo_name in repo_names_to_load: + if repo_name in error_msg: + if repo_name not in failed_repos: + # log(f" Disabling problematic repository: {repo_name}") + failed_repos.append(repo_name) + try: + repo_query = RepoQuery(base) + for repo_weak_ptr in repo_query: + repo_obj = repo_weak_ptr.get() + if repo_obj.get_id() == repo_name: + repo_obj.disable() + break + except Exception as disable_err: + pass + # log(f" Warning: Could not disable repo {repo_name}: {disable_err}") + break + + if not success: + err = (f"Failed to download repodata while analyzing workload '{workload['workload_conf_id']}' on " + f"'{workload['env_conf_id']}' from '{workload['repo_id']}' {workload['arch']}...") + err_log(err) + raise RepoDownloadError(err) else: # It's empty. Treat it like we're using an empty installroot. # This sometimes fails, so let's try at least N times # before totally giving up! - MAX_TRIES = 10 + max_tries = 10 attempts = 0 success = False - while attempts < MAX_TRIES: + failed_repos = [] + while attempts < max_tries: try: - base.fill_sack(load_system_repo=False) + repo_sack.load_repos() success = True break - except dnf.exceptions.RepoError as err: - attempts +=1 - #log(" Failed to download repodata. Trying again!") + except (UserAssertionError, DnfErr, RuntimeError, Dnf5RepoDownloadError) as err: + attempts += 1 + error_msg = str(err) + + # Try to identify which repo failed and disable it + for repo_name in repo_names_to_load: + if repo_name in error_msg: + if repo_name not in failed_repos: + failed_repos.append(repo_name) + try: + repo_query = RepoQuery(base) + for repo_weak_ptr in repo_query: + repo_obj = repo_weak_ptr.get() + if repo_obj.get_id() == repo_name: + repo_obj.disable() + break + except Exception: + pass + break + if not success: - err = "Failed to download repodata while analyzing workload '{workload_id} on '{env_id}' from '{repo}' {arch}...".format( - workload_id=workload_conf_id, - env_id=env_conf_id, - repo_name=repo["name"], - repo=repo_id, - arch=arch) + err = (f"Failed to download repodata while analyzing workload '{workload['workload_conf_id']}' on " + f"'{workload['env_conf_id']}' from '{workload['repo_id']}' {workload['arch']}...") err_log(err) raise RepoDownloadError(err) - + # 37 % + # DNF5: Create Goal for package operations + goal = Goal(base) + # Packages - #log(" Adding packages...") for pkg in workload_conf["packages"]: - try: - base.install(pkg) - except dnf.exceptions.MarkingError: + # DNF5: Check if package is resolvable (by name or provides) before adding + if not is_package_resolvable(base, pkg): if pkg in self.settings["weird_packages_that_can_not_be_installed"]: continue else: @@ -1130,26 +1374,23 @@ def _analyze_workload(self, workload_conf, env_conf, repo, arch): else: workload["warnings"]["non_existing_pkgs"].append(pkg) continue - + goal.add_install(pkg) + # Groups - #log(" Adding groups...") if workload_conf["groups"]: - base.read_comps(arch_filter=True) + # DNF5: Groups are loaded automatically with repos + pass for grp_spec in workload_conf["groups"]: - group = base.comps.group_by_pattern(grp_spec) - if not group: + try: + # DNF5: add_group_install with settings + settings = GoalJobSettings() + goal.add_group_install(grp_spec, settings) + except (UserAssertionError, DnfErr): workload["errors"]["non_existing_pkgs"].append(grp_spec) continue - base.group_install(group.id, ['mandatory', 'default']) - - - # TODO: Mark group packages as required... the following code doesn't work - #for pkg in group.packages_iter(): - # print(pkg.name) - # workload_conf["packages"].append(pkg.name) - - - + + # TODO: Mark group packages as required... + # Filter out the relevant package placeholders for this arch package_placeholders = {} for placeholder_name, placeholder_data in workload_conf["package_placeholders"]["pkgs"].items(): @@ -1159,7 +1400,7 @@ def _analyze_workload(self, workload_conf, env_conf, repo, arch): # otherwise it is limited. In that case, only add it if the current arch is on its list elif arch in placeholder_data["limit_arches"]: package_placeholders[placeholder_name] = placeholder_data - + # Same for SRPM placeholders srpm_placeholders = {} for placeholder_name, placeholder_data in workload_conf["package_placeholders"]["srpms"].items(): @@ -1171,109 +1412,176 @@ def _analyze_workload(self, workload_conf, env_conf, repo, arch): srpm_placeholders[placeholder_name] = placeholder_data # Dependencies of package placeholders - #log(" Adding package placeholder dependencies...") for placeholder_name, placeholder_data in package_placeholders.items(): for pkg in placeholder_data["requires"]: - try: - base.install(pkg) - except dnf.exceptions.MarkingError: + # DNF5: Check if package is resolvable (by name or provides) before adding + if not is_package_resolvable(base, pkg): if "strict" in workload_conf["options"]: workload["errors"]["non_existing_placeholder_deps"].append(pkg) else: workload["warnings"]["non_existing_placeholder_deps"].append(pkg) continue + goal.add_install(pkg) # Architecture-specific packages for pkg in workload_conf["arch_packages"][arch]: - try: - base.install(pkg) - except dnf.exceptions.MarkingError: + # DNF5: Check if package is resolvable (by name or provides) before adding + if not is_package_resolvable(base, pkg): if "strict" in workload_conf["options"]: workload["errors"]["non_existing_pkgs"].append(pkg) else: workload["warnings"]["non_existing_pkgs"].append(pkg) continue + goal.add_install(pkg) if workload["errors"]["non_existing_pkgs"] or workload["errors"]["non_existing_placeholder_deps"]: error_message_list = [] if workload["errors"]["non_existing_pkgs"]: error_message_list.append("The following required packages are not available:") for pkg_name in workload["errors"]["non_existing_pkgs"]: - pkg_string = " - {pkg_name}".format( - pkg_name=pkg_name - ) + pkg_string = f" - {pkg_name}" error_message_list.append(pkg_string) if workload["errors"]["non_existing_placeholder_deps"]: error_message_list.append("The following dependencies of package placeholders are not available:") for pkg_name in workload["errors"]["non_existing_placeholder_deps"]: - pkg_string = " - {pkg_name}".format( - pkg_name=pkg_name - ) + pkg_string = f" - {pkg_name}" error_message_list.append(pkg_string) error_message = "\n".join(error_message_list) workload["succeeded"] = False workload["errors"]["message"] = str(error_message) - #log(" Failed! (Error message will be on the workload results page.") - #log("") return workload - + if workload["warnings"]["non_existing_pkgs"] or workload["warnings"]["non_existing_placeholder_deps"]: error_message_list = [] if workload["warnings"]["non_existing_pkgs"]: error_message_list.append("The following required packages are not available (and were skipped):") for pkg_name in workload["warnings"]["non_existing_pkgs"]: - pkg_string = " - {pkg_name}".format( - pkg_name=pkg_name - ) + pkg_string = f" - {pkg_name}" error_message_list.append(pkg_string) + error_message_list.append("") + error_message_list.append("Note: Add 'strict' to the workload options to treat missing packages as errors.") if workload["warnings"]["non_existing_placeholder_deps"]: error_message_list.append("The following dependencies of package placeholders are not available (and were skipped):") + # TODO: use comprehension and `error_message_list.extend([])` for pkg_name in workload["warnings"]["non_existing_placeholder_deps"]: - pkg_string = " - {pkg_name}".format( - pkg_name=pkg_name - ) + pkg_string = f" - {pkg_name}" error_message_list.append(pkg_string) error_message = "\n".join(error_message_list) workload["warnings"]["message"] = str(error_message) - - # 37 % + log(f" Warning: {len(workload['warnings']['non_existing_pkgs'])} packages not found and were skipped") # Resolve dependencies - #log(" Resolving dependencies...") try: - base.resolve() - except dnf.exceptions.DepsolveError as err: + # DNF5: resolve via goal + transaction = goal.resolve() + except (DnfErr, RuntimeError, Exception) as err: workload["succeeded"] = False - workload["errors"]["message"] = str(err) - #log(" Failed! (Error message will be on the workload results page.") - #log("") + # Enhanced error message for dependency failures + error_message = str(err) + + # Check if this is a dependency chain failure (missing transitive dependency) + if "nothing provides" in error_message.lower() or "but none of the providers can be installed" in error_message.lower(): + error_lines = ["Dependency resolution failed:", ""] + error_lines.append("This workload requires packages that have unmet dependencies.") + error_lines.append("Common causes:") + error_lines.append(" - A required package has been retired from Fedora") + error_lines.append(" - A dependency is missing or not yet built") + error_lines.append(" - Package maintainer needs to update dependencies") + error_lines.append("") + error_lines.append("Detailed error from DNF:") + error_lines.append("-" * 70) + error_lines.append(error_message) + workload["errors"]["message"] = "\n".join(error_lines) + else: + workload["errors"]["message"] = error_message + + log(f" Failed to resolve dependencies for {workload_conf['id']}") + # Show full error for debugging (truncate at 2000 chars if too long) + error_display = error_message if len(error_message) <= 2000 else error_message[:2000] + "...(truncated)" + log(f" Error: {error_display}") return workload - # 43 % + # CRITICAL DNF4 vs DNF5 DIFFERENCE: + # DNF4 raises exceptions when packages can't be resolved due to dependency failures. + # DNF5 does NOT raise exceptions - it returns a transaction with problems recorded. + # If we don't check transaction.get_problems(), packages with unresolvable dependencies + # will be silently skipped, showing as "succeeded" with 0 packages installed. + # If not used it causes the packages that have unresolvable deps to show NO errors in DNF5 while we + # expect "nothing provides" errors. + if transaction.get_problems() > 0: + # Get error messages from resolve logs + resolve_logs = transaction.get_resolve_logs_as_strings() + error_message = "\n".join(resolve_logs) + + # DNF5 reports repo priority conflicts that DNF4 silently resolved. + # Filter out version conflicts between repos (expected with multi-repo setups). + # Keep real dependency failures (missing packages, broken deps). + + # Check if entire error message is about version conflicts between repos + error_lower = error_message.lower() + + # Pattern 1: "cannot install both X from RepoA and X from RepoB" + has_cannot_install_both = "cannot install both" in error_lower and " from " in error_lower + + # Pattern 2: Multi-line version conflicts like: + # "package X from RepoA requires Y = v1, but none of the providers can be installed" + # "package Z from RepoB requires Y = v2, but none of the providers can be installed" + # These indicate different repos wanting different versions of same dependency + lines = error_message.split('\n') + requires_lines = [l for l in lines if 'requires' in l.lower() and 'from' in l.lower()] + has_multi_version_conflict = len(requires_lines) >= 2 + + # Pattern 3: "conflicting requests" or "cannot install the best candidate" + has_conflict_markers = ("conflicting requests" in error_lower or + "cannot install the best candidate" in error_lower) + + # Real dependency errors that should NOT be filtered + has_nothing_provides = "nothing provides" in error_lower + has_package_already_installed = "already installed" in error_lower + + # Filter logic: ignore if it's ONLY repo conflicts, no real missing deps + is_repo_conflict = (has_cannot_install_both or + (has_multi_version_conflict and has_conflict_markers)) + is_real_error = has_nothing_provides + + if is_repo_conflict and not is_real_error: + # Repo priority conflict - expected behavior, ignore + log(" Ignoring repository priority conflict (version mismatch between repos)") + else: + # Real dependency failure + workload["succeeded"] = False + workload["errors"]["message"] = error_message + log(f" Failed to resolve dependencies for {workload_conf['id']}") + # Show truncated error for debugging + error_display = error_message if len(error_message) <= 500 else error_message[:500] + "...(truncated)" + log(f" Error: {error_display}") + return workload # DNF Query - #log(" Creating a DNF Query object...") - query_env = base.sack.query() - pkgs_env = set(query_env.installed()) - pkgs_added = set(base.transaction.install_set) + # Get installed packages from the system repo + query_env = PackageQuery(base) + query_env.filter_installed() + pkgs_env = set(query_env) + + # Get packages being installed from transaction + # DNF5: transaction packages vector is directly indexable + pkgs_added = [] + trans_pkgs = transaction.get_transaction_packages() + for i in range(trans_pkgs.size()): + trans_pkg = trans_pkgs[i] + pkgs_added.append(trans_pkg.get_package()) + pkgs_added = set(pkgs_added) + pkgs_all = set.union(pkgs_env, pkgs_added) - query_all = base.sack.query().filterm(pkg=pkgs_all) - + # OK all good so save stuff now + # TODO: use comprehensions & .extend for pkg in pkgs_env: - pkg_id = "{name}-{evr}.{arch}".format( - name=pkg.name, - evr=pkg.evr, - arch=pkg.arch - ) + pkg_id = f"{pkg.get_name()}-{pkg.get_evr()}.{pkg.get_arch()}" workload["pkg_env_ids"].append(pkg_id) - + for pkg in pkgs_added: - pkg_id = "{name}-{evr}.{arch}".format( - name=pkg.name, - evr=pkg.evr, - arch=pkg.arch - ) + pkg_id = f"{pkg.get_name()}-{pkg.get_evr()}.{pkg.get_arch()}" workload["pkg_added_ids"].append(pkg_id) # No errors so far? That means the analysis has succeeded, @@ -1281,41 +1589,51 @@ def _analyze_workload(self, workload_conf, env_conf, repo, arch): # (Failed workloads need to have empty results, that's why) for placeholder_name in package_placeholders: workload["pkg_placeholder_ids"].append(pkg_placeholder_name_to_id(placeholder_name)) - + for srpm_placeholder_name in srpm_placeholders: workload["srpm_placeholder_names"].append(srpm_placeholder_name) - # 43 % - - workload["pkg_relations"] = self._analyze_package_relations(query_all, package_placeholders) + # Use the actual packages that were installed/selected by DNF's priority logic. + # Don't re-query by name - that would return ALL versions from ALL repos, + # including lower-priority duplicates (e.g., both ELN and Rawhide versions). + workload["pkg_relations"] = self._analyze_package_relations(pkgs_all, package_placeholders) - # 100 % - - pkg_env_count = len(workload["pkg_env_ids"]) - pkg_added_count = len(workload["pkg_added_ids"]) - #log(" Done! ({pkg_count} packages in total. That's {pkg_env_count} in the environment, and {pkg_added_count} added.)".format( - # pkg_count=str(pkg_env_count + pkg_added_count), - # pkg_env_count=pkg_env_count, - # pkg_added_count=pkg_added_count - #)) - #log("") - - # How long do various parts take: - # 37 % - populatind DNF's base.sack - # 6 % - resolving deps - # 57 % - _analyze_package_relations with recommends - - # Removing recommends from _analyze_package_relations - # gets the total duration down to - # 64 % return workload - def _analyze_workload_process(self, queue_result, workload_conf, env_conf, repo, arch): - - workload = self._analyze_workload(workload_conf, env_conf, repo, arch) - queue_result.put(workload) + try: + workload = self._analyze_workload(workload_conf, env_conf, repo, arch) + queue_result.put(workload) + except Exception as e: + # Create a failed workload result instead of crashing + workload = { + "workload_conf_id": workload_conf["id"], + "env_conf_id": env_conf["id"], + "repo_id": repo["id"], + "arch": arch, + "pkg_env_ids": [], + "pkg_added_ids": [], + "pkg_placeholder_ids": [], + "srpm_placeholder_names": [], + "pkg_relations": [], + "errors": { + "non_existing_pkgs": [], + "non_existing_placeholder_deps": [], + "message": f"Workload analysis failed with exception:\n{type(e).__name__}: {str(e)}\n\nTraceback:\n{traceback.format_exc()}", + }, + "warnings": { + "non_existing_pkgs": [], + "non_existing_placeholder_deps": [], + "message": None, + }, + "succeeded": False, + "env_succeeded": False, + "labels": list(set(workload_conf["labels"]) & set(env_conf["labels"])), + } + queue_result.put(workload) + # Log error to stderr so it appears in logs + err_log(f" ERROR analyzing workload {workload_conf['id']}:{env_conf['id']}:{repo['id']}:{arch}-> {e}", file=sys.stderr) async def _analyze_workloads_subset_async(self, task_queue, results): @@ -1326,12 +1644,7 @@ async def _analyze_workloads_subset_async(self, task_queue, results): repo = task["repo"] arch = task["arch"] - workload_id = "{workload_conf_id}:{env_conf_id}:{repo_id}:{arch}".format( - workload_conf_id=workload_conf["id"], - env_conf_id=env_conf["id"], - repo_id=repo["id"], - arch=arch - ) + workload_id = f"{workload_conf['id']}:{env_conf['id']}:{repo['id']}:{arch}" # Max processes while True: @@ -1339,7 +1652,7 @@ async def _analyze_workloads_subset_async(self, task_queue, results): self.current_subprocesses += 1 break else: - await asyncio.sleep(.1) + await asyncio.sleep(0.1) # Log progress self.workload_queue_counter_current += 1 @@ -1359,17 +1672,17 @@ async def _analyze_workloads_subset_async(self, task_queue, results): # 2 seconds for _ in range(1, 20): if queue_result.empty(): - await asyncio.sleep(.1) + await asyncio.sleep(0.1) else: break - + # 20 seconds for _ in range(1, 20): if queue_result.empty(): await asyncio.sleep(1) else: break - + # 200 seconds for _ in range(1, 20): if queue_result.empty(): @@ -1380,12 +1693,13 @@ async def _analyze_workloads_subset_async(self, task_queue, results): self.current_subprocesses -= 1 # This basically means there was an exception in the processing and the process crashed + # or timed out (222 seconds total timeout) if queue_result.empty(): log("") log("") log("--------------------------------------------------------------------------") log("") - log("ERROR: Workload analysis failed") + log("WARNING: Workload analysis timed out or crashed") log("") log("Details:") log(f" workload_conf: {workload_conf['id']}") @@ -1393,16 +1707,40 @@ async def _analyze_workloads_subset_async(self, task_queue, results): log(f" repo: {repo['id']}") log(f" arch: {arch}") log("") - log("More details somewhere above.") + log("Creating failed workload entry and continuing...") log("") log("--------------------------------------------------------------------------") log("") log("") - sys.exit(1) - - workload = queue_result.get() - - results[workload_id] = workload + + # Create a failed workload result instead of crashing + results[workload_id] = { + "workload_conf_id": workload_conf["id"], + "env_conf_id": env_conf["id"], + "repo_id": repo["id"], + "arch": arch, + "pkg_env_ids": [], + "pkg_added_ids": [], + "pkg_placeholder_ids": [], + "srpm_placeholder_names": [], + "pkg_relations": [], + "errors": { + "non_existing_pkgs": [], + "non_existing_placeholder_deps": [], + "message": f"Workload analysis timed out after 222 seconds or subprocess crashed", + }, + "warnings": { + "non_existing_pkgs": [], + "non_existing_placeholder_deps": [], + "message": None, + }, + "succeeded": False, + "env_succeeded": False, + "labels": list(set(workload_conf["labels"]) & set(env_conf["labels"])), + } + else: + workload = queue_result.get() + results[workload_id] = workload async def _analyze_workloads_async(self, results): @@ -1411,24 +1749,21 @@ async def _analyze_workloads_async(self, results): for repo in self.workload_queue: for arch in self.workload_queue[repo]: - task_queue = self.workload_queue[repo][arch] - tasks.append(asyncio.create_task(self._analyze_workloads_subset_async(task_queue, results))) - + for task in tasks: await task log("DONE!") - def _queue_workload_processing(self, workload_conf, env_conf, repo, arch): - + repo_id = repo["id"] if repo_id not in self.workload_queue: self.workload_queue[repo_id] = {} - + if arch not in self.workload_queue[repo_id]: self.workload_queue[repo_id][arch] = [] @@ -1442,13 +1777,11 @@ def _queue_workload_processing(self, workload_conf, env_conf, repo, arch): self.workload_queue[repo_id][arch].append(workload_task) self.workload_queue_counter_total += 1 - def _reset_workload_processing_queue(self): self.workload_queue = {} self.workload_queue_counter_total = 0 self.workload_queue_counter_current = 0 - def _analyze_workloads(self): # Initialise @@ -1468,7 +1801,7 @@ def _analyze_workloads(self): if label in env_conf["labels"]: # And save those. workload_env_map[workload_conf_id].add(env_conf_id) - + # And now, look at all workload configs... for workload_conf_id, workload_conf in self.configs["workloads"].items(): # ... and for each, look at all env configs it should be analyzed in. @@ -1500,28 +1833,18 @@ def _analyze_workloads(self): # Before even started, look if the env succeeded. If not, there's # no point in doing anything here. - env_id = "{env_conf_id}:{repo_id}:{arch}".format( - env_conf_id=env_conf["id"], - repo_id=repo["id"], - arch=arch - ) + env_id = f"{env_conf['id']}:{repo['id']}:{arch}" env = self.data["envs"][env_id] if env["succeeded"]: self._queue_workload_processing(workload_conf, env_conf, repo, arch) else: - workload_id = "{workload_conf_id}:{env_conf_id}:{repo_id}:{arch}".format( - workload_conf_id=workload_conf_id, - env_conf_id=env_conf_id, - repo_id=repo_id, - arch=arch - ) + workload_id = f"{workload_conf_id}:{env_conf_id}:{repo_id}:{arch}" self.data["workloads"][workload_id] = self._return_failed_workload_env_err(workload_conf, env_conf, repo, arch) asyncio.run(self._analyze_workloads_async(self.data["workloads"])) - def _init_view_pkg(self, input_pkg, arch, placeholder=False, level=0): if placeholder: pkg = { @@ -1534,7 +1857,7 @@ def _init_view_pkg(self, input_pkg, arch, placeholder=False, level=0): "description": input_pkg["description"], "summary": input_pkg["description"], "source_name": input_pkg["srpm"], - "sourcerpm": "{}-000-placeholder".format(input_pkg["srpm"]), + "sourcerpm": f"{input_pkg['srpm']}-000-placeholder", "q_arch": input_pkg, "reponame": "n/a", "all_reponames": set(), @@ -1571,6 +1894,7 @@ def _init_view_pkg(self, input_pkg, arch, placeholder=False, level=0): "env": pkg["in_workload_ids_env"], }) + # FIXME: Use comprehension & extend instead on for loop # Level 1 and higher is buildroot for _ in range(level): pkg["level"].append({ @@ -1587,7 +1911,6 @@ def _init_view_pkg(self, input_pkg, arch, placeholder=False, level=0): return pkg - def _init_view_srpm(self, pkg, level=0): srpm_id = pkg["sourcerpm"].rsplit(".src.rpm")[0] @@ -1635,20 +1958,12 @@ def _init_view_srpm(self, pkg, level=0): return srpm - def _analyze_view(self, view_conf, arch, views): view_conf_id = view_conf["id"] - log("Analyzing view: {view_name} ({view_conf_id}) for {arch}".format( - view_name=view_conf["name"], - view_conf_id=view_conf_id, - arch=arch - )) + log(f"Analyzing view: {view_conf['name']} ({view_conf_id}) for {arch}") - view_id = "{view_conf_id}:{arch}".format( - view_conf_id=view_conf_id, - arch=arch - ) + view_id = f"{view_conf_id}:{arch}" repo_id = view_conf["repository"] @@ -1667,7 +1982,7 @@ def _analyze_view(self, view_conf, arch, views): for workload_id, workload in self.data["workloads"].items(): if workload["repo_id"] != repo_id: continue - + if workload["arch"] != arch: continue @@ -1676,22 +1991,29 @@ def _analyze_view(self, view_conf, arch, views): view["workload_ids"].append(workload_id) - log(" Includes {} workloads.".format(len(view["workload_ids"]))) + log(f" Includes {len(view['workload_ids'])} workloads.") # Packages + log(f" Processing packages from {len(view['workload_ids'])} workloads...") + workload_counter = 0 + total_workloads = len(view['workload_ids']) + for workload_id in view["workload_ids"]: + workload_counter += 1 + if workload_counter % 50 == 0: + log(f" Progress: {workload_counter}/{total_workloads} workloads processed") + workload = self.data["workloads"][workload_id] workload_conf_id = workload["workload_conf_id"] workload_conf = self.configs["workloads"][workload_conf_id] # Packages in the environment for pkg_id in workload["pkg_env_ids"]: - # Initialise if pkg_id not in view["pkgs"]: pkg = self.data["pkgs"][repo_id][arch][pkg_id] view["pkgs"][pkg_id] = self._init_view_pkg(pkg, arch) - + # It's in this wokrload view["pkgs"][pkg_id]["in_workload_ids_all"].add(workload_id) @@ -1703,7 +2025,7 @@ def _analyze_view(self, view_conf, arch, views): view["pkgs"][pkg_id]["in_workload_ids_req"].add(workload_id) elif view["pkgs"][pkg_id]["name"] in workload_conf["arch_packages"][arch]: view["pkgs"][pkg_id]["in_workload_ids_req"].add(workload_id) - + # pkg_relations view["pkgs"][pkg_id]["required_by"].update(workload["pkg_relations"][pkg_id]["required_by"]) view["pkgs"][pkg_id]["recommended_by"].update(workload["pkg_relations"][pkg_id]["recommended_by"]) @@ -1712,12 +2034,11 @@ def _analyze_view(self, view_conf, arch, views): # Packages added by this workload (required or dependency) for pkg_id in workload["pkg_added_ids"]: - # Initialise if pkg_id not in view["pkgs"]: pkg = self.data["pkgs"][repo_id][arch][pkg_id] view["pkgs"][pkg_id] = self._init_view_pkg(pkg, arch) - + # It's in this wokrload view["pkgs"][pkg_id]["in_workload_ids_all"].add(workload_id) @@ -1726,11 +2047,11 @@ def _analyze_view(self, view_conf, arch, views): view["pkgs"][pkg_id]["in_workload_ids_req"].add(workload_id) elif view["pkgs"][pkg_id]["name"] in workload_conf["arch_packages"][arch]: view["pkgs"][pkg_id]["in_workload_ids_req"].add(workload_id) - + # Or a dependency? else: view["pkgs"][pkg_id]["in_workload_ids_dep"].add(workload_id) - + # pkg_relations view["pkgs"][pkg_id]["required_by"].update(workload["pkg_relations"][pkg_id]["required_by"]) view["pkgs"][pkg_id]["recommended_by"].update(workload["pkg_relations"][pkg_id]["recommended_by"]) @@ -1739,18 +2060,17 @@ def _analyze_view(self, view_conf, arch, views): # And finally the non-existing, imaginary, package placeholders! for pkg_id in workload["pkg_placeholder_ids"]: - # Initialise if pkg_id not in view["pkgs"]: placeholder = workload_conf["package_placeholders"]["pkgs"][pkg_id_to_name(pkg_id)] view["pkgs"][pkg_id] = self._init_view_pkg(placeholder, arch, placeholder=True) - + # It's in this wokrload view["pkgs"][pkg_id]["in_workload_ids_all"].add(workload_id) # Placeholders are by definition required view["pkgs"][pkg_id]["in_workload_ids_req"].add(workload_id) - + # ... including the SRPM placeholders for srpm_name in workload["srpm_placeholder_names"]: srpm_id = pkg_placeholder_name_to_nevr(srpm_name) @@ -1759,31 +2079,36 @@ def _analyze_view(self, view_conf, arch, views): if srpm_id not in view["source_pkgs"]: sourcerpm = f"{srpm_id}.src.rpm" view["source_pkgs"][srpm_id] = self._init_view_srpm({"sourcerpm": sourcerpm, "source_name": srpm_name, "reponame": None}) - + # It's a placeholder view["source_pkgs"][srpm_id]["placeholder"] = True # Build requires view["source_pkgs"][srpm_id]["placeholder_directly_required_pkg_names"] = workload_conf["package_placeholders"]["srpms"][srpm_name]["buildrequires"] - + # If this is an addon view, remove all packages that are already in the parent view if view_conf["type"] == "addon": base_view_conf_id = view_conf["base_view_id"] - base_view_id = "{base_view_conf_id}:{arch}".format( - base_view_conf_id=base_view_conf_id, - arch=arch - ) + base_view_id = f"{base_view_conf_id}:{arch}" for base_view_pkg_id in views[base_view_id]["pkgs"]: if base_view_pkg_id in view["pkgs"]: del view["pkgs"][base_view_pkg_id] # Done with packages! - log(" Includes {} packages.".format(len(view["pkgs"]))) + log(f" Done processing workload packages. Total packages in view: {len(view['pkgs'])}") # But not with source packages, that's an entirely different story! + log(f" Building source package data from {len(view['pkgs'])} binary packages...") + pkg_counter = 0 + total_pkgs = len(view["pkgs"]) + for pkg_id, pkg in view["pkgs"].items(): + pkg_counter += 1 + if pkg_counter % 1000 == 0: + log(f" Progress: {pkg_counter}/{total_pkgs} packages processed for source mapping") + srpm_id = pkg["sourcerpm"].rsplit(".src.rpm")[0] if srpm_id not in view["source_pkgs"]: @@ -1796,63 +2121,72 @@ def _analyze_view(self, view_conf, arch, views): view["source_pkgs"][srpm_id]["in_workload_ids_req"].update(pkg["in_workload_ids_req"]) view["source_pkgs"][srpm_id]["in_workload_ids_dep"].update(pkg["in_workload_ids_dep"]) view["source_pkgs"][srpm_id]["in_workload_ids_env"].update(pkg["in_workload_ids_env"]) - - log(" Includes {} source packages.".format(len(view["source_pkgs"]))) + log(f" Includes {len(view['source_pkgs'])} source packages.") log(" DONE!") log("") return view - def _analyze_views(self): + log("") + log("===== Starting View Analysis =====") + log("") views = {} # First, analyse the standard views + log("Phase 1: Analyzing standard (compose) views...") + compose_view_count = sum(1 for v in self.configs["views"].values() if v["type"] == "compose") + log(f" Found {compose_view_count} compose views to process") + for view_conf_id in self.configs["views"]: view_conf = self.configs["views"][view_conf_id] if view_conf["type"] == "compose": + log(f" Processing compose view: {view_conf_id}") for arch in view_conf["architectures"]: + log(f" Architecture: {arch}") view = self._analyze_view(view_conf, arch, views) view_id = view["id"] - views[view_id] = view - + log(f" ✅ Completed {view_id}") + + log("") + log("Phase 2: Analyzing addon views...") + addon_view_count = sum(1 for v in self.configs["views"].values() if v["type"] == "addon") + log(f" Found {addon_view_count} addon views to process") + # Second, analyse the addon views # This is important as they need the standard views already available for view_conf_id in self.configs["views"]: view_conf = self.configs["views"][view_conf_id] if view_conf["type"] == "addon": + log(f" Processing addon view: {view_conf_id}") base_view_conf_id = view_conf["base_view_id"] base_view_conf = self.configs["views"][base_view_conf_id] for arch in set(view_conf["architectures"]) & set(base_view_conf["architectures"]): + log(f" Architecture: {arch}") view = self._analyze_view(view_conf, arch, views) view_id = view["id"] - views[view_id] = view - - self.data["views"] = views + log(f" ✅ Completed {view_id}") + + log("") + log(f"View analysis complete! Total views created: {len(views)}") + log("") + self.data["views"] = views def _populate_buildroot_with_view_srpms(self, view_conf, arch): view_conf_id = view_conf["id"] - log("Initialising buildroot packages of: {view_name} ({view_conf_id}) for {arch}".format( - view_name=view_conf["name"], - view_conf_id=view_conf_id, - arch=arch - )) - - view_id = "{view_conf_id}:{arch}".format( - view_conf_id=view_conf_id, - arch=arch - ) + log(f"Initialising buildroot packages of: {view_conf['name']} ({view_conf_id}) for {arch}") + view_id = f"{view_conf_id}:{arch}" view = self.data["views"][view_id] repo_id = view_conf["repository"] @@ -1868,7 +2202,7 @@ def _populate_buildroot_with_view_srpms(self, view_conf, arch): if srpm["placeholder"]: directly_required_pkg_names = srpm["placeholder_directly_required_pkg_names"] - + else: # This is the same set in both koji_srpms and srpms directly_required_pkg_names = set() @@ -1889,7 +2223,7 @@ def _populate_buildroot_with_view_srpms(self, view_conf, arch): self.data["buildroot"]["koji_urls"][koji_id] = {} self.data["buildroot"]["koji_urls"][koji_id]["api"] = koji_api_url self.data["buildroot"]["koji_urls"][koji_id]["files"] = koji_files_url - + if arch not in self.data["buildroot"]["koji_srpms"][koji_id]: self.data["buildroot"]["koji_srpms"][koji_id][arch] = {} @@ -1916,11 +2250,9 @@ def _populate_buildroot_with_view_srpms(self, view_conf, arch): self.data["buildroot"]["srpms"][repo_id][arch][srpm_id]["queued"] = False self.data["buildroot"]["srpms"][repo_id][arch][srpm_id]["processed"] = False - log(" DONE!") log("") - def _resolve_srpms_using_root_logs_parallel(self, pass_counter): """ This function is idempotent! @@ -1971,12 +2303,12 @@ def _resolve_srpms_using_root_logs_parallel(self, pass_counter): # Add to work queue work_items.append({ - 'koji_id': koji_id, - 'koji_api_url': koji_urls["api"], - 'koji_files_url': koji_urls["files"], - 'srpm_id': srpm_id, - 'arch': arch, - 'dev_buildroot': self.settings.get("dev_buildroot", False) + "koji_id": koji_id, + "koji_api_url": koji_urls["api"], + "koji_files_url": koji_urls["files"], + "srpm_id": srpm_id, + "arch": arch, + "dev_buildroot": self.settings.get("dev_buildroot", False), }) if not work_items: @@ -1988,6 +2320,11 @@ def _resolve_srpms_using_root_logs_parallel(self, pass_counter): # Process in parallel using ProcessPoolExecutor max_workers = min(self.settings["parallel_max"], len(work_items)) + # Track failures for reporting + failed_downloads = [] # Root logs that failed to download + zero_deps = [] # Packages with zero dependencies (suspicious) + low_deps = [] # Packages with very few deps (possibly truncated logs) + with ProcessPoolExecutor(max_workers=max_workers) as executor: # Submit all jobs future_to_item = { @@ -2010,35 +2347,105 @@ def _resolve_srpms_using_root_logs_parallel(self, pass_counter): if result['error']: log(f"[ Buildroot - pass {pass_counter} - {completed_count} of {total_count} ] " f"Failed {result['srpm_id']} {result['arch']}: {result['error']}") + failed_downloads.append({ + 'srpm_id': result['srpm_id'], + 'arch': result['arch'], + 'error': result['error'] + }) else: - log(f"[ Buildroot - pass {pass_counter} - {completed_count} of {total_count} ] " - f"Completed {result['srpm_id']} {result['arch']} - found {len(result['deps'])} deps") + # Log completion with warning if present + deps_count = len(result['deps']) + warning_msg = result.get('warning', '') + if warning_msg: + log(f"[ Buildroot - pass {pass_counter} - {completed_count} of {total_count} ] " + f"⚠️ {result['srpm_id']} {result['arch']} - {deps_count} deps - {warning_msg}") + else: + log(f"[ Buildroot - pass {pass_counter} - {completed_count} of {total_count} ] " + f"Completed {result['srpm_id']} {result['arch']} - found {deps_count} deps") + + # Detect suspicious results + if deps_count == 0: + zero_deps.append({ + 'srpm_id': result['srpm_id'], + 'arch': result['arch'] + }) + elif deps_count < 3: + low_deps.append({ + 'srpm_id': result['srpm_id'], + 'arch': result['arch'], + 'deps': result['deps'] + }) except Exception as e: log(f"Failed to process {work_item['srpm_id']}: {e}") # Apply empty result for failed processing error_result = { - 'srpm_id': work_item['srpm_id'], - 'arch': work_item['arch'], - 'deps': [], - 'error': str(e) + "srpm_id": work_item["srpm_id"], + "arch": work_item["arch"], + "deps": [], + "error": str(e), } self._apply_srpm_result(work_item, error_result) + failed_downloads.append({ + 'srpm_id': work_item["srpm_id"], + 'arch': work_item["arch"], + 'error': str(e) + }) # Save updated cache dump_data(self.settings["root_log_deps_cache_path"], self.cache["root_log_deps"]["next"]) + # Report summary of issues log("") - log(" DONE!") + log("=" * 80) + log("ROOT LOG PROCESSING SUMMARY") + log("=" * 80) + + if failed_downloads: + log(f"⚠️ FAILED DOWNLOADS: {len(failed_downloads)} root.log files failed to download/parse") + log("") + for item in failed_downloads[:20]: # Show first 20 + log(f" ❌ {item['srpm_id']} ({item['arch']})") + log(f" Error: {item['error'][:100]}") + if len(failed_downloads) > 20: + log(f" ... and {len(failed_downloads) - 20} more") + + if zero_deps: + log("") + log(f"⚠️ ZERO DEPENDENCIES: {len(zero_deps)} packages reported 0 build dependencies") + log(" (This is unusual - most packages have at least bash, gcc, etc.)") + log("") + for item in zero_deps[:20]: + log(f" ⚠️ {item['srpm_id']} ({item['arch']})") + if len(zero_deps) > 20: + log(f" ... and {len(zero_deps) - 20} more") + + if low_deps: + log("") + log(f"⚠️ LOW DEPENDENCIES: {len(low_deps)} packages have very few dependencies (<3)") + log(" (Possibly truncated or corrupted root.log files)") + log("") + for item in low_deps[:10]: + log(f" ⚠️ {item['srpm_id']} ({item['arch']}): {item['deps']}") + if len(low_deps) > 10: + log(f" ... and {len(low_deps) - 10} more") + + if not failed_downloads and not zero_deps and not low_deps: + log("✅ All root.log files processed successfully with reasonable dependency counts") + + log("=" * 80) log("") + log("") + log(" DONE!") + log("") def _apply_srpm_result(self, work_item, result): """Apply worker result back to main data structures""" - koji_id = work_item['koji_id'] - arch = work_item['arch'] - srpm_id = work_item['srpm_id'] - deps = result['deps'] + koji_id = work_item["koji_id"] + arch = work_item["arch"] + srpm_id = work_item["srpm_id"] + deps = result["deps"] # Update cache self.cache["root_log_deps"]["next"][koji_id][arch][srpm_id] = deps @@ -2048,7 +2455,6 @@ def _apply_srpm_result(self, work_item, result): # set, because its reference is shared between the koji_srpms and the srpm sections self.data["buildroot"]["koji_srpms"][koji_id][arch][srpm_id]["directly_required_pkg_names"].update(deps) - def _analyze_build_groups(self): log("") @@ -2062,12 +2468,9 @@ def _analyze_build_groups(self): for arch in self.data["buildroot"]["srpms"][repo_id]: - generated_id = "CR-buildroot-base-env-{repo_id}-{arch}".format( - repo_id=repo_id, - arch=arch - ) + generated_id = f"CR-buildroot-base-env-{repo_id}-{arch}" - # Using the _analyze_env function! + # Using the _analyze_env function! # So I need to reconstruct a fake env_conf fake_env_conf = {} fake_env_conf["id"] = generated_id @@ -2081,10 +2484,7 @@ def _analyze_build_groups(self): fake_env_conf["arch_packages"] = {} fake_env_conf["arch_packages"][arch] = [] - log("Resolving build group: {repo_id} {arch}".format( - repo_id=repo_id, - arch=arch - )) + log(f"Resolving build group: {repo_id} {arch}") repo = self.configs["repos"][repo_id] fake_env = self._analyze_env(fake_env_conf, repo, arch) @@ -2103,7 +2503,6 @@ def _analyze_build_groups(self): def _expand_buildroot_srpms(self): # This function is idempotent! - # # That means it can be run many times without affecting the old results. log("Expanding the SRPM set...") @@ -2124,7 +2523,7 @@ def _expand_buildroot_srpms(self): # Adding a new one! counter += 1 - + srpm_reponame = self.data["pkgs"][repo_id][arch][pkg_id]["reponame"] # This is the same set in both koji_srpms and srpms @@ -2165,7 +2564,6 @@ def _expand_buildroot_srpms(self): def _analyze_srpm_buildroots(self, pass_counter): # This function is idempotent! - # # That means it can be run many times without affecting the old results. log("") @@ -2215,20 +2613,12 @@ def _analyze_srpm_buildroots(self, pass_counter): fake_env_conf["arch_packages"][arch] = [] srpms_to_resolve_counter += 1 - - #log("[ Buildroot - pass {} - {} of {} ]".format(pass_counter, srpms_to_resolve_counter, total_srpms_to_resolve)) - #log("Resolving SRPM buildroot: {repo_id} {arch} {srpm_id}".format( - # repo_id=repo_id, - # arch=arch, - # srpm_id=srpm_id - #)) repo = self.configs["repos"][repo_id] - - #fake_workload = self._analyze_workload(fake_workload_conf, fake_env_conf, repo, arch) self._queue_workload_processing(fake_workload_conf, fake_env_conf, repo, arch) # Save the buildroot data self.data["buildroot"]["srpms"][repo_id][arch][srpm_id]["queued"] = True + log(f"fake_workload_results -> {fake_workload_results}") asyncio.run(self._analyze_workloads_async(fake_workload_results)) @@ -2239,12 +2629,7 @@ def _analyze_srpm_buildroots(self, pass_counter): if srpm["processed"]: continue - fake_workload_id = "{workload_conf_id}:{env_conf_id}:{repo_id}:{arch}".format( - workload_conf_id=srpm_id, - env_conf_id=self.data["buildroot"]["build_groups"][repo_id][arch]["generated_id"], - repo_id=repo_id, - arch=arch - ) + fake_workload_id = f"{srpm_id}:{self.data['buildroot']['build_groups'][repo_id][arch]['generated_id']}:{repo_id}:{arch}" fake_workload = fake_workload_results[fake_workload_id] @@ -2292,7 +2677,6 @@ def _analyze_buildroot(self): self._populate_buildroot_with_view_srpms(view_conf, arch) # Time to resolve the build groups! - # # This initialises and populates: # buildroot["build_groups"] self._analyze_build_groups() @@ -2308,7 +2692,6 @@ def _analyze_buildroot(self): log("") log("") # Get the directly_required_pkg_names from koji root logs - # # Adds stuff to existing: # data["buildroot"]["koji_srpms"]... # ... which also updates: @@ -2335,9 +2718,8 @@ def _analyze_buildroot(self): log("") break - def _add_missing_levels_to_pkg_or_srpm(self, pkg_or_srpm, level): - + """Add missing level entries to reach the target level.""" pkg_current_max_level = len(pkg_or_srpm["level"]) - 1 for _ in range(level - pkg_current_max_level): pkg_or_srpm["level"].append({ @@ -2352,16 +2734,12 @@ def _add_buildroot_to_view(self, view_conf, arch): view_conf_id = view_conf["id"] - view_id = "{view_conf_id}:{arch}".format( - view_conf_id=view_conf_id, - arch=arch - ) + view_id = f"{view_conf_id}:{arch}" repo_id = view_conf["repository"] view = self.data["views"][view_id] - log("") log(f"Adding buildroot to view {view_id}...") @@ -2382,7 +2760,6 @@ def _add_buildroot_to_view(self, view_conf, arch): for buildroot_srpm_id in srpm_ids_to_process: buildroot_srpm = self.data["buildroot"]["srpms"][repo_id][arch][buildroot_srpm_id] - # Packages in the base buildroot (which would be the environment in workloads) for pkg_id in buildroot_srpm["pkg_env_ids"]: added_pkg_ids.add(pkg_id) @@ -2391,7 +2768,7 @@ def _add_buildroot_to_view(self, view_conf, arch): if pkg_id not in view["pkgs"]: pkg = self.data["pkgs"][repo_id][arch][pkg_id] view["pkgs"][pkg_id] = self._init_view_pkg(pkg, arch, level=level) - + # Add missing levels to the pkg self._add_missing_levels_to_pkg_or_srpm(view["pkgs"][pkg_id], level) @@ -2407,7 +2784,7 @@ def _add_buildroot_to_view(self, view_conf, arch): if view["pkgs"][pkg_id]["name"] in buildroot_srpm["directly_required_pkg_names"]: view["pkgs"][pkg_id]["in_buildroot_of_srpm_id_req"].add(buildroot_srpm_id) view["pkgs"][pkg_id]["level"][level]["req"].add(buildroot_srpm_id) - + # pkg_relations view["pkgs"][pkg_id]["required_by"].update(buildroot_srpm["pkg_relations"][pkg_id]["required_by"]) view["pkgs"][pkg_id]["recommended_by"].update(buildroot_srpm["pkg_relations"][pkg_id]["recommended_by"]) @@ -2422,10 +2799,10 @@ def _add_buildroot_to_view(self, view_conf, arch): if pkg_id not in view["pkgs"]: pkg = self.data["pkgs"][repo_id][arch][pkg_id] view["pkgs"][pkg_id] = self._init_view_pkg(pkg, arch, level=level) - + # Add missing levels to the pkg self._add_missing_levels_to_pkg_or_srpm(view["pkgs"][pkg_id], level) - + # It's in this buildroot view["pkgs"][pkg_id]["in_buildroot_of_srpm_id_all"].add(buildroot_srpm_id) view["pkgs"][pkg_id]["level"][level]["all"].add(buildroot_srpm_id) @@ -2434,18 +2811,18 @@ def _add_buildroot_to_view(self, view_conf, arch): if view["pkgs"][pkg_id]["name"] in buildroot_srpm["directly_required_pkg_names"]: view["pkgs"][pkg_id]["in_buildroot_of_srpm_id_req"].add(buildroot_srpm_id) view["pkgs"][pkg_id]["level"][level]["req"].add(buildroot_srpm_id) - + # Or a dependency? else: view["pkgs"][pkg_id]["in_buildroot_of_srpm_id_dep"].add(buildroot_srpm_id) view["pkgs"][pkg_id]["level"][level]["dep"].add(buildroot_srpm_id) - + # pkg_relations view["pkgs"][pkg_id]["required_by"].update(buildroot_srpm["pkg_relations"][pkg_id]["required_by"]) view["pkgs"][pkg_id]["recommended_by"].update(buildroot_srpm["pkg_relations"][pkg_id]["recommended_by"]) view["pkgs"][pkg_id]["suggested_by"].update(buildroot_srpm["pkg_relations"][pkg_id]["suggested_by"]) view["pkgs"][pkg_id]["supplements"].update(buildroot_srpm["pkg_relations"][pkg_id]["supplements"]) - + # Resetting the SRPMs, so only the new ones can be added srpm_ids_to_process = set() @@ -2458,7 +2835,7 @@ def _add_buildroot_to_view(self, view_conf, arch): if srpm_id not in view["source_pkgs"]: view["source_pkgs"][srpm_id] = self._init_view_srpm(pkg, level=level) srpm_ids_to_process.add(srpm_id) - + # Add missing levels to the pkg self._add_missing_levels_to_pkg_or_srpm(view["source_pkgs"][srpm_id], level) @@ -2474,8 +2851,8 @@ def _add_buildroot_to_view(self, view_conf, arch): view["source_pkgs"][srpm_id]["level"][level]["dep"].update(pkg["level"][level]["dep"]) view["source_pkgs"][srpm_id]["level"][level]["env"].update(pkg["level"][level]["env"]) - log (f" added {len(added_pkg_ids)} RPMs") - log (f" added {len(srpm_ids_to_process)} SRPMs") + log(f" added {len(added_pkg_ids)} RPMs") + log(f" added {len(srpm_ids_to_process)} SRPMs") # More iterations needed? if not srpm_ids_to_process: @@ -2483,7 +2860,6 @@ def _add_buildroot_to_view(self, view_conf, arch): log("") break - def _add_buildroot_to_views(self): log("") @@ -2505,8 +2881,7 @@ def _add_buildroot_to_views(self): log(" DONE!") log("") - - def _init_pkg_or_srpm_relations_fields(self, target_pkg, type = None): + def _init_pkg_or_srpm_relations_fields(self, target_pkg, type=None): # I kept them all listed so they're easy to copy # Workload IDs @@ -2514,7 +2889,7 @@ def _init_pkg_or_srpm_relations_fields(self, target_pkg, type = None): target_pkg["in_workload_ids_req"] = set() target_pkg["in_workload_ids_dep"] = set() target_pkg["in_workload_ids_env"] = set() - + # Workload Conf IDs target_pkg["in_workload_conf_ids_all"] = set() target_pkg["in_workload_conf_ids_req"] = set() @@ -2528,10 +2903,10 @@ def _init_pkg_or_srpm_relations_fields(self, target_pkg, type = None): target_pkg["in_buildroot_of_srpm_id_env"] = set() # Buildroot SRPM Names - target_pkg["in_buildroot_of_srpm_name_all"] = {} # of set() of srpm_ids - target_pkg["in_buildroot_of_srpm_name_req"] = {} # of set() of srpm_ids - target_pkg["in_buildroot_of_srpm_name_dep"] = {} # of set() of srpm_ids - target_pkg["in_buildroot_of_srpm_name_env"] = {} # of set() of srpm_ids + target_pkg["in_buildroot_of_srpm_name_all"] = {} # of set() of srpm_ids + target_pkg["in_buildroot_of_srpm_name_req"] = {} # of set() of srpm_ids + target_pkg["in_buildroot_of_srpm_name_dep"] = {} # of set() of srpm_ids + target_pkg["in_buildroot_of_srpm_name_env"] = {} # of set() of srpm_ids # Unwanted target_pkg["unwanted_completely_in_list_ids"] = set() @@ -2556,14 +2931,13 @@ def _init_pkg_or_srpm_relations_fields(self, target_pkg, type = None): target_pkg["weak_dependency_of_pkg_nevrs"] = set() target_pkg["reverse_weak_dependency_of_pkg_nevrs"] = set() - # Dependency of RPM Names target_pkg["dependency_of_pkg_names"] = {} # of set() of nevrs target_pkg["hard_dependency_of_pkg_names"] = {} # of set() of nevrs target_pkg["weak_dependency_of_pkg_names"] = {} # if set() of nevrs target_pkg["reverse_weak_dependency_of_pkg_names"] = {} # if set() of nevrs - def _populate_pkg_or_srpm_relations_fields(self, target_pkg, source_pkg, type = None, view = None): + def _populate_pkg_or_srpm_relations_fields(self, target_pkg, source_pkg, type=None, view=None): # source_pkg is the arch-specific binary package # target_pkg is a representation of that pages for all arches @@ -2579,7 +2953,6 @@ def _populate_pkg_or_srpm_relations_fields(self, target_pkg, source_pkg, type = target_pkg["unwanted_completely_in_list_ids"].update(source_pkg["unwanted_completely_in_list_ids"]) target_pkg["unwanted_buildroot_in_list_ids"].update(source_pkg["unwanted_buildroot_in_list_ids"]) - # Dependency relationships for list_type in ["all", "req", "dep", "env"]: target_pkg[f"in_workload_ids_{list_type}"].update(source_pkg[f"in_workload_ids_{list_type}"]) @@ -2595,9 +2968,9 @@ def _populate_pkg_or_srpm_relations_fields(self, target_pkg, source_pkg, type = if srpm_name not in target_pkg[f"in_buildroot_of_srpm_name_{list_type}"]: target_pkg[f"in_buildroot_of_srpm_name_{list_type}"][srpm_name] = set() - + target_pkg[f"in_buildroot_of_srpm_name_{list_type}"][srpm_name].add(srpm_id) - + # Level number level_number = 0 for level in source_pkg["level"]: @@ -2611,7 +2984,7 @@ def _populate_pkg_or_srpm_relations_fields(self, target_pkg, source_pkg, type = for level_data in source_pkg["level"]: # 'level' is the number # 'level_data' is the ["all"][workload_id] or ["all"][srpm_id] or - # ["req"][workload_id] or ["req"][srpm_id] or + # ["req"][workload_id] or ["req"][srpm_id] or # ["dep"][workload_id] or ["dep"][srpm_id] or # ["env"][workload_id] or ["env"][srpm_id] @@ -2626,24 +2999,23 @@ def _populate_pkg_or_srpm_relations_fields(self, target_pkg, source_pkg, type = if level_scope not in target_pkg["level"][level]: target_pkg["level"][level][level_scope] = set() - + target_pkg["level"][level][level_scope].update(those_ids) - - level +=1 - - + + level += 1 + if type == "rpm": # Hard dependency of for pkg_id in source_pkg["required_by"]: pkg_name = pkg_id_to_name(pkg_id) - + # This only happens in addon views, and only rarely. # Basically means that a package in the addon view is required # by a package in the base view. # Doesn't make sense? # Think of 'glibc-all-langpacks' being in the addon, # while the proper langpacks along with 'glibc' are in the base view. - # + # # In that case, 'glibc' is not in the addon, but 'glibc-all-langpacks' # requires it. # @@ -2658,10 +3030,7 @@ def _populate_pkg_or_srpm_relations_fields(self, target_pkg, source_pkg, type = continue pkg = view["pkgs"][pkg_id] - pkg_nevr = "{name}-{evr}".format( - name=pkg["name"], - evr=pkg["evr"] - ) + pkg_nevr = f"{pkg['name']}-{pkg['evr']}" target_pkg["hard_dependency_of_pkg_nevrs"].add(pkg_nevr) if pkg_name not in target_pkg["hard_dependency_of_pkg_names"]: @@ -2682,10 +3051,7 @@ def _populate_pkg_or_srpm_relations_fields(self, target_pkg, source_pkg, type = continue pkg = view["pkgs"][pkg_id] - pkg_nevr = "{name}-{evr}".format( - name=pkg["name"], - evr=pkg["evr"] - ) + pkg_nevr = f"{pkg['name']}-{pkg['evr']}" target_pkg["weak_dependency_of_pkg_nevrs"].add(pkg_nevr) if pkg_name not in target_pkg["weak_dependency_of_pkg_names"]: @@ -2705,16 +3071,13 @@ def _populate_pkg_or_srpm_relations_fields(self, target_pkg, source_pkg, type = continue pkg = view["pkgs"][pkg_id] - pkg_nevr = "{name}-{evr}".format( - name=pkg["name"], - evr=pkg["evr"] - ) + pkg_nevr = f"{pkg['name']}-{pkg['evr']}" target_pkg["reverse_weak_dependency_of_pkg_nevrs"].add(pkg_nevr) if pkg_name not in target_pkg["reverse_weak_dependency_of_pkg_names"]: target_pkg["reverse_weak_dependency_of_pkg_names"][pkg_name] = set() target_pkg["reverse_weak_dependency_of_pkg_names"][pkg_name].add(pkg_nevr) - + # All types of dependency target_pkg["dependency_of_pkg_nevrs"].update(target_pkg["hard_dependency_of_pkg_nevrs"]) target_pkg["dependency_of_pkg_nevrs"].update(target_pkg["weak_dependency_of_pkg_nevrs"]) @@ -2723,13 +3086,13 @@ def _populate_pkg_or_srpm_relations_fields(self, target_pkg, source_pkg, type = for pkg_name, pkg_nevrs in target_pkg["hard_dependency_of_pkg_names"].items(): if pkg_name not in target_pkg["dependency_of_pkg_names"]: target_pkg["dependency_of_pkg_names"][pkg_name] = set() - + target_pkg["dependency_of_pkg_names"][pkg_name].update(pkg_nevrs) for pkg_name, pkg_nevrs in target_pkg["weak_dependency_of_pkg_names"].items(): if pkg_name not in target_pkg["dependency_of_pkg_names"]: target_pkg["dependency_of_pkg_names"][pkg_name] = set() - + target_pkg["dependency_of_pkg_names"][pkg_name].update(pkg_nevrs) for pkg_name, pkg_nevrs in target_pkg["reverse_weak_dependency_of_pkg_names"].items(): @@ -2737,11 +3100,9 @@ def _populate_pkg_or_srpm_relations_fields(self, target_pkg, source_pkg, type = target_pkg["dependency_of_pkg_names"][pkg_name] = set() target_pkg["dependency_of_pkg_names"][pkg_name].update(pkg_nevrs) - # TODO: add the levels - def _generate_views_all_arches(self): views_all_arches = {} @@ -2750,7 +3111,6 @@ def _generate_views_all_arches(self): #if view_conf["type"] == "compose": if True: - repo_id = view_conf["repository"] view_all_arches = {} @@ -2794,13 +3154,8 @@ def _generate_views_all_arches(self): view_all_arches["numbers"]["srpms"]["build_level_1"] = 0 view_all_arches["numbers"]["srpms"]["build_level_2_plus"] = 0 - for arch in view_conf["architectures"]: - view_id = "{view_conf_id}:{arch}".format( - view_conf_id=view_conf_id, - arch=arch - ) - + view_id = f"{view_conf_id}:{arch}" view = self.data["views"][view_id] # Workloads @@ -2817,19 +3172,17 @@ def _generate_views_all_arches(self): view_all_arches["workloads"][workload_conf_id]["succeeded"] = True view_all_arches["workloads"][workload_conf_id]["no_warnings"] = True # ... - + if not workload["succeeded"]: view_all_arches["workloads"][workload_conf_id]["succeeded"] = False view_all_arches["everything_succeeded"] = False - + if workload["warnings"]["message"]: view_all_arches["workloads"][workload_conf_id]["no_warnings"] = False view_all_arches["no_warnings"] = False - # Binary Packages for package in view["pkgs"].values(): - # Binary Packages by name key = "pkgs_by_name" identifier = package["name"] @@ -2875,7 +3228,7 @@ def _generate_views_all_arches(self): view_all_arches[key][identifier]["category"] = None self._init_pkg_or_srpm_relations_fields(view_all_arches[key][identifier], type="rpm") - + view_all_arches[key][identifier]["arches"].add(arch) view_all_arches[key][identifier]["reponame_per_arch"][arch] = package["reponame"] view_all_arches[key][identifier]["highest_priority_reponames_per_arch"][arch] = package["highest_priority_reponames"] @@ -2886,10 +3239,9 @@ def _generate_views_all_arches(self): self._populate_pkg_or_srpm_relations_fields(view_all_arches[key][identifier], package, type="rpm", view=view) - + # Source Packages for package in view["source_pkgs"].values(): - # Source Packages by name key = "source_pkgs_by_name" identifier = package["name"] @@ -2909,7 +3261,6 @@ def _generate_views_all_arches(self): view_all_arches[key][identifier]["category"] = None self._init_pkg_or_srpm_relations_fields(view_all_arches[key][identifier]) - if view_all_arches["has_buildroot"]: if not self.data["buildroot"]["srpms"][repo_id][arch][package["id"]]["succeeded"]: @@ -2921,11 +3272,11 @@ def _generate_views_all_arches(self): view_all_arches[key][identifier]["buildroot_no_warnings"] = False view_all_arches[key][identifier]["warnings"][arch] = self.data["buildroot"]["srpms"][repo_id][arch][package["id"]]["warnings"] - + view_all_arches[key][identifier]["arches"].add(arch) self._populate_pkg_or_srpm_relations_fields(view_all_arches[key][identifier], package, type="srpm") - + # Add binary packages to source packages for pkg_id, pkg in view["pkgs"].items(): @@ -2936,13 +3287,8 @@ def _generate_views_all_arches(self): view_all_arches["source_pkgs_by_name"][source_name]["pkg_names"].add(pkg["name"]) # Add package nevrs - pkg_nevr = "{name}-{evr}".format( - name=pkg["name"], - evr=pkg["evr"] - ) + pkg_nevr = f"{pkg['name']}-{pkg['evr']}" view_all_arches["source_pkgs_by_name"][source_name]["pkg_nevrs"].add(pkg_nevr) - - # RPMs for pkg in view_all_arches["pkgs_by_nevr"].values(): @@ -2960,12 +3306,19 @@ def _generate_views_all_arches(self): category = "build_level_1" elif pkg["level_number"] > 1: category = "build_level_2_plus" - - view_all_arches["numbers"]["pkgs"][category] += 1 - - view_all_arches["numbers"]["pkgs"]["runtime"] = view_all_arches["numbers"]["pkgs"]["env"] + view_all_arches["numbers"]["pkgs"]["req"] + view_all_arches["numbers"]["pkgs"]["dep"] - view_all_arches["numbers"]["pkgs"]["build"] = view_all_arches["numbers"]["pkgs"]["build_base"] + view_all_arches["numbers"]["pkgs"]["build_level_1"] + view_all_arches["numbers"]["pkgs"]["build_level_2_plus"] - + + if category is not None: + view_all_arches["numbers"]["pkgs"][category] += 1 + + view_all_arches["numbers"]["pkgs"]["runtime"] = \ + view_all_arches["numbers"]["pkgs"]["env"] + \ + view_all_arches["numbers"]["pkgs"]["req"] + \ + view_all_arches["numbers"]["pkgs"]["dep"] + view_all_arches["numbers"]["pkgs"]["build"] = \ + view_all_arches["numbers"]["pkgs"]["build_base"] + \ + view_all_arches["numbers"]["pkgs"]["build_level_1"] + \ + view_all_arches["numbers"]["pkgs"]["build_level_2_plus"] + # SRPMs for pkg in view_all_arches["source_pkgs_by_name"].values(): category = None @@ -2982,9 +3335,10 @@ def _generate_views_all_arches(self): category = "build_level_1" elif pkg["level_number"] > 1: category = "build_level_2_plus" - - view_all_arches["numbers"]["srpms"][category] += 1 - + + if category is not None: + view_all_arches["numbers"]["srpms"][category] += 1 + view_all_arches["numbers"]["srpms"]["runtime"] = \ view_all_arches["numbers"]["srpms"]["env"] + \ view_all_arches["numbers"]["srpms"]["req"] + \ @@ -2995,15 +3349,10 @@ def _generate_views_all_arches(self): view_all_arches["numbers"]["srpms"]["build_level_1"] + \ view_all_arches["numbers"]["srpms"]["build_level_2_plus"] - - - - # Done views_all_arches[view_conf_id] = view_all_arches - - self.data["views_all_arches"] = views_all_arches + self.data["views_all_arches"] = views_all_arches def _add_unwanted_packages_to_view(self, view, view_conf): @@ -3016,7 +3365,7 @@ def _add_unwanted_packages_to_view(self, view, view_conf): for unwanted_label in unwanted["labels"]: if view_label == unwanted_label: unwanted_conf_ids.add(unwanted_conf_id) - + # Dicts pkgs_unwanted_buildroot = {} pkgs_unwanted_completely = {} @@ -3057,7 +3406,7 @@ def _add_unwanted_packages_to_view(self, view, view_conf): if srpm_name in srpms_unwanted_completely: list_ids = srpms_unwanted_completely[srpm_name] view["pkgs"][pkg_id]["unwanted_completely_in_list_ids"].update(list_ids) - + # Add it to the srpms for srpm_id, srpm in view["source_pkgs"].items(): srpm_name = srpm["name"] @@ -3066,7 +3415,6 @@ def _add_unwanted_packages_to_view(self, view, view_conf): list_ids = srpms_unwanted_completely[srpm_name] view["source_pkgs"][srpm_id]["unwanted_completely_in_list_ids"].update(list_ids) - def _add_unwanted_packages_to_views(self): log("") @@ -3081,11 +3429,7 @@ def _add_unwanted_packages_to_views(self): if view_conf["buildroot_strategy"] == "root_logs": for arch in view_conf["architectures"]: - view_id = "{view_conf_id}:{arch}".format( - view_conf_id=view_conf_id, - arch=arch - ) - + view_id = f"{view_conf_id}:{arch}" view = self.data["views"][view_id] self._add_unwanted_packages_to_view(view, view_conf) @@ -3110,13 +3454,12 @@ def _recommend_maintainers(self): # ... etc # # I'll call a combination of these a _score_ because I can't think of - # anything better at this point. It's a tuple! - # + # anything better at this point. It's a tuple! + # # (0, 0) # | '-- sub-level 0 == explicitly required # '---- level 0 == runtime - # - + # for view_conf_id in self.configs["views"]: view_conf = self.configs["views"][view_conf_id] @@ -3145,7 +3488,7 @@ def _recommend_maintainers(self): # Take all explicitly required packages and assign them # to the maintainer of their workloads. # - # Or of this is the buildroot levels, + # Or of this is the buildroot levels, for pkg_name, pkg in view_all_arches["pkgs_by_name"].items(): source_name = pkg["source_name"] @@ -3160,43 +3503,32 @@ def _recommend_maintainers(self): # 1/ maintainer_recommendation if workload_maintainer not in pkg["maintainer_recommendation"]: - #pkg["maintainer_recommendation"][workload_maintainer] = set() self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation"][workload_maintainer] = set() - - #pkg["maintainer_recommendation"][workload_maintainer].add(score) + self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation"][workload_maintainer].add(score) # 2/ maintainer_recommendation_details if level not in pkg["maintainer_recommendation_details"]: - #pkg["maintainer_recommendation_details"][level] = {} self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation_details"][level] = {} - + if sublevel not in pkg["maintainer_recommendation_details"][level]: - #pkg["maintainer_recommendation_details"][level][sublevel] = {} self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation_details"][level][sublevel] = {} - + if workload_maintainer not in pkg["maintainer_recommendation_details"][level][sublevel]: - #pkg["maintainer_recommendation_details"][level][sublevel][workload_maintainer] = {} - #pkg["maintainer_recommendation_details"][level][sublevel][workload_maintainer]["reasons"] = {} - #pkg["maintainer_recommendation_details"][level][sublevel][workload_maintainer]["locations"] = {} self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation_details"][level][sublevel][workload_maintainer] = {} self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation_details"][level][sublevel][workload_maintainer]["reasons"] = set() self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation_details"][level][sublevel][workload_maintainer]["locations"] = set() - #pkg["maintainer_recommendation_details"][level][sublevel][workload_maintainer]["locations"].add(workload_conf_id) self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation_details"][level][sublevel][workload_maintainer]["locations"].add(workload_conf_id) # Lie to the while loop so it runs at least once level_changes_made = True level_change_detection = set() - while level_changes_made: - # Level 1 and higher if int(level) > 0: - level_changes_made = False log(f" {score}") @@ -3242,31 +3574,23 @@ def _recommend_maintainers(self): # 1/ maintainer_recommendation if buildroot_srpm_maintainer not in pkg["maintainer_recommendation"]: - #pkg["maintainer_recommendation"][workload_maintainer] = set() self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation"][buildroot_srpm_maintainer] = set() - #pkg["maintainer_recommendation"][buildroot_srpm_maintainer].add(score) self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation"][buildroot_srpm_maintainer].add(score) # 2/ maintainer_recommendation_details if level not in pkg["maintainer_recommendation_details"]: - #pkg["maintainer_recommendation_details"][level] = {} self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation_details"][level] = {} - + if sublevel not in pkg["maintainer_recommendation_details"][level]: - #pkg["maintainer_recommendation_details"][level][sublevel] = {} self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation_details"][level][sublevel] = {} - + if buildroot_srpm_maintainer not in pkg["maintainer_recommendation_details"][level][sublevel]: - #pkg["maintainer_recommendation_details"][level][sublevel][buildroot_srpm_maintainer] = {} - #pkg["maintainer_recommendation_details"][level][sublevel][buildroot_srpm_maintainer]["reasons"] = {} - #pkg["maintainer_recommendation_details"][level][sublevel][buildroot_srpm_maintainer]["locations"] = {} self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation_details"][level][sublevel][buildroot_srpm_maintainer] = {} self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation_details"][level][sublevel][buildroot_srpm_maintainer]["reasons"] = set() self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation_details"][level][sublevel][buildroot_srpm_maintainer]["locations"] = set() - #pkg["maintainer_recommendation_details"][level][sublevel][buildroot_srpm_maintainer]["locations"].add(buildroot_srpm_name) self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation_details"][level][sublevel][buildroot_srpm_maintainer]["locations"].add(buildroot_srpm_name) @@ -3318,30 +3642,23 @@ def _recommend_maintainers(self): else: sublevel_changes_made = True sublevel_change_detection.add(sublevel_change_detection_tuple) - + # 1/ maintainer_recommendation if superior_pkg_maintainer not in pkg["maintainer_recommendation"]: - #pkg["maintainer_recommendation"][workload_maintainer] = set() self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation"][superior_pkg_maintainer] = set() - #pkg["maintainer_recommendation"][superior_pkg_maintainer].add(score) self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation"][superior_pkg_maintainer].add(score) # 2/ maintainer_recommendation_details if level not in pkg["maintainer_recommendation_details"]: - #pkg["maintainer_recommendation_details"][level] = {} self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation_details"][level] = {} - + if sublevel not in pkg["maintainer_recommendation_details"][level]: - #pkg["maintainer_recommendation_details"][level][sublevel] = {} self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation_details"][level][sublevel] = {} - + if superior_pkg_maintainer not in pkg["maintainer_recommendation_details"][level][sublevel]: - #pkg["maintainer_recommendation_details"][level][sublevel][superior_pkg_maintainer] = {} - #pkg["maintainer_recommendation_details"][level][sublevel][superior_pkg_maintainer]["reasons"] = {} - #pkg["maintainer_recommendation_details"][level][sublevel][superior_pkg_maintainer]["locations"] = {} self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation_details"][level][sublevel][superior_pkg_maintainer] = {} self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation_details"][level][sublevel][superior_pkg_maintainer]["reasons"] = set() self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation_details"][level][sublevel][superior_pkg_maintainer]["locations"] = set() @@ -3351,10 +3668,9 @@ def _recommend_maintainers(self): self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation_details"][level][sublevel][superior_pkg_maintainer]["locations"].update(locations) reason = (superior_pkg_name, superior_srpm_name, pkg_name) - #pkg["maintainer_recommendation_details"][level][sublevel][superior_pkg_maintainer]["reasons"].add(reason) self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation_details"][level][sublevel][superior_pkg_maintainer]["reasons"].add(reason) - + # Now add this info to the source packages for pkg_name, pkg in view_all_arches["pkgs_by_name"].items(): source_name = pkg["source_name"] @@ -3365,13 +3681,13 @@ def _recommend_maintainers(self): if maintainer not in self.data["views_all_arches"][view_conf_id]["source_pkgs_by_name"][source_name]["maintainer_recommendation"]: self.data["views_all_arches"][view_conf_id]["source_pkgs_by_name"][source_name]["maintainer_recommendation"][maintainer] = set() - + self.data["views_all_arches"][view_conf_id]["source_pkgs_by_name"][source_name]["maintainer_recommendation"][maintainer].update(maintainer_scores) # Add it here so it's not processed again in the another level this_level_srpms.add(source_name) - + # 2/ maintainer_recommendation_details for loop_level, loop_sublevels in pkg["maintainer_recommendation_details"].items(): @@ -3403,14 +3719,12 @@ def _recommend_maintainers(self): # And set stuff for the next level prev_level = level level = str(int(level) + 1) - #level += 1 + # level += 1 sublevel = str(0) score = (level, sublevel) previous_level_srpms.update(this_level_srpms) this_level_srpms = set() - - # And elect the best owners for each srpm for source_name, srpm in view_all_arches["source_pkgs_by_name"].items(): @@ -3455,14 +3769,10 @@ def _recommend_maintainers(self): self.data["views_all_arches"][view_conf_id]["source_pkgs_by_name"][source_name]["best_maintainers"].update(best_maintainers) - - - log("") log(" DONE!") log("") - def analyze_things(self): log("") log("###############################################################################") @@ -3485,9 +3795,6 @@ def analyze_things(self): self.tmp_dnf_cachedir = os.path.join(tmp, "dnf_cachedir") self.tmp_installroots = os.path.join(tmp, "installroots") - # List of supported arches - all_arches = self.settings["allowed_arches"] - # Repos log("") log("===== Analyzing Repos =====") @@ -3541,7 +3848,7 @@ def analyze_things(self): # This completely creates: # data["buildroot"]["koji_srpms"][koji_id][arch][srpm_id]... # data["buildroot"]["srpms"][repo_id][arch][srpm_id]... - # + log("") log("===== Analyzing Buildroot =====") log("") @@ -3550,7 +3857,6 @@ def analyze_things(self): self._record_metric("finished _analyze_buildroot()") # Add buildroot packages to views - # # Further extends the following with buildroot packages: # data["views"][view_id]["pkgs"] # data["views"][view_id]["source_pkgs"] @@ -3586,13 +3892,11 @@ def analyze_things(self): self._record_metric("finished _recommend_maintainers()") - # Finally, save the cache for next time dump_data(self.settings["root_log_deps_cache_path"], self.cache["root_log_deps"]["next"]) self._record_metric("finished dumping the root log data cache") - - self._record_metric("finished analyze_things()") + self._record_metric("finished analyze_things()") return self.data diff --git a/content_resolver/config_manager.py b/content_resolver/config_manager.py index bdc12ed..eaad42e 100644 --- a/content_resolver/config_manager.py +++ b/content_resolver/config_manager.py @@ -98,9 +98,7 @@ def _load_config_repo_v2(self, document_id, document, settings): for arch_raw in document["data"]["source"]["architectures"]: arch = str(arch_raw) if arch not in settings["allowed_arches"]: - log(" Warning: {file}.yaml lists an unsupported architecture: {arch}. Moving on...".format( - file=document_id, - arch=arch)) + log(f" Warning: {document_id}.yaml lists an unsupported architecture: {arch}. Moving on...") continue config["source"]["architectures"].append(str(arch)) except KeyError: @@ -120,11 +118,8 @@ def _load_config_repo_v2(self, document_id, document, settings): config["source"]["repos"][id]["name"] = name try: config["source"]["repos"][id]["baseurl"] = repo_data["baseurl"] - except KeyError: - # FIXME: No reference for `yaml_file` - raise ConfigError("'{file}.yaml' - is invalid. Repo {id} doesn't list baseurl.".format( - file=yml_file, - id=id)) + except KeyError as err: + raise ConfigError(f"'{document_id}.yaml' - is invalid. Repo {id} doesn't list baseurl.") from err config["source"]["repos"][id]["priority"] = priority config["source"]["repos"][id]["exclude"] = exclude config["source"]["repos"][id]["limit_arches"] = limit_arches @@ -190,10 +185,7 @@ def _load_config_env(self, document_id, document, settings): if "arch_packages" in document["data"]: for arch, pkgs in document["data"]["arch_packages"].items(): if arch not in settings["allowed_arches"]: - log(" Warning: {file}.yaml lists an unsupported architecture: {arch}. Moving on...".format( - file=document_id, - arch=arch - )) + log(f" Warning: {document_id}.yaml lists an unsupported architecture: {arch}. Moving on...") continue for pkg_raw in pkgs: pkg = str(pkg_raw) @@ -274,9 +266,7 @@ def _load_config_workload(self, document_id, document, settings): config["packages"].append(str(pkg)) except (TypeError, KeyError): pass # Because it's now valid - #log(" Warning: {file} has an empty 'packages' field defined which is invalid. Moving on...".format( - # file=document_id - #)) + #log(f" Warning: {document_id} has an empty 'packages' field defined which is invalid. Moving on...") # Architecture-specific packages. config["arch_packages"] = {} @@ -285,10 +275,7 @@ def _load_config_workload(self, document_id, document, settings): if "arch_packages" in document["data"]: for arch, pkgs in document["data"]["arch_packages"].items(): if arch not in settings["allowed_arches"]: - log(" Warning: {file}.yaml lists an unsupported architecture: {arch}. Moving on...".format( - file=document_id, - arch=arch - )) + log(f" Warning: {document_id}.yaml lists an unsupported architecture: {arch}. Moving on...") continue # This workaround allows for "arch_packages/ARCH" to be left empty in the config try: @@ -296,10 +283,7 @@ def _load_config_workload(self, document_id, document, settings): pkg = str(pkg_raw) config["arch_packages"][arch].append(pkg) except TypeError: - log(" Warning: {file} has an empty 'arch_packages/{arch}' field defined which is invalid. Moving on...".format( - file=document_id, - arch=arch - )) + log(f" Warning: {document_id} has an empty 'arch_packages/{arch}' field defined which is invalid. Moving on...") # Extra installation options. # The following are now supported: @@ -460,10 +444,7 @@ def _load_config_compose_view(self, document_id, document, settings): if "unwanted_arch_packages" in document["data"]: for arch, pkgs in document["data"]["unwanted_arch_packages"].items(): if arch not in settings["allowed_arches"]: - log(" Warning: {file}.yaml lists an unsupported architecture: {arch}. Moving on...".format( - file=document_id, - arch=arch - )) + log(f" Warning: {document_id}.yaml lists an unsupported architecture: {arch}. Moving on...") continue for pkg_raw in pkgs: pkg = str(pkg_raw) @@ -524,10 +505,7 @@ def _load_config_addon_view(self, document_id, document, settings): if "unwanted_arch_packages" in document["data"]: for arch, pkgs in document["data"]["unwanted_arch_packages"].items(): if arch not in settings["allowed_arches"]: - log(" Warning: {file}.yaml lists an unsupported architecture: {arch}. Moving on...".format( - file=document_id, - arch=arch - )) + log(f" Warning: {document_id}.yaml lists an unsupported architecture: {arch}. Moving on...") continue for pkg_raw in pkgs: pkg = str(pkg_raw) @@ -600,10 +578,7 @@ def _load_config_unwanted(self, document_id, document, settings): if "unwanted_arch_packages" in document["data"]: for arch, pkgs in document["data"]["unwanted_arch_packages"].items(): if arch not in settings["allowed_arches"]: - log(" Warning: {file}.yaml lists an unsupported architecture: {arch}. Moving on...".format( - file=document_id, - arch=arch - )) + log(f" Warning: {document_id}.yaml lists an unsupported architecture: {arch}. Moving on...") continue for pkg_raw in pkgs: pkg = str(pkg_raw) @@ -622,10 +597,7 @@ def _load_config_unwanted(self, document_id, document, settings): if "unwanted_arch_source_packages" in document["data"]: for arch, pkgs in document["data"]["unwanted_arch_source_packages"].items(): if arch not in settings["allowed_arches"]: - log(" Warning: {file}.yaml lists an unsupported architecture: {arch}. Moving on...".format( - file=document_id, - arch=arch - )) + log(f" Warning: {document_id}.yaml lists an unsupported architecture: {arch}. Moving on...") continue for pkg_raw in pkgs: pkg = str(pkg_raw) @@ -656,10 +628,7 @@ def _load_config_buildroot(self, document_id, document, settings): if "base_buildroot" in document["data"]: for arch, pkgs in document["data"]["base_buildroot"].items(): if arch not in settings["allowed_arches"]: - log(" Warning: {file}.yaml lists an unsupported architecture: {arch}. Moving on...".format( - file=document_id, - arch=arch - )) + log(f" Warning: {document_id}.yaml lists an unsupported architecture: {arch}. Moving on...") continue if pkgs: for pkg_raw in pkgs: @@ -672,10 +641,7 @@ def _load_config_buildroot(self, document_id, document, settings): if "source_packages" in document["data"]: for arch, srpms_dict in document["data"]["source_packages"].items(): if arch not in settings["allowed_arches"]: - log(" Warning: {file}.yaml lists an unsupported architecture: {arch}. Moving on...".format( - file=document_id, - arch=arch - )) + log(f" Warning: {document_id}.yaml lists an unsupported architecture: {arch}. Moving on...") continue if not srpms_dict: continue @@ -686,9 +652,7 @@ def _load_config_buildroot(self, document_id, document, settings): for pkg_raw in srpm_data["requires"]: requires.append(str(pkg_raw)) except TypeError: - log(" Warning: {file} has an empty 'requires' field defined which is invalid. Moving on...".format( - file=document_id - )) + log(f" Warning: {document_id} has an empty 'requires' field defined which is invalid. Moving on...") continue config["source_packages"][arch][str(srpm_name)] = {} @@ -708,10 +672,7 @@ def _load_json_data_buildroot_pkg_relations(self, document_id, document, setting # Arch arch = document["data"]["arch"] if arch not in settings["allowed_arches"]: - raise ConfigError("Error: '{file}.json' lists an unsupported architecture: {arch}.".format( - file=document_id, - arch=arch - )) + raise ConfigError(f"Error: '{document_id}.json' lists an unsupported architecture: {arch}.") config["arch"] = arch #pkg_relations @@ -861,9 +822,7 @@ def get_configs(self): try: document = yaml.safe_load(file) except yaml.YAMLError as err: - raise ConfigError("Error loading a config '{filename}': {err}".format( - filename=yml_file, - err=err)) + raise ConfigError(f"Error loading a config '{yml_file}': {err}") from err # Only accept yaml files stating their purpose! if not ("document" in document and "version" in document): @@ -970,10 +929,8 @@ def get_configs(self): try: try: json_data = self.load_data(os.path.join(directory, json_file)) - except: - raise ConfigError("Error loading a JSON data file '{filename}': {err}".format( - filename=json_file, - err=err)) + except Exception as err: + raise ConfigError(f"Error loading a JSON data file '{json_file}': {err}") from err # Only accept json files stating their purpose! if not ("document_type" in json_data and "version" in json_data): diff --git a/content_resolver/historia_data.py b/content_resolver/historia_data.py index 6784bf7..3e15ef8 100644 --- a/content_resolver/historia_data.py +++ b/content_resolver/historia_data.py @@ -15,10 +15,7 @@ def _save_current_historic_data(query): # Where to save it year = datetime.datetime.now().strftime("%Y") week = datetime.datetime.now().strftime("%W") - filename = "historic_data-{year}-week_{week}.json".format( - year=year, - week=week - ) + filename = f"historic_data-{year}-week_{week}.json" output_dir = os.path.join(query.settings["output"], "history") os.makedirs(output_dir, exist_ok=True) file_path = os.path.join(output_dir, filename) @@ -84,9 +81,7 @@ def _save_current_historic_data(query): history_data["views"][view_conf_id]["srpm_count_build_level_2_plus"] = view_all_arches["numbers"]["srpms"]["build_level_2_plus"] # And save it - log(" Saving in: {file_path}".format( - file_path=file_path - )) + log(f" Saving in: {file_path}") dump_data(file_path, history_data) log(" Done!") @@ -117,23 +112,15 @@ def _read_historic_data(query): date = datetime.datetime.strptime(document["date"],"%Y-%m-%d") year = date.strftime("%Y") week = date.strftime("%W") - key = "{year}-week_{week}".format( - year=year, - week=week - ) + key = f"{year}-week_{week}" except (KeyError, ValueError): - err_log("Invalid file in historic data: {filename}. Ignoring.".format( - filename=filename - )) + err_log(f"Invalid file in historic data: {filename}. Ignoring.") continue historic_data[key] = document return historic_data - log(" Done!") - log("") - def _generate_chartjs_data(historic_data, query): @@ -172,9 +159,7 @@ def _generate_chartjs_data(historic_data, query): entry_data["datasets"].append(dataset) - entry_name = "chartjs-data--workload--{workload_id}".format( - workload_id=workload_id - ) + entry_name = f"chartjs-data--workload--{workload_id}" _generate_json_file(entry_data, entry_name, query.settings) # Data for workload overview pages @@ -219,10 +204,7 @@ def _generate_chartjs_data(historic_data, query): entry_data["datasets"].append(dataset) - entry_name = "chartjs-data--workload-overview--{workload_conf_id}--{repo_id}".format( - workload_conf_id=workload_conf_id, - repo_id=repo_id - ) + entry_name = f"chartjs-data--workload-overview--{workload_conf_id}--{repo_id}" _generate_json_file(entry_data, entry_name, query.settings) # Data for workload cmp arches pages @@ -270,11 +252,7 @@ def _generate_chartjs_data(historic_data, query): entry_data["datasets"].append(dataset) - entry_name = "chartjs-data--workload-cmp-arches--{workload_conf_id}--{env_conf_id}--{repo_id}".format( - workload_conf_id=workload_conf_id, - env_conf_id=env_conf_id, - repo_id=repo_id - ) + entry_name = f"chartjs-data--workload-cmp-arches--{workload_conf_id}--{env_conf_id}--{repo_id}" _generate_json_file(entry_data, entry_name, query.settings) # Data for workload cmp envs pages @@ -322,11 +300,7 @@ def _generate_chartjs_data(historic_data, query): entry_data["datasets"].append(dataset) - entry_name = "chartjs-data--workload-cmp-envs--{workload_conf_id}--{repo_id}--{arch}".format( - workload_conf_id=workload_conf_id, - repo_id=repo_id, - arch=arch - ) + entry_name = f"chartjs-data--workload-cmp-envs--{workload_conf_id}--{repo_id}--{arch}" _generate_json_file(entry_data, entry_name, query.settings) # Data for env pages @@ -365,9 +339,7 @@ def _generate_chartjs_data(historic_data, query): entry_data["datasets"].append(dataset) - entry_name = "chartjs-data--env--{env_id}".format( - env_id=env_id - ) + entry_name = f"chartjs-data--env--{env_id}" _generate_json_file(entry_data, entry_name, query.settings) # Data for env overview pages @@ -412,10 +384,7 @@ def _generate_chartjs_data(historic_data, query): entry_data["datasets"].append(dataset) - entry_name = "chartjs-data--env-overview--{env_conf_id}--{repo_id}".format( - env_conf_id=env_conf_id, - repo_id=repo_id - ) + entry_name = f"chartjs-data--env-overview--{env_conf_id}--{repo_id}" _generate_json_file(entry_data, entry_name, query.settings) # Data for env cmp arches pages @@ -460,10 +429,7 @@ def _generate_chartjs_data(historic_data, query): entry_data["datasets"].append(dataset) - entry_name = "chartjs-data--env-cmp-arches--{env_conf_id}--{repo_id}".format( - env_conf_id=env_conf_id, - repo_id=repo_id - ) + entry_name = f"chartjs-data--env-cmp-arches--{env_conf_id}--{repo_id}" _generate_json_file(entry_data, entry_name, query.settings) # Data for view pages @@ -482,21 +448,9 @@ def _generate_chartjs_data(historic_data, query): # Second, get the actual data for everything that's needed entry_data["datasets"] = [] + dataset_names = ["env", "req", "dep"] if view_all_arches["has_buildroot"]: - dataset_names = [ - "env", - "req", - "dep", - "build_base", - "build_level_1", - "build_level_2_plus" - ] - else: - dataset_names = [ - "env", - "req", - "dep" - ] + dataset_names.extend(["build_base", "build_level_1", "build_level_2_plus"]) dataset_metadata = { "env": { @@ -552,9 +506,7 @@ def _generate_chartjs_data(historic_data, query): entry_data["datasets"].append(dataset) - entry_name = "chartjs-data--view--{view_conf_id}".format( - view_conf_id=view_conf_id - ) + entry_name = f"chartjs-data--view--{view_conf_id}" _generate_json_file(entry_data, entry_name, query.settings) diff --git a/content_resolver/page_generation.py b/content_resolver/page_generation.py index 0950f36..dfce6ab 100644 --- a/content_resolver/page_generation.py +++ b/content_resolver/page_generation.py @@ -6,17 +6,13 @@ def _generate_html_page(template_name, template_data, page_name, settings): - log("Generating the '{page_name}' page...".format( - page_name=page_name - )) + log(f"Generating the '{page_name}' page...") output = settings["output"] template_env = settings["jinja2_template_env"] - template = template_env.get_template("{template_name}.html".format( - template_name=template_name - )) + template = template_env.get_template(f"{template_name}.html") if not template_data: template_data = {} @@ -24,16 +20,14 @@ def _generate_html_page(template_name, template_data, page_name, settings): page = template.render(**template_data) - filename = ("{page_name}.html".format( - page_name=page_name.replace(":", "--") - )) + # Prevent path traversal by extracting only the basename + safe_page_name = os.path.basename(page_name.replace(":", "--")) + filename = f"{safe_page_name}.html" - log(" Writing file... ({filename})".format( - filename=filename - )) + log(f" Writing file... ({filename})") with open(os.path.join(output, filename), "w") as file: file.write(page) - + log(" Done!") log("") @@ -50,10 +44,7 @@ def _generate_workload_pages(query): "repo_id": repo_id } - page_name = "workload-overview--{workload_conf_id}--{repo_id}".format( - workload_conf_id=workload_conf_id, - repo_id=repo_id - ) + page_name = f"workload-overview--{workload_conf_id}--{repo_id}" _generate_html_page("workload_overview", template_data, page_name, query.settings) # Workload detail pages @@ -79,13 +70,9 @@ def _generate_workload_pages(query): "repo": repo } - page_name = "workload--{workload_id}".format( - workload_id=workload_id - ) + page_name = f"workload--{workload_id}" _generate_html_page("workload", template_data, page_name, query.settings) - page_name = "workload-dependencies--{workload_id}".format( - workload_id=workload_id - ) + page_name = f"workload-dependencies--{workload_id}" _generate_html_page("workload_dependencies", template_data, page_name, query.settings) # Workload compare arches pages @@ -122,11 +109,7 @@ def _generate_workload_pages(query): "rows": rows } - page_name = "workload-cmp-arches--{workload_conf_id}--{env_conf_id}--{repo_id}".format( - workload_conf_id=workload_conf_id, - env_conf_id=env_conf_id, - repo_id=repo_id - ) + page_name = f"workload-cmp-arches--{workload_conf_id}--{env_conf_id}--{repo_id}" _generate_html_page("workload_cmp_arches", template_data, page_name, query.settings) @@ -162,11 +145,7 @@ def _generate_workload_pages(query): "rows": rows } - page_name = "workload-cmp-envs--{workload_conf_id}--{repo_id}--{arch}".format( - workload_conf_id=workload_conf_id, - repo_id=repo_id, - arch=arch - ) + page_name = f"workload-cmp-envs--{workload_conf_id}--{repo_id}--{arch}" _generate_html_page("workload_cmp_envs", template_data, page_name, query.settings) @@ -185,10 +164,7 @@ def _generate_env_pages(query): "repo_id": repo_id } - page_name = "env-overview--{env_conf_id}--{repo_id}".format( - env_conf_id=env_conf_id, - repo_id=repo_id - ) + page_name = f"env-overview--{env_conf_id}--{repo_id}" _generate_html_page("env_overview", template_data, page_name, query.settings) # env detail pages @@ -209,14 +185,10 @@ def _generate_env_pages(query): "repo": repo } - page_name = "env--{env_id}".format( - env_id=env_id - ) + page_name = f"env--{env_id}" _generate_html_page("env", template_data, page_name, query.settings) - page_name = "env-dependencies--{env_id}".format( - env_id=env_id - ) + page_name = f"env-dependencies--{env_id}" _generate_html_page("env_dependencies", template_data, page_name, query.settings) # env compare arches pages @@ -249,10 +221,7 @@ def _generate_env_pages(query): "rows": rows } - page_name = "env-cmp-arches--{env_conf_id}--{repo_id}".format( - env_conf_id=env_conf_id, - repo_id=repo_id - ) + page_name = f"env-cmp-arches--{env_conf_id}--{repo_id}" _generate_html_page("env_cmp_arches", template_data, page_name, query.settings) @@ -271,15 +240,11 @@ def _generate_maintainer_pages(query): } # Overview page - page_name = "maintainer--{maintainer}".format( - maintainer=maintainer - ) + page_name = f"maintainer--{maintainer}" _generate_html_page("maintainer_overview", template_data, page_name, query.settings) # My Workloads page - page_name = "maintainer-workloads--{maintainer}".format( - maintainer=maintainer - ) + page_name = f"maintainer-workloads--{maintainer}" _generate_html_page("maintainer_workloads", template_data, page_name, query.settings) log(" Done!") @@ -294,9 +259,7 @@ def _generate_config_pages(query): "query": query, "conf_type": conf_type } - page_name = "configs_{conf_type}".format( - conf_type=conf_type - ) + page_name = f"configs_{conf_type}" _generate_html_page("configs", template_data, page_name, query.settings) # Config repo pages @@ -305,9 +268,7 @@ def _generate_config_pages(query): "query": query, "repo_conf": repo_conf } - page_name = "config-repo--{repo_id}".format( - repo_id=repo_id - ) + page_name = f"config-repo--{repo_id}" _generate_html_page("config_repo", template_data, page_name, query.settings) # Config env pages @@ -316,9 +277,7 @@ def _generate_config_pages(query): "query": query, "env_conf": env_conf } - page_name = "config-env--{env_conf_id}".format( - env_conf_id=env_conf_id - ) + page_name = f"config-env--{env_conf_id}" _generate_html_page("config_env", template_data, page_name, query.settings) # Config workload pages @@ -327,9 +286,7 @@ def _generate_config_pages(query): "query": query, "workload_conf": workload_conf } - page_name = "config-workload--{workload_conf_id}".format( - workload_conf_id=workload_conf_id - ) + page_name = f"config-workload--{workload_conf_id}" _generate_html_page("config_workload", template_data, page_name, query.settings) # Config label pages @@ -338,9 +295,7 @@ def _generate_config_pages(query): "query": query, "label_conf": label_conf } - page_name = "config-label--{label_conf_id}".format( - label_conf_id=label_conf_id - ) + page_name = f"config-label--{label_conf_id}" _generate_html_page("config_label", template_data, page_name, query.settings) # Config view pages @@ -349,9 +304,7 @@ def _generate_config_pages(query): "query": query, "view_conf": view_conf } - page_name = "config-view--{view_conf_id}".format( - view_conf_id=view_conf_id - ) + page_name = f"config-view--{view_conf_id}" _generate_html_page("config_view", template_data, page_name, query.settings) # Config unwanted pages @@ -360,9 +313,7 @@ def _generate_config_pages(query): "query": query, "unwanted_conf": unwanted_conf } - page_name = "config-unwanted--{unwanted_conf_id}".format( - unwanted_conf_id=unwanted_conf_id - ) + page_name = f"config-unwanted--{unwanted_conf_id}" _generate_html_page("config_unwanted", template_data, page_name, query.settings) log(" Done!") @@ -379,10 +330,7 @@ def _generate_repo_pages(query): "repo": repo, "arch": arch } - page_name = "repo--{repo_id}--{arch}".format( - repo_id=repo_id, - arch=arch - ) + page_name = f"repo--{repo_id}--{arch}" _generate_html_page("repo", template_data, page_name, query.settings) @@ -404,39 +352,27 @@ def _generate_view_pages(query): } # Generate the overview page - page_name = "view--{view_conf_id}".format( - view_conf_id=view_conf_id - ) + page_name = f"view--{view_conf_id}" _generate_html_page("view_overview", template_data, page_name, query.settings) # Generate the packages page - page_name = "view-packages--{view_conf_id}".format( - view_conf_id=view_conf_id - ) + page_name = f"view-packages--{view_conf_id}" _generate_html_page("view_packages", template_data, page_name, query.settings) # Generate the source packages page - page_name = "view-sources--{view_conf_id}".format( - view_conf_id=view_conf_id - ) + page_name = f"view-sources--{view_conf_id}" _generate_html_page("view_sources", template_data, page_name, query.settings) # Generate the unwanted packages page - page_name = "view-unwanted--{view_conf_id}".format( - view_conf_id=view_conf_id - ) + page_name = f"view-unwanted--{view_conf_id}" _generate_html_page("view_unwanted", template_data, page_name, query.settings) # Generate the workloads page - page_name = "view-workloads--{view_conf_id}".format( - view_conf_id=view_conf_id - ) + page_name = f"view-workloads--{view_conf_id}" _generate_html_page("view_workloads", template_data, page_name, query.settings) # Generate the errors page - page_name = "view-errors--{view_conf_id}".format( - view_conf_id=view_conf_id - ) + page_name = f"view-errors--{view_conf_id}" _generate_html_page("view_errors", template_data, page_name, query.settings) @@ -445,10 +381,7 @@ def _generate_view_pages(query): # Generate the arch lists for arch in view_conf["architectures"]: - view_id = "{view_conf_id}:{arch}".format( - view_conf_id=view_conf_id, - arch=arch - ) + view_id = f"{view_conf_id}:{arch}" view = query.data["views"][view_id] @@ -458,10 +391,7 @@ def _generate_view_pages(query): "view": view, "arch": arch, } - page_name = "view--{view_conf_id}--{arch}".format( - view_conf_id=view_conf_id, - arch=arch - ) + page_name = f"view--{view_conf_id}--{arch}" #_generate_html_page("view_packages", template_data, page_name, query.settings) # ... @@ -475,10 +405,7 @@ def _generate_view_pages(query): "view_all_arches": view_all_arches, "pkg": pkg, } - page_name = "view-rpm--{view_conf_id}--{pkg_name}".format( - view_conf_id=view_conf_id, - pkg_name=pkg_name - ) + page_name = f"view-rpm--{view_conf_id}--{pkg_name}" _generate_html_page("view_rpm", template_data, page_name, query.settings) _generate_json_file(pkg, page_name, query.settings) @@ -492,10 +419,7 @@ def _generate_view_pages(query): "view_all_arches": view_all_arches, "srpm": srpm, } - page_name = "view-srpm--{view_conf_id}--{srpm_name}".format( - view_conf_id=view_conf_id, - srpm_name=srpm_name - ) + page_name = f"view-srpm--{view_conf_id}--{srpm_name}" _generate_html_page("view_srpm", template_data, page_name, query.settings) _generate_json_file(srpm, page_name, query.settings) diff --git a/content_resolver/utils.py b/content_resolver/utils.py index c4e4b91..b80d1d2 100644 --- a/content_resolver/utils.py +++ b/content_resolver/utils.py @@ -2,7 +2,12 @@ import json import re import sys +from contextlib import contextmanager +from typing import Iterator + import jinja2 +import libdnf5 + class SetEncoder(json.JSONEncoder): def default(self, obj): @@ -66,4 +71,27 @@ def url_to_id(url): def datetime_now_string(): - return datetime.datetime.now().strftime("%m/%d/%Y, %H:%M:%S") \ No newline at end of file + return datetime.datetime.now().strftime("%m/%d/%Y, %H:%M:%S") + + +@contextmanager +def dnf5_base() -> Iterator[libdnf5.base.Base]: + """ + Context manager wrapper for libdnf5.base.Base(). + + DNF5's Base class doesn't natively support the context manager protocol, + so this wrapper provides proper resource management. + + Usage: + with dnf5_base() as base: + config = base.get_config() + # ... use base ... + """ + base = libdnf5.base.Base() + try: + yield base + finally: + # DNF5 Base cleanup is handled by Python's garbage collector + # No explicit cleanup needed, but the finally block ensures + # proper exception handling and resource cleanup if needed in future + pass