From 36214650f415041a49a4faea20c2e41719f44135 Mon Sep 17 00:00:00 2001 From: Yaakov Selkowitz Date: Mon, 10 Aug 2026 12:19:29 -0400 Subject: [PATCH] Revert "DNF4 to DNF5" --- .github/workflows/docker-image.yml | 12 +- .github/workflows/fork-only-workflow.yml | 7 +- Dockerfile | 13 +- README.md | 84 +- content_resolver/analyzer.py | 2116 +++++++++------------- content_resolver/config_manager.py | 79 +- content_resolver/historia_data.py | 76 +- content_resolver/page_generation.py | 148 +- content_resolver/utils.py | 30 +- 9 files changed, 1149 insertions(+), 1416 deletions(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 0f5bb97..f69e94e 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -20,17 +20,7 @@ jobs: run: dnf -y update fedora-gpg-keys - name: Install git and Python libraries - run: | - dnf -y install \ - git-core \ - python3-flake8 \ - python3-jinja2 \ - python3-koji \ - python3-libdnf5 \ - python3-pip \ - python3-pytest \ - python3-pytest-cov \ - python3-yaml + run: dnf -y install git-core python3-yaml python3-jinja2 python3-koji python3-pytest python3-flake8 python3-dnf - name: Clean up run: dnf clean all diff --git a/.github/workflows/fork-only-workflow.yml b/.github/workflows/fork-only-workflow.yml index f136392..73e61eb 100644 --- a/.github/workflows/fork-only-workflow.yml +++ b/.github/workflows/fork-only-workflow.yml @@ -33,13 +33,12 @@ jobs: run: | dnf -y install \ git \ - python3-flake8 \ python3-jinja2 \ python3-koji \ - python3-libdnf5 \ + python3-yaml \ + python3-dnf \ python3-pytest \ - python3-pytest-cov \ - python3-yaml + python3-flake8 - name: Clean up run: dnf clean all diff --git a/Dockerfile b/Dockerfile index a964bb9..e5c9660 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,16 +1,7 @@ FROM registry.fedoraproject.org/fedora:44 -RUN dnf -y update fedora-gpg-keys \ - --setopt=tsflags=nodocs \ - --setopt=install_weak_deps=False && \ - dnf -y install \ - git \ - python3-jinja2 \ - python3-koji \ - python3-libdnf5 \ - python3-yaml \ - --setopt=tsflags=nodocs \ - --setopt=install_weak_deps=False && \ +RUN dnf -y update fedora-gpg-keys && \ + dnf -y install git python3-jinja2 python3-koji python3-yaml python3-dnf && \ dnf clean all && \ rm -rf /var/cache/dnf diff --git a/README.md b/README.md index cb224fc..3a936c8 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,6 @@ # Content Resolver -[![Content Resolver CI](https://github.com/fedora-eln/content-resolver/actions/workflows/docker-image.yml/badge.svg)](https://github.com/fedora-eln/content-resolver/actions/workflows/docker-image.yml) - -Content Resolver makes it easy to define and inspect package sets of RPM-based Linux distributions. +Content Resolver makes it easy to define and inspect package sets of RPM-based Linux distribution. You define what packages you need, and Content Resolver gives you the whole picture including all the dependencies. And it keeps it up-to-date as packages get updated over time. @@ -14,28 +12,6 @@ Content Resolver also helps with minimisation efforts by showing detailed depend ## Using Content Resolver -### Code Structure - -Core functionality - -``` -content-resolver/ -├── content_resolver/ -│ ├── analyzer.py # Core resolution engine (DNF5) -│ ├── config_manager.py # YAML config loading -│ ├── data_generation.py # Generate non HTML output -│ ├── exceptions.py # Custom exceptions -│ ├── history_data.py # Handle history for graph generation -│ ├── page_generation.py # Generate HTML output -│ ├── query.py # Internal data query mechanism -│ └── utils.py # Helper functions -├── templates/ # Jinja2 HTML templates -├── test_configs/ # Test configurations -├── content_resolver.py # Main entry point -└── refresh.sh # Production deployment script -``` - - ### Controlling Content Resolver Content Resolver is entirely controlled by a set of YAML files stored in a git repository. Changes to the content can easily be modified and reviewed via pull requests. @@ -168,61 +144,3 @@ $ docker run --rm -it --tmpfs /dnf_cachedir -v $(pwd):/workspace content-resolve ``` The output will be generated in the `output` directory. Open the `output/index.html` in your web browser of choice to see the result. - -### Resolution Process - -**Phase 1: Repository Analysis** -``` -For each repository and architecture: - 1. Create DNF5 Base with repository configuration - 2. Apply repository priorities (1=highest) - 3. Apply package exclude lists (e.g., rust-*-devel, golang-*-devel) - 4. Load repository metadata - 5. Query all available packages - 6. Deduplicate by NEVRA (Name-Epoch:Version-Release.Arch) - 7. Track which repo provides each package -``` - -**Phase 2: Environment Resolution** -``` -For each environment: - 1. Create DNF5 Goal with environment packages - 2. Resolve dependencies using DNF5 transaction - 3. Check transaction.get_problems() for conflicts - 4. Extract installed packages (base environment) - 5. Build package relationship graph -``` - -**Phase 3: Workload Resolution** -``` -For each workload on top of each environment: - 1. Load base environment packages - 2. Add workload-specific packages to Goal - 3. Resolve additional dependencies - 4. Separate environment packages from added packages - 5. Track which workloads require which packages -``` - -**Phase 4: Buildroot Resolution** (for views with `buildroot_strategy: root_logs`) -``` -For each source package in the view: - 1. Download Koji root.log via Koji API - 2. Parse build dependencies from root log - 3. Create fake workload with build dependencies - 4. Resolve buildroot on top of build group - 5. Track buildroot packages separately from runtime - 6. Iterate until all transitive build deps resolved -``` - -### Repository Priority System - -Repositories are assigned priorities (1=highest, 5=lowest) to control package selection when multiple versions exist: - -```yaml -BaseOS: priority: 1 # Prefer packages from BaseOS -AppStream: priority: 1 # Prefer packages from AppStream -CRB: priority: 1 # CodeReady Builder -Extras: priority: 3 # Additional packages -buildroot: priority: 4 # Build-time only packages -Rawhide: priority: 5 # Fallback for missing packages -``` diff --git a/content_resolver/analyzer.py b/content_resolver/analyzer.py index 38bebb5..2099275 100644 --- a/content_resolver/analyzer.py +++ b/content_resolver/analyzer.py @@ -1,79 +1,20 @@ -import asyncio -import datetime -import json -import multiprocessing -import os -import sys -import tempfile -import time -import traceback -import urllib.request +import tempfile, os, json, datetime, dnf, urllib.request, sys, koji +import re, time from concurrent.futures import ProcessPoolExecutor, as_completed -import koji -from libdnf5.base import Base, Goal, GoalJobSettings -from libdnf5.exception import BaseTransactionError -from libdnf5.exception import Error as DnfErr -from libdnf5.exception import RepoDownloadError as Dnf5RepoDownloadError -from libdnf5.exception import UserAssertionError -from libdnf5.repo import RepoQuery -from libdnf5.rpm import PackageQuery - -from content_resolver.exceptions import ( - AnalysisError, - BuildGroupAnalysisError, - KojiRootLogError, - RepoDownloadError, -) -from content_resolver.utils import ( - dnf5_base, - dump_data, - err_log, - load_data, - log, - pkg_id_to_name, - size, - url_to_id, - workload_id_to_conf_id, -) +import multiprocessing, asyncio +from content_resolver.utils import dump_data, load_data, log, err_log, pkg_id_to_name, size, workload_id_to_conf_id, url_to_id +from content_resolver.exceptions import RepoDownloadError, BuildGroupAnalysisError, KojiRootLogError, AnalysisError def pkg_placeholder_name_to_id(placeholder_name): - return f"{placeholder_name}-000-placeholder.placeholder" + placeholder_id = f"{placeholder_name}-000-placeholder.placeholder" + return placeholder_id def pkg_placeholder_name_to_nevr(placeholder_name): - return f"{placeholder_name}-000-placeholder" - - -def is_package_resolvable(base: Base, pkg_name: str) -> bool: - """ - Check if a package name is resolvable, either as a real package or via provides. - - Args: - base: DNF5 base object - pkg_name: Package name to check - - Returns: - bool: True if the package can be resolved, False otherwise - """ - query = PackageQuery(base) - - # First check if it exists as a real package name - query.filter_name([pkg_name]) - if not query.empty(): - return True - - # If not found by name, check if any package provides it. - # Note: Must create a new query because DNF5 filters are cumulative. - # Reusing the filtered query would search for provides within the - # already-filtered (possibly empty) result set, not all packages. - query = PackageQuery(base) - query.filter_provides([pkg_name]) - if not query.empty(): - return True - - return False + placeholder_id = f"{placeholder_name}-000-placeholder" + return placeholder_id ##################################################### @@ -91,26 +32,27 @@ def _get_build_deps_from_a_root_log(root_log): state = 0 for file_line in root_log.splitlines(): - split_line = file_line.split() - line_len = len(split_line) # 0/ # parts of the log I don't really care about if state == 0: + # The next installation is the build deps! # So I start caring. Next state! if "'builddep', '--installroot'" in file_line: state += 1 + # 1/ # getting the "already installed" packages to the list elif state == 1: + # "Package already installed" indicates it's directly required, # so save it. # DNF5 does this after "Repositories loaded" and quotes the NVR; # DNF4 does this before "Dependencies resolved" without the quotes. if "is already installed." in file_line: - pkg_name = split_line[3].strip('"').rsplit("-", 2)[0] + pkg_name = file_line.split()[3].strip('"').rsplit("-",2)[0] required_pkgs.append(pkg_name) # That's all! Next state! (DNF4) @@ -121,29 +63,33 @@ def _get_build_deps_from_a_root_log(root_log): elif "Repositories loaded." in file_line: state += 1 + # 2/ # going through the log right before the first package name elif state == 2: + # "Package already installed" indicates it's directly required, # so save it. # DNF4 does this before "Dependencies resolved" without the quotes; # DNF5 does this after "Repositories loaded" and quotes the NVR, but # sometimes prints this in the middle of a dependency line. if "is already installed." in file_line: - pkg_index = split_line.index("already") - 2 - pkg_name = split_line[pkg_index].strip('"').rsplit("-", 2)[0] + pkg_index = file_line.split().index("already") - 2 + pkg_name = file_line.split()[pkg_index].strip('"').rsplit("-",2)[0] required_pkgs.append(pkg_name) # The next line will be the first package. Next state! # DNF5 reports "Installing: ## packages" in the Transaction Summary, # which we need to ignore - if "Installing:" in file_line and line_len == 3: + if "Installing:" in file_line and len(file_line.split()) == 3: state += 1 + # 3/ # And now just saving the packages until the "installing dependencies" part # or the "transaction summary" part if there's no dependencies elif state == 3: + if "Installing dependencies:" in file_line: state = 2 @@ -151,12 +97,12 @@ def _get_build_deps_from_a_root_log(root_log): state = 2 # Sometimes DNF5 prints "Package ... is already installed" in middle of the output. - elif split_line[2] == "Package" and split_line[-1] == "installed.": - pkg_name = split_line[3].strip('"').rsplit("-", 2)[0] + elif file_line.split()[2] == "Package" and file_line.split()[-1] == "installed.": + pkg_name = file_line.split()[3].strip('"').rsplit("-",2)[0] required_pkgs.append(pkg_name) else: - # We need to deal with the following... + # I need to deal with the following thing... # # DEBUG util.py:446: gobject-introspection-devel aarch64 1.70.0-1.fc36 build 1.1 M # DEBUG util.py:446: graphene-devel aarch64 1.10.6-3.fc35 build 159 k @@ -168,12 +114,12 @@ def _get_build_deps_from_a_root_log(root_log): # The "gstreamer1-plugins-bad-free-devel" package name is too long to fit in the column, # so it gets split on two lines. # - # When using the usual split_line[2] we get the correct name, - # but the next line gives us "aarch64" as a package name which is wrong. + # Which if I take the usual file_line.split()[2] I get the correct name, + # but the next line gives me "aarch64" as a package name which is wrong. # - # So the usual line has line_len == 8 - # The one with the long package name has line_len == 3 - # and the one following it has line_len == 7 + # So the usual line has file_line.split() == 8 + # The one with the long package name has file_line.split() == 3 + # and the one following it has file_line.split() == 7 # # One more thing... long release! # @@ -182,46 +128,51 @@ def _get_build_deps_from_a_root_log(root_log): # DEBUG util.py:446: build 71 k # DEBUG util.py:446: tpm2-tss-devel aarch64 3.1.0-4.fc36 build 315 k # - # So the good one here is line_len == 5. - # And the following is also line_len == 5. Fun! + # So the good one here is file_line.split() == 5. + # And the following is also file_line.split() == 5. Fun! # # So if it ends with B, k, M, G it's the wrong line, so skip, otherwise take the package name. # - # we can also anticipate both get long... that would mean we need to skip line_len == 4. + # I can also anticipate both get long... that would mean I need to skip file_line.split() == 4. - if line_len in (10, 11): + if len(file_line.split()) == 10 or len(file_line.split()) == 11: # Sometimes DNF5 prints "Package ... is already installed" in the middle of a line - pkg_index = split_line.index("already") - 2 - pkg_name = split_line[pkg_index].strip('"').rsplit("-", 2)[0] + pkg_index = file_line.split().index("already") - 2 + pkg_name = file_line.split()[pkg_index].strip('"').rsplit("-",2)[0] required_pkgs.append(pkg_name) - pkg_name = split_line[7] if pkg_index == 3 else split_line[2] + if pkg_index == 3: + pkg_name = file_line.split()[7] + else: + pkg_name = file_line.split()[2] required_pkgs.append(pkg_name) - # TODO: line_len == 9 ?? + # TODO: len(file_line.split()) == 9 ?? - elif line_len in (3, 8): - pkg_name = split_line[2] + elif len(file_line.split()) == 8 or len(file_line.split()) == 3: + pkg_name = file_line.split()[2] required_pkgs.append(pkg_name) - elif line_len in (4, 7): + elif len(file_line.split()) == 7 or len(file_line.split()) == 4: continue - elif line_len in (5, 6): + elif len(file_line.split()) == 6 or len(file_line.split()) == 5: # DNF5 uses B/KiB/MiB/GiB, DNF4 uses B/k/M/G - if split_line[4] in ["B", "KiB", "k", "MiB", "M", "GiB", "G"]: + if file_line.split()[4] in ["B", "KiB", "k", "MiB", "M", "GiB", "G"]: continue else: - pkg_name = split_line[2] + pkg_name = file_line.split()[2] required_pkgs.append(pkg_name) else: raise KojiRootLogError + # 4/ # I'm done. So I can break out of the loop. elif state == 4: break + return required_pkgs @@ -229,21 +180,20 @@ def _get_koji_log_path(srpm_id, arch, koji_session): """ Get koji log path for a given SRPM. """ - max_tries = 10 + MAX_TRIES = 10 attempts = 0 - while attempts < max_tries: + while attempts < MAX_TRIES: try: koji_pkg_data = koji_session.getRPM(f"{srpm_id}.src") koji_logs = koji_session.getBuildLogs(koji_pkg_data["build_id"]) break - except koji.GenericError: + except Exception: attempts += 1 - if attempts == max_tries: + if attempts == MAX_TRIES: raise KojiRootLogError("Could not talk to Koji API") time.sleep(1) - # TODO: use `next` and comprehension for lazy evaluation next(x for x in koji_logs) koji_log_path = None for koji_log in koji_logs: if koji_log["name"] == "root.log": @@ -258,25 +208,24 @@ def _download_root_log_with_retry(root_log_url): """ Download root.log file with retry logic. """ - max_tries = 10 + MAX_TRIES = 10 attempts = 0 request = urllib.request.Request(root_log_url) request.add_header("Accept", "text/plain") request.add_header("User-Agent", "ContentResolver/1.0") - while attempts < max_tries: + while attempts < MAX_TRIES: try: with urllib.request.urlopen(request, timeout=20) as response: root_log_data = response.read() - return root_log_data.decode("utf-8") + return root_log_data.decode('utf-8') except Exception: attempts += 1 - if attempts == max_tries: + if attempts == MAX_TRIES: raise KojiRootLogError(f"Could not download root.log from {root_log_url}") time.sleep(1) - def process_single_srpm_root_log(work_item): """ Process a single SRPM's root.log file. @@ -288,11 +237,11 @@ def process_single_srpm_root_log(work_item): dict: Contains srpm_id, arch, deps (list), error (str or None) """ try: - koji_api_url = work_item["koji_api_url"] - koji_files_url = work_item["koji_files_url"] - srpm_id = work_item["srpm_id"] - arch = work_item["arch"] - dev_buildroot = work_item.get("dev_buildroot", False) + koji_api_url = work_item['koji_api_url'] + koji_files_url = work_item['koji_files_url'] + srpm_id = work_item['srpm_id'] + arch = work_item['arch'] + dev_buildroot = work_item.get('dev_buildroot', False) # Handle development buildroot mode if dev_buildroot: @@ -344,42 +293,16 @@ def process_single_srpm_root_log(work_item): # Download root.log root_log_url = f"{koji_files_url}/{koji_log_path}" - try: - root_log_contents = _download_root_log_with_retry(root_log_url) - except KojiRootLogError as e: - # Download failed - return { - 'srpm_id': srpm_id, - 'arch': arch, - 'deps': [], - 'error': f"Download failed: {str(e)}" - } + root_log_contents = _download_root_log_with_retry(root_log_url) # Parse dependencies - try: - deps = _get_build_deps_from_a_root_log(root_log_contents) - except Exception as e: - # Parsing failed - return { - 'srpm_id': srpm_id, - 'arch': arch, - 'deps': [], - 'error': f"Parse failed: {str(e)}" - } - - # Check if parsing yielded suspiciously few dependencies - warning = None - if len(deps) == 0: - warning = "WARNING: Zero dependencies found (possible truncated/corrupted log)" - elif len(deps) < 3: - warning = f"WARNING: Only {len(deps)} dependencies found (expected more)" + deps = _get_build_deps_from_a_root_log(root_log_contents) return { 'srpm_id': srpm_id, 'arch': arch, 'deps': deps, - 'error': None, - 'warning': warning + 'error': None } except Exception as e: @@ -390,7 +313,7 @@ def process_single_srpm_root_log(work_item): 'error': str(e) } -class Analyzer: +class Analyzer(): ############################################################################### ### Analyzing stuff! ########################################################## @@ -420,7 +343,7 @@ class Analyzer: # - "dnf_generic_installroot-{repo}-{arch}" <-- installroots for _analyze_pkgs # - "dnf_env_installroot-{env_conf}-{repo}-{arch}" <-- installroots for envs and workloads and buildroots # - # + # def __init__(self, configs, settings): self.workload_queue = {} @@ -430,6 +353,8 @@ def __init__(self, configs, settings): self.configs = configs self.settings = settings + + self.global_dnf_repo_cache = {} self.data = {} self.cache = {} @@ -437,10 +362,6 @@ def __init__(self, configs, settings): self.cache["root_log_deps"]["current"] = {} self.cache["root_log_deps"]["next"] = {} - # Cache for compose metadata (composeinfo.json) - # Maps repo_id -> {arch -> [list of available variant names]} - self.compose_metadata_cache = {} - self.metrics_data = [] # When analysing buildroot, we don't need metadata about @@ -450,11 +371,10 @@ def __init__(self, configs, settings): try: self.cache["root_log_deps"]["current"] = load_data(self.settings["root_log_deps_cache_path"]) - # Validate cached root.log data for issues - self._validate_root_log_cache() except FileNotFoundError: pass + def _record_metric(self, name): this_record = { "name": name, @@ -462,9 +382,10 @@ def _record_metric(self, name): } self.metrics_data.append(this_record) + def print_metrics(self): log("Additional metrics:") - # TODO: use `enumerate(self.metrics_data)` instead of incrementing counter + counter = 0 for this_record in self.metrics_data: @@ -476,399 +397,248 @@ def print_metrics(self): time_diff = this_record["timestamp"] - prev_timestamp - print( - f" {this_record['timestamp'].strftime('%H:%M:%S')} " - f"(+{str(int(time_diff.seconds / 60)).zfill(3)} mins): " - f"{this_record['name']}" - ) + print(" {} (+{} mins): {}".format( + this_record["timestamp"].strftime("%H:%M:%S"), + str(int(time_diff.seconds/60)).zfill(3), + this_record["name"] + )) counter += 1 - def _validate_root_log_cache(self): - """ - Validate cached root.log dependency data for invalid entries. - Reports packages with zero or very low dependency counts. - """ - cache = self.cache["root_log_deps"]["current"] - if not cache: - return - - zero_deps_cached = [] - low_deps_cached = [] - - for koji_id, arches in cache.items(): - for arch, srpms in arches.items(): - for srpm_id, deps in srpms.items(): - if not isinstance(deps, list): - continue - - if len(deps) == 0: - zero_deps_cached.append({ - 'srpm_id': srpm_id, - 'arch': arch, - 'koji_id': koji_id - }) - elif len(deps) < 3: - low_deps_cached.append({ - 'srpm_id': srpm_id, - 'arch': arch, - 'deps': deps, - 'koji_id': koji_id - }) - - # Some verbose logging for debugging - # TODO: Enable some form of log level to increase/decrease verbosity of logs - if zero_deps_cached or low_deps_cached: - log("") - log("=" * 80) - log("⚠️ ROOT LOG CACHE VALIDATION") - log("=" * 80) - - if zero_deps_cached: - log(f"Found {len(zero_deps_cached)} cached packages with ZERO dependencies") - log("These packages may have had root.log download/parse failures in previous runs:") - log("") - for item in zero_deps_cached[:15]: - log(f" ⚠️ {item['srpm_id']} ({item['arch']})") - if len(zero_deps_cached) > 15: - log(f" ... and {len(zero_deps_cached) - 15} more") - log("") - log("Consider clearing cache for these packages to retry root.log download") - - if low_deps_cached: - log("") - log(f"Found {len(low_deps_cached)} cached packages with very few dependencies (<3)") - for item in low_deps_cached[:10]: - log(f" ⚠️ {item['srpm_id']} ({item['arch']}): {item['deps']}") - if len(low_deps_cached) > 10: - log(f" ... and {len(low_deps_cached) - 10} more") - - if zero_deps_cached or low_deps_cached: - log("=" * 80) - log("") - - def _get_available_compose_variants(self, repo, arch): - """ - Fetch and parse composeinfo.json to determine which variants are available - for a given repository and architecture. - - Returns: - dict: Mapping of config repo names to boolean availability - e.g., {'BaseOS': True, 'HA': True, 'RS': False, 'Rawhide': True} - Returns empty dict if composeinfo is not available or on error. - - Caches results in self.compose_metadata_cache[repo_id][arch]. - - Note: Maps config repo names (like 'HA', 'buildroot') to compose variant names - (like 'HighAvailability', 'Buildroot') and marks external fallback repos like - 'Rawhide' as always available. - """ - repo_id = repo["id"] - - # Check cache first - if repo_id not in self.compose_metadata_cache: - self.compose_metadata_cache[repo_id] = {} - - if arch in self.compose_metadata_cache[repo_id]: - return self.compose_metadata_cache[repo_id][arch] - - # If no composeinfo URL is configured, return empty dict (no filtering) - composeinfo_url = repo["source"].get("composeinfo") - if not composeinfo_url: - self.compose_metadata_cache[repo_id][arch] = {} - return {} - - # Mapping of config repo names to compose variant names - # This handles cases where the names differ - repo_name_mapping = { - "HA": "HighAvailability", - "buildroot": "Buildroot", - } - - # External fallback repos that aren't part of the compose - # These should always be marked as available - external_repos = {"Rawhide"} - - try: - # Fetch composeinfo.json - with urllib.request.urlopen(composeinfo_url, timeout=10) as response: - compose_data = json.loads(response.read().decode('utf-8')) - - # Extract variants available for this architecture - compose_variants = set() - variants = compose_data.get("payload", {}).get("variants", {}) - - for variant_name, variant_data in variants.items(): - variant_arches = variant_data.get("arches", []) - if arch in variant_arches: - compose_variants.add(variant_name) - - # Build availability map for all config repo names - availability = {} - for config_name in repo["source"]["repos"].keys(): - # Check if this is an external fallback repo - if config_name in external_repos: - availability[config_name] = True - continue - - # Map config name to compose variant name - compose_name = repo_name_mapping.get(config_name, config_name) - - # Check if the variant exists in the compose - availability[config_name] = compose_name in compose_variants - - # Cache and return - self.compose_metadata_cache[repo_id][arch] = availability - return availability - - except Exception as e: - # On any error (network, parsing, etc.), log and return empty dict - # Empty dict means no filtering - we'll try all repos as before - log(f" Warning: Could not fetch composeinfo from {composeinfo_url}: {e}") - self.compose_metadata_cache[repo_id][arch] = {} - return {} - + def _load_repo_cached(self, base, repo, arch): - """ - Load repository configuration for the given base, repo, and architecture. - - DNF5 MIGRATION NOTE - Repo Caching Disabled: - ============================================= - - DNF5: Repo caching code removed (see _load_repo_cached docstring for explanation) - In DNF4, we cached repo objects here for reuse across Base instances: - DNF5's architecture prevents this - repos are tightly bound to their parent Base - and cannot be safely transferred between instances without memory corruption. - - DNF4 Behavior (OLD): - ------------------- - This function cached repo objects in self.global_dnf_repo_cache and reused them - across multiple dnf.Base() instances. This was a major performance optimization: - - First analysis: ~10s to load repos from network - - Subsequent analyses: ~1s using cached repo objects - - For 1000+ analyses, this saved ~150 minutes of execution time - - DNF5 Limitation (CURRENT): - -------------------------- - DNF5 has tighter coupling between Repo objects and their parent Base instance: - 1. Repos are created via repo_sack.create_repo(), not standalone constructors - 2. Repo objects hold internal references to their parent Base - 3. RepoSack owns the repos and manages their lifecycle - 4. Attempting to reuse a Repo from Base1 in Base2 may cause - Segmentation faults, Memory corruption or other Undefined behavior - - Future Solutions (TODO): - ------------------------ - 1. Use DNF5's built-in metadata cache (set metadata_expire to cache for longer) - 2. Keep a single long-lived Base instance for all analyses (requires state isolation) - 3. Wait for DNF5 upstream to add repo serialization/deserialization API - 4. Cache downloaded metadata files manually, let DNF5 reload from cache - - """ - repo_id = repo["id"] - # Get available variants from composeinfo.json (if configured) - available_variants = self._get_available_compose_variants(repo, arch) + exists = True + + if repo_id not in self.global_dnf_repo_cache: + exists = False + self.global_dnf_repo_cache[repo_id] = {} - for repo_name, repo_data in repo["source"]["repos"].items(): - if repo_data["limit_arches"] and arch not in repo_data["limit_arches"]: - continue + elif arch not in self.global_dnf_repo_cache[repo_id]: + exists = False + + if exists: + #log(" Loading repos from cache...") - # Check if variant exists in compose (if composeinfo is available) - if available_variants and not available_variants.get(repo_name, True): - log(f" Skipping {repo_name} on {arch} (not in compose)") - continue + for repo in self.global_dnf_repo_cache[repo_id][arch]: + base.repos.add(repo) - config = base.get_config() - repo_sack = base.get_repo_sack() - additional_repo = repo_sack.create_repo(repo_name) - repo_config = additional_repo.get_config() - repo_config.get_baseurl_option().set([repo_data["baseurl"]]) - repo_config.get_priority_option().set(repo_data["priority"]) - # DNF5: Set excludes via repo config, not repo_sack - if repo_data["exclude"]: - repo_config.get_excludepkgs_option().set(repo_data["exclude"]) + else: + #log(" Loading repos using DNF...") + for repo_name, repo_data in repo["source"]["repos"].items(): + if repo_data["limit_arches"]: + if arch not in repo_data["limit_arches"]: + #log(" Skipping {} on {}".format(repo_name, arch)) + continue + #log(" Including {}".format(repo_name)) + + additional_repo = dnf.repo.Repo( + name=repo_name, + parent_conf=base.conf + ) + additional_repo.baseurl = repo_data["baseurl"] + additional_repo.priority = repo_data["priority"] + additional_repo.exclude = repo_data["exclude"] + base.repos.add(additional_repo) + + # Additional repository (if configured) + #if repo["source"]["additional_repository"]: + # additional_repo = dnf.repo.Repo(name="additional-repository",parent_conf=base.conf) + # additional_repo.baseurl = [repo["source"]["additional_repository"]] + # additional_repo.priority = 1 + # base.repos.add(additional_repo) + + # All other system repos + #base.read_all_repos() + + self.global_dnf_repo_cache[repo_id][arch] = [] + for repo in base.repos.iter_enabled(): + self.global_dnf_repo_cache[repo_id][arch].append(repo) + def _analyze_pkgs(self, repo, arch): - log(f"Analyzing pkgs for {repo['name']} ({repo['id']}) {arch}") - - # TODO: Move away from context manager, only implemented to reduce changes from DNF4-> DNF5 - with dnf5_base() as base: - config = base.get_config() - config.get_debuglevel_option().set(0) - # Note: DNF5 doesn't have errorlevel/logfilelevel in same way + log("Analyzing pkgs for {repo_name} ({repo_id}) {arch}".format( + repo_name=repo["name"], + repo_id=repo["id"], + arch=arch + )) + + with dnf.Base() as base: + + base.conf.debuglevel = 0 + base.conf.errorlevel = 0 + base.conf.logfilelevel = 0 # Local DNF cache - cachedir_name = f"dnf_cachedir-{repo['id']}-{arch}" - config.get_cachedir_option().set(os.path.join(self.tmp_dnf_cachedir, cachedir_name)) + cachedir_name = "dnf_cachedir-{repo}-{arch}".format( + repo=repo["id"], + arch=arch + ) + base.conf.cachedir = os.path.join(self.tmp_dnf_cachedir, cachedir_name) # Generic installroot - root_name = f"dnf_generic_installroot-{repo['id']}-{arch}" - config.get_installroot_option().set(os.path.join(self.tmp_installroots, root_name)) - - # Architecture and Releasever - vars = base.get_vars() - vars.set("arch", arch) - vars.set("basearch", arch) - vars.set("releasever", repo["source"]["releasever"]) + root_name = "dnf_generic_installroot-{repo}-{arch}".format( + repo=repo["id"], + arch=arch + ) + base.conf.installroot = os.path.join(self.tmp_installroots, root_name) - config.get_ignorearch_option().set(True) + # Architecture + base.conf.arch = arch + base.conf.ignorearch = True - # DNF5: setup() must be called after configuration but before using repo_sack - base.setup() - # Get available variants from composeinfo.json (if configured) - available_variants = self._get_available_compose_variants(repo, arch) + # Releasever + base.conf.substitutions['releasever'] = repo["source"]["releasever"] - # Initialize repo_sack & repo_names_to_load - repo_sack = base.get_repo_sack() - repo_names_to_load = [] for repo_name, repo_data in repo["source"]["repos"].items(): - if repo_data["limit_arches"] and arch not in repo_data["limit_arches"]: - log(f" Skipping {repo_name} on {arch}") - continue - - # Check if variant exists in compose (if composeinfo is available) - if available_variants and not available_variants.get(repo_name, True): - log(f" Skipping {repo_name} on {arch} (not in compose)") - continue - - + if repo_data["limit_arches"]: + if arch not in repo_data["limit_arches"]: + log(f" Skipping {repo_name} on {arch}") + continue log(f" Including {repo_name}") - new_repo = repo_sack.create_repo(repo_name) - repo_config = new_repo.get_config() - repo_config.get_baseurl_option().set([repo_data["baseurl"]]) - repo_config.get_priority_option().set(repo_data["priority"]) - repo_names_to_load.append(repo_name) + additional_repo = dnf.repo.Repo( + name=repo_name, + parent_conf=base.conf + ) + additional_repo.baseurl = repo_data["baseurl"] + additional_repo.priority = repo_data["priority"] + base.repos.add(additional_repo) + + # Additional repository (if configured) + #if repo["source"]["additional_repository"]: + # additional_repo = dnf.repo.Repo(name="additional-repository",parent_conf=base.conf) + # additional_repo.baseurl = [repo["source"]["additional_repository"]] + # additional_repo.priority = 1 + # base.repos.add(additional_repo) # Load repos log(" Loading repos...") + #base.read_all_repos() + - # At this stage, we need to get all packages from the repo listed. + # At this stage, I need to get all packages from the repo listed. # That also includes modular packages. Modular packages in non-enabled - # streams would be normally hidden. So we mark all the available repos as + # streams would be normally hidden. So I mark all the available repos as # hotfix repos to make all packages visible, including non-enabled streams. - # DNF5: Iterate repos using RepoQuery - repo_query = RepoQuery(base) - for repo_weak_ptr in repo_query: - repo_obj = repo_weak_ptr.get() - # DNF5: module_hotfixes is a config option - repo_obj.get_config().get_module_hotfixes_option().set(True) + for dnf_repo in base.repos.all(): + dnf_repo.module_hotfixes = True # This sometimes fails, so let's try at least N times # before totally giving up! - # If individual repos fail, disable them and continue with others - max_tries = 10 + MAX_TRIES = 10 attempts = 0 success = False - failed_repos = [] - - while attempts < max_tries and not success: + while attempts < MAX_TRIES: try: - # DNF5: load repos instead of fill_sack - repo_sack.load_repos() + base.fill_sack(load_system_repo=False) success = True break - except (UserAssertionError, DnfErr) as err: - attempts += 1 - error_msg = str(err) - log(f" Failed to download repodata (attempt {attempts}/{max_tries}). Error: {error_msg}") - - # Try to identify which repo failed and disable it - for repo_name in repo_names_to_load: - if repo_name in error_msg: - if repo_name not in failed_repos: - log(f" Disabling problematic repository: {repo_name}") - failed_repos.append(repo_name) - try: - repo_query = RepoQuery(base) - for repo_weak_ptr in repo_query: - repo_obj = repo_weak_ptr.get() - if repo_obj.get_id() == repo_name: - repo_obj.disable() - break - except Exception as disable_err: - log(f" Warning: Could not disable repo {repo_name}: {disable_err}") - break - + except dnf.exceptions.RepoError as err: + attempts +=1 + log(" Failed to download repodata. Trying again!") if not success: - # If we still failed after trying to disable problematic repos, give up - err = f"Failed to download repodata while analyzing repo '{repo['name']} ({repo['id']}) {arch}'" + err = "Failed to download repodata while analyzing repo '{repo_name} ({repo_id}) {arch}".format( + repo_name=repo["name"], + repo_id=repo["id"], + arch=arch + ) err_log(err) raise RepoDownloadError(err) - if failed_repos: - log(f" WARNING: Proceeding without repositories: {', '.join(failed_repos)}") - - # DNF5: Create PackageQuery instead of using base.sack.query - query = PackageQuery(base) + # DNF query + query = base.sack.query # Get all packages - # DNF5: PackageQuery is directly iterable, don't call it - all_pkgs_set = set(query) + all_pkgs_set = set(query()) pkgs = {} for pkg_object in all_pkgs_set: - pkg_nevra = f"{pkg_object.get_name()}-{pkg_object.get_evr()}.{pkg_object.get_arch()}" - pkg_nevr = f"{pkg_object.get_name()}-{pkg_object.get_evr()}" - - pkgs[pkg_nevra] = { - "id": pkg_nevra, - "name": pkg_object.get_name(), - "evr": pkg_object.get_evr(), - "nevr": pkg_nevr, - "arch": pkg_object.get_arch(), - "installsize": pkg_object.get_install_size(), - "description": pkg_object.get_description(), - "summary": pkg_object.get_summary(), - "source_name": pkg_object.get_source_name(), - "sourcerpm": pkg_object.get_sourcerpm(), - "reponame": pkg_object.get_repo_id(), - } - + pkg_nevra = "{name}-{evr}.{arch}".format( + name=pkg_object.name, + evr=pkg_object.evr, + arch=pkg_object.arch + ) + pkg_nevr = "{name}-{evr}".format( + name=pkg_object.name, + evr=pkg_object.evr + ) + pkg = {} + pkg["id"] = pkg_nevra + pkg["name"] = pkg_object.name + pkg["evr"] = pkg_object.evr + pkg["nevr"] = pkg_nevr + pkg["arch"] = pkg_object.arch + pkg["installsize"] = pkg_object.installsize + pkg["description"] = pkg_object.description + #pkg["provides"] = pkg_object.provides + #pkg["requires"] = pkg_object.requires + #pkg["recommends"] = pkg_object.recommends + #pkg["suggests"] = pkg_object.suggests + pkg["summary"] = pkg_object.summary + pkg["source_name"] = pkg_object.source_name + pkg["sourcerpm"] = pkg_object.sourcerpm + pkg["reponame"] = pkg_object.reponame + + pkgs[pkg_nevra] = pkg + # There shouldn't be multiple packages of the same NVR # But the world isn't as simple! So add all reponames # to every package, in case it's in multiple repos - repo_priorities = { - repo_name: repo_data["priority"] for repo_name, repo_data in repo["source"]["repos"].items() - } + repo_priorities = {} + for repo_name, repo_data in repo["source"]["repos"].items(): + repo_priorities[repo_name] = repo_data["priority"] for pkg_object in all_pkgs_set: - pkg_nevra = f"{pkg_object.get_name()}-{pkg_object.get_evr()}.{pkg_object.get_arch()}" - reponame = pkg_object.get_repo_id() - pkgs[pkg_nevra].setdefault("all_reponames", set()).add(reponame) - + pkg_nevra = "{name}-{evr}.{arch}".format( + name=pkg_object.name, + evr=pkg_object.evr, + arch=pkg_object.arch + ) + reponame = pkg_object.reponame + + if "all_reponames" not in pkgs[pkg_nevra]: + pkgs[pkg_nevra]["all_reponames"] = set() + + pkgs[pkg_nevra]["all_reponames"].add(reponame) + for pkg_nevra, pkg in pkgs.items(): - all_repo_priorities = {repo_priorities[reponame] for reponame in pkg["all_reponames"]} + pkgs[pkg_nevra]["highest_priority_reponames"] = set() - highest_repo_priority = min(all_repo_priorities) + all_repo_priorities = set() + for reponame in pkg["all_reponames"]: + all_repo_priorities.add(repo_priorities[reponame]) + + highest_repo_priority = sorted(list(all_repo_priorities))[0] - pkgs[pkg_nevra]["highest_priority_reponames"] = { - reponame for reponame in pkg["all_reponames"] if repo_priorities[reponame] == highest_repo_priority - } + for reponame in pkg["all_reponames"]: + if repo_priorities[reponame] == highest_repo_priority: + pkgs[pkg_nevra]["highest_priority_reponames"].add(reponame) - log(f" Done! ({len(pkgs)} packages in total)") + log(" Done! ({pkg_count} packages in total)".format( + pkg_count=len(pkgs) + )) log("") return pkgs - + def _analyze_repos(self): self.data["repos"] = {} - for repo in self.configs["repos"].values(): + for _,repo in self.configs["repos"].items(): repo_id = repo["id"] - self.data["pkgs"][repo_id] = { - arch: self._analyze_pkgs(repo, arch) for arch in repo["source"]["architectures"] - } + self.data["pkgs"][repo_id] = {} self.data["repos"][repo_id] = {} - + for arch in repo["source"]["architectures"]: + self.data["pkgs"][repo_id][arch] = self._analyze_pkgs(repo, arch) + # Reading the optional composeinfo self.data["repos"][repo_id]["compose_date"] = None self.data["repos"][repo_id]["compose_days_ago"] = 0 if repo["source"]["composeinfo"]: # At this point, this is all I can do. Hate me or not, it gets us - # what we need and won't break anything in case things go badly. + # what we need and won't brake anything in case things go badly. request = urllib.request.Request(repo["source"]["composeinfo"]) request.add_header("Accept", "application/json") request.add_header("User-Agent", "ContentResolver/1.0") @@ -888,43 +658,69 @@ def _analyze_repos(self): except: pass - def _analyze_package_relations(self, packages, package_placeholders=None): - """ - Analyze package relationships for the given set of packages. - - Args: - packages: Iterable of DNF5 package objects (e.g., set, PackageQuery) - package_placeholders: Optional dict of placeholder packages - - Returns: - dict: Package relations mapping pkg_id -> relation data - - Note: Accepts any iterable of package objects - typically the actual packages - selected by DNF during installation. Do NOT pass a query filtered by name, - as that would include all versions from all repos (including low-priority duplicates). - """ - # TODO: Implement DNF5 package relationship analysis - # DNF5 PackageQuery.filter() API is different - # Temporarily returning minimal relations structure to test rest of migration - # DNF5 Migration Note: Unlike DNF4's filter(requires=[pkg]) API, DNF5 requires - # manual iteration to build reverse dependency maps. We invert the relationship: - # instead of asking "each dependency, record "package Y is required by package X". - # who requires package X?", we iterate all packages and for + def _analyze_package_relations(self, dnf_query, package_placeholders = None): relations = {} - # Create empty relation entries for all packages - for pkg in packages: - pkg_id = f"{pkg.get_name()}-{pkg.get_evr()}.{pkg.get_arch()}" + for pkg in dnf_query: + pkg_id = "{name}-{evr}.{arch}".format( + name=pkg.name, + evr=pkg.evr, + arch=pkg.arch + ) + + required_by = set() + recommended_by = set() + suggested_by = set() + supplements = set() + + for dep_pkg in dnf_query.filter(requires=[pkg]): + dep_pkg_id = "{name}-{evr}.{arch}".format( + name=dep_pkg.name, + evr=dep_pkg.evr, + arch=dep_pkg.arch + ) + required_by.add(dep_pkg_id) + + if self._global_performance_hack_run_recommends_queries: + for dep_pkg in dnf_query.filter(recommends=[pkg]): + dep_pkg_id = "{name}-{evr}.{arch}".format( + name=dep_pkg.name, + evr=dep_pkg.evr, + arch=dep_pkg.arch + ) + recommended_by.add(dep_pkg_id) + + #for dep_pkg in dnf_query.filter(suggests=[pkg]): + # dep_pkg_id = "{name}-{evr}.{arch}".format( + # name=dep_pkg.name, + # evr=dep_pkg.evr, + # arch=dep_pkg.arch + # ) + # suggested_by.add(dep_pkg_id) + + # Find packages that this pkg supplements + for supplement_reldep in pkg.supplements: + # Find packages in the query that provide this supplement + providing_pkgs = dnf_query.filter(provides=[supplement_reldep]) + for providing_pkg in providing_pkgs: + supplement_pkg_id = "{name}-{evr}.{arch}".format( + name=providing_pkg.name, + evr=providing_pkg.evr, + arch=providing_pkg.arch + ) + supplements.add(supplement_pkg_id) + relations[pkg_id] = {} - relations[pkg_id]["required_by"] = [] - relations[pkg_id]["recommended_by"] = [] + relations[pkg_id]["required_by"] = sorted(list(required_by)) + relations[pkg_id]["recommended_by"] = sorted(list(recommended_by)) + relations[pkg_id]["supplements"] = sorted(list(supplements)) + #relations[pkg_id]["suggested_by"] = sorted(list(suggested_by)) relations[pkg_id]["suggested_by"] = [] - relations[pkg_id]["supplements"] = [] - relations[pkg_id]["source_name"] = pkg.get_source_name() - relations[pkg_id]["reponame"] = pkg.get_repo_id() - + relations[pkg_id]["source_name"] = pkg.source_name + relations[pkg_id]["reponame"] = pkg.reponame + if package_placeholders: - for placeholder_name, placeholder_data in package_placeholders.items(): + for placeholder_name,placeholder_data in package_placeholders.items(): placeholder_id = pkg_placeholder_name_to_id(placeholder_name) relations[placeholder_id] = {} @@ -933,14 +729,23 @@ def _analyze_package_relations(self, packages, package_placeholders=None): relations[placeholder_id]["suggested_by"] = [] relations[placeholder_id]["supplements"] = [] relations[placeholder_id]["reponame"] = None - + + # TODO: triple for loop!!!! + for placeholder_name,placeholder_data in package_placeholders.items(): + placeholder_id = pkg_placeholder_name_to_id(placeholder_name) + for placeholder_dependency_name in placeholder_data["requires"]: + for pkg_id in relations: + pkg_name = pkg_id_to_name(pkg_id) + if pkg_name == placeholder_dependency_name: + relations[pkg_id]["required_by"].append(placeholder_id) + return relations def _analyze_env_without_leaking(self, env_conf, repo, arch): # DNF leaks memory and file descriptors :/ - # + # # So, this workaround runs it in a subprocess that should have its resources # freed when done! @@ -952,8 +757,10 @@ def _analyze_env_without_leaking(self, env_conf, repo, arch): # This basically means there was an exception in the processing and the process crashed if queue_result.empty(): raise AnalysisError + + env = queue_result.get() - return queue_result.get() + return env def _analyze_env_process(self, queue_result, env_conf, repo, arch): @@ -961,9 +768,10 @@ def _analyze_env_process(self, queue_result, env_conf, repo, arch): env = self._analyze_env(env_conf, repo, arch) queue_result.put(env) + def _analyze_env(self, env_conf, repo, arch): env = {} - + env["env_conf_id"] = env_conf["id"] env["pkg_ids"] = [] env["repo_id"] = repo["id"] @@ -976,133 +784,110 @@ def _analyze_env(self, env_conf, repo, arch): env["succeeded"] = True - # TODO: migrate away from context manager - with dnf5_base() as base: - config = base.get_config() - config.get_debuglevel_option().set(0) + with dnf.Base() as base: + + base.conf.debuglevel = 0 + base.conf.errorlevel = 0 + base.conf.logfilelevel = 0 # Local DNF cache - cachedir_name = f"dnf_cachedir-{repo['id']}-{arch}" - config.get_cachedir_option().set(os.path.join(self.tmp_dnf_cachedir, cachedir_name)) + cachedir_name = "dnf_cachedir-{repo}-{arch}".format( + repo=repo["id"], + arch=arch + ) + base.conf.cachedir = os.path.join(self.tmp_dnf_cachedir, cachedir_name) # Environment installroot - root_name = f"dnf_env_installroot-{env_conf['id']}-{repo['id']}-{arch}" - config.get_installroot_option().set(os.path.join(self.tmp_installroots, root_name)) + root_name = "dnf_env_installroot-{env_conf}-{repo}-{arch}".format( + env_conf=env_conf["id"], + repo=repo["id"], + arch=arch + ) + base.conf.installroot = os.path.join(self.tmp_installroots, root_name) - # Architecture and Releasever - vars = base.get_vars() - vars.set("arch", arch) - vars.set("basearch", arch) - vars.set("releasever", repo["source"]["releasever"]) + # Architecture + base.conf.arch = arch + base.conf.ignorearch = True - config.get_ignorearch_option().set(True) + # Releasever + base.conf.substitutions['releasever'] = repo["source"]["releasever"] # Additional DNF Settings - # Note: DNF5 tsflags are handled differently - these affect transaction behavior - # justdb and noscripts are implicit in how DNF5 handles test transactions + base.conf.tsflags.append('justdb') + base.conf.tsflags.append('noscripts') # Environment config if "include-weak-deps" not in env_conf["options"]: - config.get_install_weak_deps_option().set(False) - # Note: nodocs tsflags - DNF5 handles this via separate config - - # DNF5: setup() must be called after configuration but before using repo_sack - base.setup() + base.conf.install_weak_deps = False + if "include-docs" not in env_conf["options"]: + base.conf.tsflags.append('nodocs') # Load repos + #log(" Loading repos...") + #base.read_all_repos() self._load_repo_cached(base, repo, arch) - # Build list of repo names for error handling - repo_names_to_load = [] - for repo_name, repo_data in repo["source"]["repos"].items(): - if repo_data["limit_arches"] and arch not in repo_data["limit_arches"]: - continue - repo_names_to_load.append(repo_name) - # This sometimes fails, so let's try at least N times # before totally giving up! - max_tries = 10 + MAX_TRIES = 10 attempts = 0 success = False - failed_repos = [] - while attempts < max_tries: + while attempts < MAX_TRIES: try: - # DNF5: load repos instead of fill_sack - repo_sack = base.get_repo_sack() - repo_sack.load_repos() + base.fill_sack(load_system_repo=False) success = True break - except (UserAssertionError, DnfErr, RuntimeError) as err: - attempts += 1 - error_msg = str(err) - log(f" Failed to download repodata (attempt {attempts}/{max_tries}). Error: {err}") - - # Try to identify which repo failed and disable it - for repo_name in repo_names_to_load: - if repo_name in error_msg: - if repo_name not in failed_repos: - log(f" Disabling problematic repository: {repo_name}") - failed_repos.append(repo_name) - try: - repo_query = RepoQuery(base) - for repo_weak_ptr in repo_query: - repo_obj = repo_weak_ptr.get() - if repo_obj.get_id() == repo_name: - repo_obj.disable() - break - except Exception as disable_err: - log(f" Warning: Could not disable repo {repo_name}: {disable_err}") - break - + except dnf.exceptions.RepoError as err: + attempts +=1 + log(" Failed to download repodata. Trying again!") if not success: - err = f"Failed to download repodata while analyzing environment '{env_conf['id']}' from '{repo['id']}' {arch}:" + err = "Failed to download repodata while analyzing environment '{env_conf}' from '{repo}' {arch}:".format( + env_conf=env_conf["id"], + repo=repo["id"], + arch=arch + ) err_log(err) raise RepoDownloadError(err) - if failed_repos: - log(f" WARNING: Proceeding without repositories: {', '.join(failed_repos)}") - - # DNF5: Create a Goal for package operations - goal = Goal(base) # Packages log(" Adding packages...") for pkg in env_conf["packages"]: - # DNF5: Check if package is resolvable (by name or provides) before adding - if not is_package_resolvable(base, pkg): + try: + base.install(pkg) + except dnf.exceptions.MarkingError: env["errors"]["non_existing_pkgs"].append(pkg) continue - goal.add_install(pkg) - + # Groups log(" Adding groups...") if env_conf["groups"]: - # DNF5: Groups are loaded as part of repos - pass + base.read_comps(arch_filter=True) for grp_spec in env_conf["groups"]: - try: - # DNF5: add_group_install takes spec and options - settings = GoalJobSettings() - goal.add_group_install(grp_spec, settings) - except (UserAssertionError, DnfErr): + group = base.comps.group_by_pattern(grp_spec) + if not group: env["errors"]["non_existing_pkgs"].append(grp_spec) continue + base.group_install(group.id, ['mandatory', 'default']) # Architecture-specific packages for pkg in env_conf["arch_packages"][arch]: - # DNF5: Check if package is resolvable (by name or provides) before adding - if not is_package_resolvable(base, pkg): + try: + base.install(pkg) + except dnf.exceptions.MarkingError: env["errors"]["non_existing_pkgs"].append(pkg) continue - goal.add_install(pkg) - + # Resolve dependencies log(" Resolving dependencies...") try: - # DNF5: resolve via goal - transaction = goal.resolve() - except DnfErr as err: - err_log(f"Failed to analyze environment '{env_conf['id']}' from '{repo['id']}' {arch}:") + base.resolve() + except dnf.exceptions.DepsolveError as err: + err_log("Failed to analyze environment '{env_conf}' from '{repo}' {arch}:".format( + env_conf=env_conf["id"], + repo=repo["id"], + arch=arch + )) err_log(f" - {err}") env["succeeded"] = False env["errors"]["message"] = str(err) @@ -1113,10 +898,13 @@ def _analyze_env(self, env_conf, repo, arch): # So let's do that to make it happy. log(" Downloading packages...") try: - # DNF5: download packages from transaction - transaction.download() - except Dnf5RepoDownloadError as err: - err_log(f"Failed to analyze environment '{env_conf['id']}' from '{repo['id']}' {arch}:") + base.download_packages(base.transaction.install_set) + except dnf.exceptions.DownloadError as err: + err_log("Failed to analyze environment '{env_conf}' from '{repo}' {arch}:".format( + env_conf=env_conf["id"], + repo=repo["id"], + arch=arch + )) err_log(f" - {err}") env["succeeded"] = False env["errors"]["message"] = str(err) @@ -1124,38 +912,40 @@ def _analyze_env(self, env_conf, repo, arch): log(" Running DNF transaction, writing RPMDB...") try: - # DNF5: run transaction - transaction.run() - except BaseTransactionError as err: - err_log(f"Failed to analyze environment '{env_conf['id']}' from '{repo['id']}' {arch}:") + base.do_transaction() + except (dnf.exceptions.TransactionCheckError, dnf.exceptions.Error) as err: + err_log("Failed to analyze environment '{env_conf}' from '{repo}' {arch}:".format( + env_conf=env_conf["id"], + repo=repo["id"], + arch=arch + )) err_log(f" - {err}") env["succeeded"] = False env["errors"]["message"] = str(err) return env - # Get packages from transaction - log(" Extracting packages from transaction...") - # DNF5: transaction packages vector is directly indexable - pkg_list = [] - trans_pkgs = transaction.get_transaction_packages() - for i in range(trans_pkgs.size()): - trans_pkg = trans_pkgs[i] - pkg_list.append(trans_pkg.get_package()) - - # Use the actual packages from the transaction. - # Don't re-query by name - that would return ALL versions from ALL repos, - # including lower-priority duplicates (e.g., both ELN and Rawhide versions). - for pkg in pkg_list: - pkg_id = f"{pkg.get_name()}-{pkg.get_evr()}.{pkg.get_arch()}" + # DNF Query + log(" Creating a DNF Query object...") + query = base.sack.query().filterm(pkg=base.transaction.install_set) + + for pkg in query: + pkg_id = "{name}-{evr}.{arch}".format( + name=pkg.name, + evr=pkg.evr, + arch=pkg.arch + ) env["pkg_ids"].append(pkg_id) + + env["pkg_relations"] = self._analyze_package_relations(query) - env["pkg_relations"] = self._analyze_package_relations(pkg_list) - - log(f" Done! ({len(env['pkg_ids'])} packages in total)") + log(" Done! ({pkg_count} packages in total)".format( + pkg_count=len(env["pkg_ids"]) + )) log("") - + return env + def _analyze_envs(self): envs = {} @@ -1171,201 +961,167 @@ def _analyze_envs(self): # repos each config lists * # archeas each repo supports # Analyze all of that! - log(f"Analyzing {env_conf['name']} ({env_conf_id}) from {repo['name']} ({repo_id}) {arch}...") - - env_id = f"{env_conf_id}:{repo_id}:{arch}" + log("Analyzing {env_name} ({env_id}) from {repo_name} ({repo}) {arch}...".format( + env_name=env_conf["name"], + env_id=env_conf_id, + repo_name=repo["name"], + repo=repo_id, + arch=arch + )) + + env_id = "{env_conf_id}:{repo_id}:{arch}".format( + env_conf_id=env_conf_id, + repo_id=repo_id, + arch=arch + ) envs[env_id] = self._analyze_env(env_conf, repo, arch) - + self.data["envs"] = envs + def _return_failed_workload_env_err(self, workload_conf, env_conf, repo, arch): - workload = { - "workload_conf_id": workload_conf["id"], - "env_conf_id": env_conf["id"], - "repo_id": repo["id"], - "arch": arch, - "pkg_env_ids": [], - "pkg_added_ids": [], - "pkg_placeholder_ids": [], - "pkg_relations": [], - "errors": { - "non_existing_pkgs": [], - "message": """ + workload = {} + + workload["workload_conf_id"] = workload_conf["id"] + workload["env_conf_id"] = env_conf["id"] + workload["repo_id"] = repo["id"] + workload["arch"] = arch + + workload["pkg_env_ids"] = [] + workload["pkg_added_ids"] = [] + workload["pkg_placeholder_ids"] = [] + + workload["pkg_relations"] = [] + + workload["errors"] = {} + workload["errors"]["non_existing_pkgs"] = [] + workload["succeeded"] = False + workload["env_succeeded"] = False + + workload["errors"]["message"] = """ Failed to analyze this workload because of an error while analyzing the environment. Please see the associated environment results for a detailed error message. - """, - }, - "succeeded": False, - "env_succeeded": False, - } + """ return workload def _analyze_workload(self, workload_conf, env_conf, repo, arch): + workload = {} + + workload["workload_conf_id"] = workload_conf["id"] + workload["env_conf_id"] = env_conf["id"] + workload["repo_id"] = repo["id"] + workload["arch"] = arch + + workload["pkg_env_ids"] = [] + workload["pkg_added_ids"] = [] + workload["pkg_placeholder_ids"] = [] + workload["srpm_placeholder_names"] = [] + + workload["pkg_relations"] = [] + + workload["errors"] = {} + workload["errors"]["non_existing_pkgs"] = [] + workload["errors"]["non_existing_placeholder_deps"] = [] + + workload["warnings"] = {} + workload["warnings"]["non_existing_pkgs"] = [] + workload["warnings"]["non_existing_placeholder_deps"] = [] + workload["warnings"]["message"] = None + + workload["succeeded"] = True + workload["env_succeeded"] = True + + # Figure out the workload labels # It can only have labels that are in both the workload_conf and the env_conf - workload = { - "workload_conf_id": workload_conf["id"], - "env_conf_id": env_conf["id"], - "repo_id": repo["id"], - "arch": arch, - "pkg_env_ids": [], - "pkg_added_ids": [], - "pkg_placeholder_ids": [], - "srpm_placeholder_names": [], - "pkg_relations": [], - "errors": { - "non_existing_pkgs": [], - "non_existing_placeholder_deps": [], - }, - "warnings": { - "non_existing_pkgs": [], - "non_existing_placeholder_deps": [], - "message": None, - }, - "succeeded": True, - "env_succeeded": True, - "labels": list(set(workload_conf["labels"]) & set(env_conf["labels"])), - } + workload["labels"] = list(set(workload_conf["labels"]) & set(env_conf["labels"])) - with dnf5_base() as base: - config = base.get_config() - config.get_debuglevel_option().set(0) + with dnf.Base() as base: + + base.conf.debuglevel = 0 + base.conf.errorlevel = 0 + base.conf.logfilelevel = 0 # Local DNF cache - cachedir_name = f"dnf_cachedir-{repo['id']}-{arch}" - config.get_cachedir_option().set(os.path.join(self.tmp_dnf_cachedir, cachedir_name)) + cachedir_name = "dnf_cachedir-{repo}-{arch}".format( + repo=repo["id"], + arch=arch + ) + base.conf.cachedir = os.path.join(self.tmp_dnf_cachedir, cachedir_name) # Environment installroot # Since we're not writing anything into the installroot, # let's just use the base image's installroot! - root_name = f"dnf_env_installroot-{env_conf['id']}-{repo['id']}-{arch}" - config.get_installroot_option().set(os.path.join(self.tmp_installroots, root_name)) + root_name = "dnf_env_installroot-{env_conf}-{repo}-{arch}".format( + env_conf=env_conf["id"], + repo=repo["id"], + arch=arch + ) + base.conf.installroot = os.path.join(self.tmp_installroots, root_name) - # Architecture and Releasever - vars = base.get_vars() - vars.set("arch", arch) - vars.set("basearch", arch) - vars.set("releasever", repo["source"]["releasever"]) + # Architecture + base.conf.arch = arch + base.conf.ignorearch = True - config.get_ignorearch_option().set(True) + # Releasever + base.conf.substitutions['releasever'] = repo["source"]["releasever"] # Environment config if "include-weak-deps" not in workload_conf["options"]: - config.get_install_weak_deps_option().set(False) - # Note: nodocs is handled differently in DNF5 - - # DNF5: setup() must be called after configuration but before using repo_sack - base.setup() + base.conf.install_weak_deps = False + if "include-docs" not in workload_conf["options"]: + base.conf.tsflags.append('nodocs') # Load repos + #log(" Loading repos...") + #base.read_all_repos() self._load_repo_cached(base, repo, arch) - # Build list of repo names for error handling - repo_names_to_load = [] - for repo_name, repo_data in repo["source"]["repos"].items(): - if repo_data["limit_arches"] and arch not in repo_data["limit_arches"]: - continue - repo_names_to_load.append(repo_name) - - # 0 % + # 0 % # Now I need to load the local RPMDB. # However, if the environment is empty, it wasn't created, so I need to treat # it differently. So let's check! - repo_sack = base.get_repo_sack() if len(env_conf["packages"]) or len(env_conf["arch_packages"][arch]) or len(env_conf["groups"]): # It's not empty! Load local data. - # DNF5: This loads both repos and system data - # This sometimes fails, so let's try at least N times with repo-disabling logic - max_tries = 10 - attempts = 0 - success = False - failed_repos = [] - while attempts < max_tries: - try: - repo_sack.load_repos() - success = True - break - except (UserAssertionError, DnfErr, RuntimeError, Dnf5RepoDownloadError) as err: - attempts += 1 - error_msg = str(err) - # log(f" Failed to download repodata (attempt {attempts}/{MAX_TRIES}). Error: {error_msg}") - - # Try to identify which repo failed and disable it - for repo_name in repo_names_to_load: - if repo_name in error_msg: - if repo_name not in failed_repos: - # log(f" Disabling problematic repository: {repo_name}") - failed_repos.append(repo_name) - try: - repo_query = RepoQuery(base) - for repo_weak_ptr in repo_query: - repo_obj = repo_weak_ptr.get() - if repo_obj.get_id() == repo_name: - repo_obj.disable() - break - except Exception as disable_err: - pass - # log(f" Warning: Could not disable repo {repo_name}: {disable_err}") - break - - if not success: - err = (f"Failed to download repodata while analyzing workload '{workload['workload_conf_id']}' on " - f"'{workload['env_conf_id']}' from '{workload['repo_id']}' {workload['arch']}...") - err_log(err) - raise RepoDownloadError(err) + base.fill_sack(load_system_repo=True) else: # It's empty. Treat it like we're using an empty installroot. # This sometimes fails, so let's try at least N times # before totally giving up! - max_tries = 10 + MAX_TRIES = 10 attempts = 0 success = False - failed_repos = [] - while attempts < max_tries: + while attempts < MAX_TRIES: try: - repo_sack.load_repos() + base.fill_sack(load_system_repo=False) success = True break - except (UserAssertionError, DnfErr, RuntimeError, Dnf5RepoDownloadError) as err: - attempts += 1 - error_msg = str(err) - - # Try to identify which repo failed and disable it - for repo_name in repo_names_to_load: - if repo_name in error_msg: - if repo_name not in failed_repos: - failed_repos.append(repo_name) - try: - repo_query = RepoQuery(base) - for repo_weak_ptr in repo_query: - repo_obj = repo_weak_ptr.get() - if repo_obj.get_id() == repo_name: - repo_obj.disable() - break - except Exception: - pass - break - + except dnf.exceptions.RepoError as err: + attempts +=1 + #log(" Failed to download repodata. Trying again!") if not success: - err = (f"Failed to download repodata while analyzing workload '{workload['workload_conf_id']}' on " - f"'{workload['env_conf_id']}' from '{workload['repo_id']}' {workload['arch']}...") + err = "Failed to download repodata while analyzing workload '{workload_id} on '{env_id}' from '{repo}' {arch}...".format( + workload_id=workload_conf_id, + env_id=env_conf_id, + repo_name=repo["name"], + repo=repo_id, + arch=arch) err_log(err) raise RepoDownloadError(err) - + # 37 % - # DNF5: Create Goal for package operations - goal = Goal(base) - # Packages + #log(" Adding packages...") for pkg in workload_conf["packages"]: - # DNF5: Check if package is resolvable (by name or provides) before adding - if not is_package_resolvable(base, pkg): + try: + base.install(pkg) + except dnf.exceptions.MarkingError: if pkg in self.settings["weird_packages_that_can_not_be_installed"]: continue else: @@ -1374,23 +1130,26 @@ def _analyze_workload(self, workload_conf, env_conf, repo, arch): else: workload["warnings"]["non_existing_pkgs"].append(pkg) continue - goal.add_install(pkg) - + # Groups + #log(" Adding groups...") if workload_conf["groups"]: - # DNF5: Groups are loaded automatically with repos - pass + base.read_comps(arch_filter=True) for grp_spec in workload_conf["groups"]: - try: - # DNF5: add_group_install with settings - settings = GoalJobSettings() - goal.add_group_install(grp_spec, settings) - except (UserAssertionError, DnfErr): + group = base.comps.group_by_pattern(grp_spec) + if not group: workload["errors"]["non_existing_pkgs"].append(grp_spec) continue - - # TODO: Mark group packages as required... - + base.group_install(group.id, ['mandatory', 'default']) + + + # TODO: Mark group packages as required... the following code doesn't work + #for pkg in group.packages_iter(): + # print(pkg.name) + # workload_conf["packages"].append(pkg.name) + + + # Filter out the relevant package placeholders for this arch package_placeholders = {} for placeholder_name, placeholder_data in workload_conf["package_placeholders"]["pkgs"].items(): @@ -1400,7 +1159,7 @@ def _analyze_workload(self, workload_conf, env_conf, repo, arch): # otherwise it is limited. In that case, only add it if the current arch is on its list elif arch in placeholder_data["limit_arches"]: package_placeholders[placeholder_name] = placeholder_data - + # Same for SRPM placeholders srpm_placeholders = {} for placeholder_name, placeholder_data in workload_conf["package_placeholders"]["srpms"].items(): @@ -1412,176 +1171,109 @@ def _analyze_workload(self, workload_conf, env_conf, repo, arch): srpm_placeholders[placeholder_name] = placeholder_data # Dependencies of package placeholders + #log(" Adding package placeholder dependencies...") for placeholder_name, placeholder_data in package_placeholders.items(): for pkg in placeholder_data["requires"]: - # DNF5: Check if package is resolvable (by name or provides) before adding - if not is_package_resolvable(base, pkg): + try: + base.install(pkg) + except dnf.exceptions.MarkingError: if "strict" in workload_conf["options"]: workload["errors"]["non_existing_placeholder_deps"].append(pkg) else: workload["warnings"]["non_existing_placeholder_deps"].append(pkg) continue - goal.add_install(pkg) # Architecture-specific packages for pkg in workload_conf["arch_packages"][arch]: - # DNF5: Check if package is resolvable (by name or provides) before adding - if not is_package_resolvable(base, pkg): + try: + base.install(pkg) + except dnf.exceptions.MarkingError: if "strict" in workload_conf["options"]: workload["errors"]["non_existing_pkgs"].append(pkg) else: workload["warnings"]["non_existing_pkgs"].append(pkg) continue - goal.add_install(pkg) if workload["errors"]["non_existing_pkgs"] or workload["errors"]["non_existing_placeholder_deps"]: error_message_list = [] if workload["errors"]["non_existing_pkgs"]: error_message_list.append("The following required packages are not available:") for pkg_name in workload["errors"]["non_existing_pkgs"]: - pkg_string = f" - {pkg_name}" + pkg_string = " - {pkg_name}".format( + pkg_name=pkg_name + ) error_message_list.append(pkg_string) if workload["errors"]["non_existing_placeholder_deps"]: error_message_list.append("The following dependencies of package placeholders are not available:") for pkg_name in workload["errors"]["non_existing_placeholder_deps"]: - pkg_string = f" - {pkg_name}" + pkg_string = " - {pkg_name}".format( + pkg_name=pkg_name + ) error_message_list.append(pkg_string) error_message = "\n".join(error_message_list) workload["succeeded"] = False workload["errors"]["message"] = str(error_message) + #log(" Failed! (Error message will be on the workload results page.") + #log("") return workload - + if workload["warnings"]["non_existing_pkgs"] or workload["warnings"]["non_existing_placeholder_deps"]: error_message_list = [] if workload["warnings"]["non_existing_pkgs"]: error_message_list.append("The following required packages are not available (and were skipped):") for pkg_name in workload["warnings"]["non_existing_pkgs"]: - pkg_string = f" - {pkg_name}" + pkg_string = " - {pkg_name}".format( + pkg_name=pkg_name + ) error_message_list.append(pkg_string) - error_message_list.append("") - error_message_list.append("Note: Add 'strict' to the workload options to treat missing packages as errors.") if workload["warnings"]["non_existing_placeholder_deps"]: error_message_list.append("The following dependencies of package placeholders are not available (and were skipped):") - # TODO: use comprehension and `error_message_list.extend([])` for pkg_name in workload["warnings"]["non_existing_placeholder_deps"]: - pkg_string = f" - {pkg_name}" + pkg_string = " - {pkg_name}".format( + pkg_name=pkg_name + ) error_message_list.append(pkg_string) error_message = "\n".join(error_message_list) workload["warnings"]["message"] = str(error_message) - log(f" Warning: {len(workload['warnings']['non_existing_pkgs'])} packages not found and were skipped") + + # 37 % # Resolve dependencies + #log(" Resolving dependencies...") try: - # DNF5: resolve via goal - transaction = goal.resolve() - except (DnfErr, RuntimeError, Exception) as err: + base.resolve() + except dnf.exceptions.DepsolveError as err: workload["succeeded"] = False - # Enhanced error message for dependency failures - error_message = str(err) - - # Check if this is a dependency chain failure (missing transitive dependency) - if "nothing provides" in error_message.lower() or "but none of the providers can be installed" in error_message.lower(): - error_lines = ["Dependency resolution failed:", ""] - error_lines.append("This workload requires packages that have unmet dependencies.") - error_lines.append("Common causes:") - error_lines.append(" - A required package has been retired from Fedora") - error_lines.append(" - A dependency is missing or not yet built") - error_lines.append(" - Package maintainer needs to update dependencies") - error_lines.append("") - error_lines.append("Detailed error from DNF:") - error_lines.append("-" * 70) - error_lines.append(error_message) - workload["errors"]["message"] = "\n".join(error_lines) - else: - workload["errors"]["message"] = error_message - - log(f" Failed to resolve dependencies for {workload_conf['id']}") - # Show full error for debugging (truncate at 2000 chars if too long) - error_display = error_message if len(error_message) <= 2000 else error_message[:2000] + "...(truncated)" - log(f" Error: {error_display}") + workload["errors"]["message"] = str(err) + #log(" Failed! (Error message will be on the workload results page.") + #log("") return workload - # CRITICAL DNF4 vs DNF5 DIFFERENCE: - # DNF4 raises exceptions when packages can't be resolved due to dependency failures. - # DNF5 does NOT raise exceptions - it returns a transaction with problems recorded. - # If we don't check transaction.get_problems(), packages with unresolvable dependencies - # will be silently skipped, showing as "succeeded" with 0 packages installed. - # If not used it causes the packages that have unresolvable deps to show NO errors in DNF5 while we - # expect "nothing provides" errors. - if transaction.get_problems() > 0: - # Get error messages from resolve logs - resolve_logs = transaction.get_resolve_logs_as_strings() - error_message = "\n".join(resolve_logs) - - # DNF5 reports repo priority conflicts that DNF4 silently resolved. - # Filter out version conflicts between repos (expected with multi-repo setups). - # Keep real dependency failures (missing packages, broken deps). - - # Check if entire error message is about version conflicts between repos - error_lower = error_message.lower() - - # Pattern 1: "cannot install both X from RepoA and X from RepoB" - has_cannot_install_both = "cannot install both" in error_lower and " from " in error_lower - - # Pattern 2: Multi-line version conflicts like: - # "package X from RepoA requires Y = v1, but none of the providers can be installed" - # "package Z from RepoB requires Y = v2, but none of the providers can be installed" - # These indicate different repos wanting different versions of same dependency - lines = error_message.split('\n') - requires_lines = [l for l in lines if 'requires' in l.lower() and 'from' in l.lower()] - has_multi_version_conflict = len(requires_lines) >= 2 - - # Pattern 3: "conflicting requests" or "cannot install the best candidate" - has_conflict_markers = ("conflicting requests" in error_lower or - "cannot install the best candidate" in error_lower) - - # Real dependency errors that should NOT be filtered - has_nothing_provides = "nothing provides" in error_lower - has_package_already_installed = "already installed" in error_lower - - # Filter logic: ignore if it's ONLY repo conflicts, no real missing deps - is_repo_conflict = (has_cannot_install_both or - (has_multi_version_conflict and has_conflict_markers)) - is_real_error = has_nothing_provides - - if is_repo_conflict and not is_real_error: - # Repo priority conflict - expected behavior, ignore - log(" Ignoring repository priority conflict (version mismatch between repos)") - else: - # Real dependency failure - workload["succeeded"] = False - workload["errors"]["message"] = error_message - log(f" Failed to resolve dependencies for {workload_conf['id']}") - # Show truncated error for debugging - error_display = error_message if len(error_message) <= 500 else error_message[:500] + "...(truncated)" - log(f" Error: {error_display}") - return workload + # 43 % # DNF Query - # Get installed packages from the system repo - query_env = PackageQuery(base) - query_env.filter_installed() - pkgs_env = set(query_env) - - # Get packages being installed from transaction - # DNF5: transaction packages vector is directly indexable - pkgs_added = [] - trans_pkgs = transaction.get_transaction_packages() - for i in range(trans_pkgs.size()): - trans_pkg = trans_pkgs[i] - pkgs_added.append(trans_pkg.get_package()) - pkgs_added = set(pkgs_added) - + #log(" Creating a DNF Query object...") + query_env = base.sack.query() + pkgs_env = set(query_env.installed()) + pkgs_added = set(base.transaction.install_set) pkgs_all = set.union(pkgs_env, pkgs_added) - + query_all = base.sack.query().filterm(pkg=pkgs_all) + # OK all good so save stuff now - # TODO: use comprehensions & .extend for pkg in pkgs_env: - pkg_id = f"{pkg.get_name()}-{pkg.get_evr()}.{pkg.get_arch()}" + pkg_id = "{name}-{evr}.{arch}".format( + name=pkg.name, + evr=pkg.evr, + arch=pkg.arch + ) workload["pkg_env_ids"].append(pkg_id) - + for pkg in pkgs_added: - pkg_id = f"{pkg.get_name()}-{pkg.get_evr()}.{pkg.get_arch()}" + pkg_id = "{name}-{evr}.{arch}".format( + name=pkg.name, + evr=pkg.evr, + arch=pkg.arch + ) workload["pkg_added_ids"].append(pkg_id) # No errors so far? That means the analysis has succeeded, @@ -1589,51 +1281,41 @@ def _analyze_workload(self, workload_conf, env_conf, repo, arch): # (Failed workloads need to have empty results, that's why) for placeholder_name in package_placeholders: workload["pkg_placeholder_ids"].append(pkg_placeholder_name_to_id(placeholder_name)) - + for srpm_placeholder_name in srpm_placeholders: workload["srpm_placeholder_names"].append(srpm_placeholder_name) - # Use the actual packages that were installed/selected by DNF's priority logic. - # Don't re-query by name - that would return ALL versions from ALL repos, - # including lower-priority duplicates (e.g., both ELN and Rawhide versions). - workload["pkg_relations"] = self._analyze_package_relations(pkgs_all, package_placeholders) + # 43 % + + workload["pkg_relations"] = self._analyze_package_relations(query_all, package_placeholders) + # 100 % + + pkg_env_count = len(workload["pkg_env_ids"]) + pkg_added_count = len(workload["pkg_added_ids"]) + #log(" Done! ({pkg_count} packages in total. That's {pkg_env_count} in the environment, and {pkg_added_count} added.)".format( + # pkg_count=str(pkg_env_count + pkg_added_count), + # pkg_env_count=pkg_env_count, + # pkg_added_count=pkg_added_count + #)) + #log("") + + # How long do various parts take: + # 37 % - populatind DNF's base.sack + # 6 % - resolving deps + # 57 % - _analyze_package_relations with recommends + + # Removing recommends from _analyze_package_relations + # gets the total duration down to + # 64 % return workload + def _analyze_workload_process(self, queue_result, workload_conf, env_conf, repo, arch): - try: - workload = self._analyze_workload(workload_conf, env_conf, repo, arch) - queue_result.put(workload) - except Exception as e: - # Create a failed workload result instead of crashing - workload = { - "workload_conf_id": workload_conf["id"], - "env_conf_id": env_conf["id"], - "repo_id": repo["id"], - "arch": arch, - "pkg_env_ids": [], - "pkg_added_ids": [], - "pkg_placeholder_ids": [], - "srpm_placeholder_names": [], - "pkg_relations": [], - "errors": { - "non_existing_pkgs": [], - "non_existing_placeholder_deps": [], - "message": f"Workload analysis failed with exception:\n{type(e).__name__}: {str(e)}\n\nTraceback:\n{traceback.format_exc()}", - }, - "warnings": { - "non_existing_pkgs": [], - "non_existing_placeholder_deps": [], - "message": None, - }, - "succeeded": False, - "env_succeeded": False, - "labels": list(set(workload_conf["labels"]) & set(env_conf["labels"])), - } - queue_result.put(workload) - # Log error to stderr so it appears in logs - err_log(f" ERROR analyzing workload {workload_conf['id']}:{env_conf['id']}:{repo['id']}:{arch}-> {e}", file=sys.stderr) + + workload = self._analyze_workload(workload_conf, env_conf, repo, arch) + queue_result.put(workload) async def _analyze_workloads_subset_async(self, task_queue, results): @@ -1644,7 +1326,12 @@ async def _analyze_workloads_subset_async(self, task_queue, results): repo = task["repo"] arch = task["arch"] - workload_id = f"{workload_conf['id']}:{env_conf['id']}:{repo['id']}:{arch}" + workload_id = "{workload_conf_id}:{env_conf_id}:{repo_id}:{arch}".format( + workload_conf_id=workload_conf["id"], + env_conf_id=env_conf["id"], + repo_id=repo["id"], + arch=arch + ) # Max processes while True: @@ -1652,7 +1339,7 @@ async def _analyze_workloads_subset_async(self, task_queue, results): self.current_subprocesses += 1 break else: - await asyncio.sleep(0.1) + await asyncio.sleep(.1) # Log progress self.workload_queue_counter_current += 1 @@ -1672,17 +1359,17 @@ async def _analyze_workloads_subset_async(self, task_queue, results): # 2 seconds for _ in range(1, 20): if queue_result.empty(): - await asyncio.sleep(0.1) + await asyncio.sleep(.1) else: break - + # 20 seconds for _ in range(1, 20): if queue_result.empty(): await asyncio.sleep(1) else: break - + # 200 seconds for _ in range(1, 20): if queue_result.empty(): @@ -1693,13 +1380,12 @@ async def _analyze_workloads_subset_async(self, task_queue, results): self.current_subprocesses -= 1 # This basically means there was an exception in the processing and the process crashed - # or timed out (222 seconds total timeout) if queue_result.empty(): log("") log("") log("--------------------------------------------------------------------------") log("") - log("WARNING: Workload analysis timed out or crashed") + log("ERROR: Workload analysis failed") log("") log("Details:") log(f" workload_conf: {workload_conf['id']}") @@ -1707,40 +1393,16 @@ async def _analyze_workloads_subset_async(self, task_queue, results): log(f" repo: {repo['id']}") log(f" arch: {arch}") log("") - log("Creating failed workload entry and continuing...") + log("More details somewhere above.") log("") log("--------------------------------------------------------------------------") log("") log("") - - # Create a failed workload result instead of crashing - results[workload_id] = { - "workload_conf_id": workload_conf["id"], - "env_conf_id": env_conf["id"], - "repo_id": repo["id"], - "arch": arch, - "pkg_env_ids": [], - "pkg_added_ids": [], - "pkg_placeholder_ids": [], - "srpm_placeholder_names": [], - "pkg_relations": [], - "errors": { - "non_existing_pkgs": [], - "non_existing_placeholder_deps": [], - "message": f"Workload analysis timed out after 222 seconds or subprocess crashed", - }, - "warnings": { - "non_existing_pkgs": [], - "non_existing_placeholder_deps": [], - "message": None, - }, - "succeeded": False, - "env_succeeded": False, - "labels": list(set(workload_conf["labels"]) & set(env_conf["labels"])), - } - else: - workload = queue_result.get() - results[workload_id] = workload + sys.exit(1) + + workload = queue_result.get() + + results[workload_id] = workload async def _analyze_workloads_async(self, results): @@ -1749,21 +1411,24 @@ async def _analyze_workloads_async(self, results): for repo in self.workload_queue: for arch in self.workload_queue[repo]: + task_queue = self.workload_queue[repo][arch] - tasks.append(asyncio.create_task(self._analyze_workloads_subset_async(task_queue, results))) + tasks.append(asyncio.create_task(self._analyze_workloads_subset_async(task_queue, results))) + for task in tasks: await task log("DONE!") + def _queue_workload_processing(self, workload_conf, env_conf, repo, arch): - + repo_id = repo["id"] if repo_id not in self.workload_queue: self.workload_queue[repo_id] = {} - + if arch not in self.workload_queue[repo_id]: self.workload_queue[repo_id][arch] = [] @@ -1777,11 +1442,13 @@ def _queue_workload_processing(self, workload_conf, env_conf, repo, arch): self.workload_queue[repo_id][arch].append(workload_task) self.workload_queue_counter_total += 1 + def _reset_workload_processing_queue(self): self.workload_queue = {} self.workload_queue_counter_total = 0 self.workload_queue_counter_current = 0 + def _analyze_workloads(self): # Initialise @@ -1801,7 +1468,7 @@ def _analyze_workloads(self): if label in env_conf["labels"]: # And save those. workload_env_map[workload_conf_id].add(env_conf_id) - + # And now, look at all workload configs... for workload_conf_id, workload_conf in self.configs["workloads"].items(): # ... and for each, look at all env configs it should be analyzed in. @@ -1833,18 +1500,28 @@ def _analyze_workloads(self): # Before even started, look if the env succeeded. If not, there's # no point in doing anything here. - env_id = f"{env_conf['id']}:{repo['id']}:{arch}" + env_id = "{env_conf_id}:{repo_id}:{arch}".format( + env_conf_id=env_conf["id"], + repo_id=repo["id"], + arch=arch + ) env = self.data["envs"][env_id] if env["succeeded"]: self._queue_workload_processing(workload_conf, env_conf, repo, arch) else: - workload_id = f"{workload_conf_id}:{env_conf_id}:{repo_id}:{arch}" + workload_id = "{workload_conf_id}:{env_conf_id}:{repo_id}:{arch}".format( + workload_conf_id=workload_conf_id, + env_conf_id=env_conf_id, + repo_id=repo_id, + arch=arch + ) self.data["workloads"][workload_id] = self._return_failed_workload_env_err(workload_conf, env_conf, repo, arch) asyncio.run(self._analyze_workloads_async(self.data["workloads"])) + def _init_view_pkg(self, input_pkg, arch, placeholder=False, level=0): if placeholder: pkg = { @@ -1857,7 +1534,7 @@ def _init_view_pkg(self, input_pkg, arch, placeholder=False, level=0): "description": input_pkg["description"], "summary": input_pkg["description"], "source_name": input_pkg["srpm"], - "sourcerpm": f"{input_pkg['srpm']}-000-placeholder", + "sourcerpm": "{}-000-placeholder".format(input_pkg["srpm"]), "q_arch": input_pkg, "reponame": "n/a", "all_reponames": set(), @@ -1894,7 +1571,6 @@ def _init_view_pkg(self, input_pkg, arch, placeholder=False, level=0): "env": pkg["in_workload_ids_env"], }) - # FIXME: Use comprehension & extend instead on for loop # Level 1 and higher is buildroot for _ in range(level): pkg["level"].append({ @@ -1911,6 +1587,7 @@ def _init_view_pkg(self, input_pkg, arch, placeholder=False, level=0): return pkg + def _init_view_srpm(self, pkg, level=0): srpm_id = pkg["sourcerpm"].rsplit(".src.rpm")[0] @@ -1958,12 +1635,20 @@ def _init_view_srpm(self, pkg, level=0): return srpm + def _analyze_view(self, view_conf, arch, views): view_conf_id = view_conf["id"] - log(f"Analyzing view: {view_conf['name']} ({view_conf_id}) for {arch}") + log("Analyzing view: {view_name} ({view_conf_id}) for {arch}".format( + view_name=view_conf["name"], + view_conf_id=view_conf_id, + arch=arch + )) - view_id = f"{view_conf_id}:{arch}" + view_id = "{view_conf_id}:{arch}".format( + view_conf_id=view_conf_id, + arch=arch + ) repo_id = view_conf["repository"] @@ -1982,7 +1667,7 @@ def _analyze_view(self, view_conf, arch, views): for workload_id, workload in self.data["workloads"].items(): if workload["repo_id"] != repo_id: continue - + if workload["arch"] != arch: continue @@ -1991,29 +1676,22 @@ def _analyze_view(self, view_conf, arch, views): view["workload_ids"].append(workload_id) - log(f" Includes {len(view['workload_ids'])} workloads.") + log(" Includes {} workloads.".format(len(view["workload_ids"]))) # Packages - log(f" Processing packages from {len(view['workload_ids'])} workloads...") - workload_counter = 0 - total_workloads = len(view['workload_ids']) - for workload_id in view["workload_ids"]: - workload_counter += 1 - if workload_counter % 50 == 0: - log(f" Progress: {workload_counter}/{total_workloads} workloads processed") - workload = self.data["workloads"][workload_id] workload_conf_id = workload["workload_conf_id"] workload_conf = self.configs["workloads"][workload_conf_id] # Packages in the environment for pkg_id in workload["pkg_env_ids"]: + # Initialise if pkg_id not in view["pkgs"]: pkg = self.data["pkgs"][repo_id][arch][pkg_id] view["pkgs"][pkg_id] = self._init_view_pkg(pkg, arch) - + # It's in this wokrload view["pkgs"][pkg_id]["in_workload_ids_all"].add(workload_id) @@ -2025,7 +1703,7 @@ def _analyze_view(self, view_conf, arch, views): view["pkgs"][pkg_id]["in_workload_ids_req"].add(workload_id) elif view["pkgs"][pkg_id]["name"] in workload_conf["arch_packages"][arch]: view["pkgs"][pkg_id]["in_workload_ids_req"].add(workload_id) - + # pkg_relations view["pkgs"][pkg_id]["required_by"].update(workload["pkg_relations"][pkg_id]["required_by"]) view["pkgs"][pkg_id]["recommended_by"].update(workload["pkg_relations"][pkg_id]["recommended_by"]) @@ -2034,11 +1712,12 @@ def _analyze_view(self, view_conf, arch, views): # Packages added by this workload (required or dependency) for pkg_id in workload["pkg_added_ids"]: + # Initialise if pkg_id not in view["pkgs"]: pkg = self.data["pkgs"][repo_id][arch][pkg_id] view["pkgs"][pkg_id] = self._init_view_pkg(pkg, arch) - + # It's in this wokrload view["pkgs"][pkg_id]["in_workload_ids_all"].add(workload_id) @@ -2047,11 +1726,11 @@ def _analyze_view(self, view_conf, arch, views): view["pkgs"][pkg_id]["in_workload_ids_req"].add(workload_id) elif view["pkgs"][pkg_id]["name"] in workload_conf["arch_packages"][arch]: view["pkgs"][pkg_id]["in_workload_ids_req"].add(workload_id) - + # Or a dependency? else: view["pkgs"][pkg_id]["in_workload_ids_dep"].add(workload_id) - + # pkg_relations view["pkgs"][pkg_id]["required_by"].update(workload["pkg_relations"][pkg_id]["required_by"]) view["pkgs"][pkg_id]["recommended_by"].update(workload["pkg_relations"][pkg_id]["recommended_by"]) @@ -2060,17 +1739,18 @@ def _analyze_view(self, view_conf, arch, views): # And finally the non-existing, imaginary, package placeholders! for pkg_id in workload["pkg_placeholder_ids"]: + # Initialise if pkg_id not in view["pkgs"]: placeholder = workload_conf["package_placeholders"]["pkgs"][pkg_id_to_name(pkg_id)] view["pkgs"][pkg_id] = self._init_view_pkg(placeholder, arch, placeholder=True) - + # It's in this wokrload view["pkgs"][pkg_id]["in_workload_ids_all"].add(workload_id) # Placeholders are by definition required view["pkgs"][pkg_id]["in_workload_ids_req"].add(workload_id) - + # ... including the SRPM placeholders for srpm_name in workload["srpm_placeholder_names"]: srpm_id = pkg_placeholder_name_to_nevr(srpm_name) @@ -2079,36 +1759,31 @@ def _analyze_view(self, view_conf, arch, views): if srpm_id not in view["source_pkgs"]: sourcerpm = f"{srpm_id}.src.rpm" view["source_pkgs"][srpm_id] = self._init_view_srpm({"sourcerpm": sourcerpm, "source_name": srpm_name, "reponame": None}) - + # It's a placeholder view["source_pkgs"][srpm_id]["placeholder"] = True # Build requires view["source_pkgs"][srpm_id]["placeholder_directly_required_pkg_names"] = workload_conf["package_placeholders"]["srpms"][srpm_name]["buildrequires"] - + # If this is an addon view, remove all packages that are already in the parent view if view_conf["type"] == "addon": base_view_conf_id = view_conf["base_view_id"] - base_view_id = f"{base_view_conf_id}:{arch}" + base_view_id = "{base_view_conf_id}:{arch}".format( + base_view_conf_id=base_view_conf_id, + arch=arch + ) for base_view_pkg_id in views[base_view_id]["pkgs"]: if base_view_pkg_id in view["pkgs"]: del view["pkgs"][base_view_pkg_id] # Done with packages! - log(f" Done processing workload packages. Total packages in view: {len(view['pkgs'])}") + log(" Includes {} packages.".format(len(view["pkgs"]))) # But not with source packages, that's an entirely different story! - log(f" Building source package data from {len(view['pkgs'])} binary packages...") - pkg_counter = 0 - total_pkgs = len(view["pkgs"]) - for pkg_id, pkg in view["pkgs"].items(): - pkg_counter += 1 - if pkg_counter % 1000 == 0: - log(f" Progress: {pkg_counter}/{total_pkgs} packages processed for source mapping") - srpm_id = pkg["sourcerpm"].rsplit(".src.rpm")[0] if srpm_id not in view["source_pkgs"]: @@ -2121,72 +1796,63 @@ def _analyze_view(self, view_conf, arch, views): view["source_pkgs"][srpm_id]["in_workload_ids_req"].update(pkg["in_workload_ids_req"]) view["source_pkgs"][srpm_id]["in_workload_ids_dep"].update(pkg["in_workload_ids_dep"]) view["source_pkgs"][srpm_id]["in_workload_ids_env"].update(pkg["in_workload_ids_env"]) + + log(" Includes {} source packages.".format(len(view["source_pkgs"]))) - log(f" Includes {len(view['source_pkgs'])} source packages.") log(" DONE!") log("") return view + def _analyze_views(self): - log("") - log("===== Starting View Analysis =====") - log("") views = {} # First, analyse the standard views - log("Phase 1: Analyzing standard (compose) views...") - compose_view_count = sum(1 for v in self.configs["views"].values() if v["type"] == "compose") - log(f" Found {compose_view_count} compose views to process") - for view_conf_id in self.configs["views"]: view_conf = self.configs["views"][view_conf_id] if view_conf["type"] == "compose": - log(f" Processing compose view: {view_conf_id}") for arch in view_conf["architectures"]: - log(f" Architecture: {arch}") view = self._analyze_view(view_conf, arch, views) view_id = view["id"] - views[view_id] = view - log(f" ✅ Completed {view_id}") - - log("") - log("Phase 2: Analyzing addon views...") - addon_view_count = sum(1 for v in self.configs["views"].values() if v["type"] == "addon") - log(f" Found {addon_view_count} addon views to process") + views[view_id] = view + # Second, analyse the addon views # This is important as they need the standard views already available for view_conf_id in self.configs["views"]: view_conf = self.configs["views"][view_conf_id] if view_conf["type"] == "addon": - log(f" Processing addon view: {view_conf_id}") base_view_conf_id = view_conf["base_view_id"] base_view_conf = self.configs["views"][base_view_conf_id] for arch in set(view_conf["architectures"]) & set(base_view_conf["architectures"]): - log(f" Architecture: {arch}") view = self._analyze_view(view_conf, arch, views) view_id = view["id"] - views[view_id] = view - log(f" ✅ Completed {view_id}") - - log("") - log(f"View analysis complete! Total views created: {len(views)}") - log("") + views[view_id] = view + self.data["views"] = views + def _populate_buildroot_with_view_srpms(self, view_conf, arch): view_conf_id = view_conf["id"] - log(f"Initialising buildroot packages of: {view_conf['name']} ({view_conf_id}) for {arch}") + log("Initialising buildroot packages of: {view_name} ({view_conf_id}) for {arch}".format( + view_name=view_conf["name"], + view_conf_id=view_conf_id, + arch=arch + )) + + view_id = "{view_conf_id}:{arch}".format( + view_conf_id=view_conf_id, + arch=arch + ) - view_id = f"{view_conf_id}:{arch}" view = self.data["views"][view_id] repo_id = view_conf["repository"] @@ -2202,7 +1868,7 @@ def _populate_buildroot_with_view_srpms(self, view_conf, arch): if srpm["placeholder"]: directly_required_pkg_names = srpm["placeholder_directly_required_pkg_names"] - + else: # This is the same set in both koji_srpms and srpms directly_required_pkg_names = set() @@ -2223,7 +1889,7 @@ def _populate_buildroot_with_view_srpms(self, view_conf, arch): self.data["buildroot"]["koji_urls"][koji_id] = {} self.data["buildroot"]["koji_urls"][koji_id]["api"] = koji_api_url self.data["buildroot"]["koji_urls"][koji_id]["files"] = koji_files_url - + if arch not in self.data["buildroot"]["koji_srpms"][koji_id]: self.data["buildroot"]["koji_srpms"][koji_id][arch] = {} @@ -2250,9 +1916,11 @@ def _populate_buildroot_with_view_srpms(self, view_conf, arch): self.data["buildroot"]["srpms"][repo_id][arch][srpm_id]["queued"] = False self.data["buildroot"]["srpms"][repo_id][arch][srpm_id]["processed"] = False + log(" DONE!") log("") + def _resolve_srpms_using_root_logs_parallel(self, pass_counter): """ This function is idempotent! @@ -2303,12 +1971,12 @@ def _resolve_srpms_using_root_logs_parallel(self, pass_counter): # Add to work queue work_items.append({ - "koji_id": koji_id, - "koji_api_url": koji_urls["api"], - "koji_files_url": koji_urls["files"], - "srpm_id": srpm_id, - "arch": arch, - "dev_buildroot": self.settings.get("dev_buildroot", False), + 'koji_id': koji_id, + 'koji_api_url': koji_urls["api"], + 'koji_files_url': koji_urls["files"], + 'srpm_id': srpm_id, + 'arch': arch, + 'dev_buildroot': self.settings.get("dev_buildroot", False) }) if not work_items: @@ -2320,11 +1988,6 @@ def _resolve_srpms_using_root_logs_parallel(self, pass_counter): # Process in parallel using ProcessPoolExecutor max_workers = min(self.settings["parallel_max"], len(work_items)) - # Track failures for reporting - failed_downloads = [] # Root logs that failed to download - zero_deps = [] # Packages with zero dependencies (suspicious) - low_deps = [] # Packages with very few deps (possibly truncated logs) - with ProcessPoolExecutor(max_workers=max_workers) as executor: # Submit all jobs future_to_item = { @@ -2347,105 +2010,35 @@ def _resolve_srpms_using_root_logs_parallel(self, pass_counter): if result['error']: log(f"[ Buildroot - pass {pass_counter} - {completed_count} of {total_count} ] " f"Failed {result['srpm_id']} {result['arch']}: {result['error']}") - failed_downloads.append({ - 'srpm_id': result['srpm_id'], - 'arch': result['arch'], - 'error': result['error'] - }) else: - # Log completion with warning if present - deps_count = len(result['deps']) - warning_msg = result.get('warning', '') - if warning_msg: - log(f"[ Buildroot - pass {pass_counter} - {completed_count} of {total_count} ] " - f"⚠️ {result['srpm_id']} {result['arch']} - {deps_count} deps - {warning_msg}") - else: - log(f"[ Buildroot - pass {pass_counter} - {completed_count} of {total_count} ] " - f"Completed {result['srpm_id']} {result['arch']} - found {deps_count} deps") - - # Detect suspicious results - if deps_count == 0: - zero_deps.append({ - 'srpm_id': result['srpm_id'], - 'arch': result['arch'] - }) - elif deps_count < 3: - low_deps.append({ - 'srpm_id': result['srpm_id'], - 'arch': result['arch'], - 'deps': result['deps'] - }) + log(f"[ Buildroot - pass {pass_counter} - {completed_count} of {total_count} ] " + f"Completed {result['srpm_id']} {result['arch']} - found {len(result['deps'])} deps") except Exception as e: log(f"Failed to process {work_item['srpm_id']}: {e}") # Apply empty result for failed processing error_result = { - "srpm_id": work_item["srpm_id"], - "arch": work_item["arch"], - "deps": [], - "error": str(e), + 'srpm_id': work_item['srpm_id'], + 'arch': work_item['arch'], + 'deps': [], + 'error': str(e) } self._apply_srpm_result(work_item, error_result) - failed_downloads.append({ - 'srpm_id': work_item["srpm_id"], - 'arch': work_item["arch"], - 'error': str(e) - }) # Save updated cache dump_data(self.settings["root_log_deps_cache_path"], self.cache["root_log_deps"]["next"]) - # Report summary of issues - log("") - log("=" * 80) - log("ROOT LOG PROCESSING SUMMARY") - log("=" * 80) - - if failed_downloads: - log(f"⚠️ FAILED DOWNLOADS: {len(failed_downloads)} root.log files failed to download/parse") - log("") - for item in failed_downloads[:20]: # Show first 20 - log(f" ❌ {item['srpm_id']} ({item['arch']})") - log(f" Error: {item['error'][:100]}") - if len(failed_downloads) > 20: - log(f" ... and {len(failed_downloads) - 20} more") - - if zero_deps: - log("") - log(f"⚠️ ZERO DEPENDENCIES: {len(zero_deps)} packages reported 0 build dependencies") - log(" (This is unusual - most packages have at least bash, gcc, etc.)") - log("") - for item in zero_deps[:20]: - log(f" ⚠️ {item['srpm_id']} ({item['arch']})") - if len(zero_deps) > 20: - log(f" ... and {len(zero_deps) - 20} more") - - if low_deps: - log("") - log(f"⚠️ LOW DEPENDENCIES: {len(low_deps)} packages have very few dependencies (<3)") - log(" (Possibly truncated or corrupted root.log files)") - log("") - for item in low_deps[:10]: - log(f" ⚠️ {item['srpm_id']} ({item['arch']}): {item['deps']}") - if len(low_deps) > 10: - log(f" ... and {len(low_deps) - 10} more") - - if not failed_downloads and not zero_deps and not low_deps: - log("✅ All root.log files processed successfully with reasonable dependency counts") - - log("=" * 80) - log("") - log("") log(" DONE!") log("") + def _apply_srpm_result(self, work_item, result): """Apply worker result back to main data structures""" - koji_id = work_item["koji_id"] - arch = work_item["arch"] - srpm_id = work_item["srpm_id"] - deps = result["deps"] + koji_id = work_item['koji_id'] + arch = work_item['arch'] + srpm_id = work_item['srpm_id'] + deps = result['deps'] # Update cache self.cache["root_log_deps"]["next"][koji_id][arch][srpm_id] = deps @@ -2455,6 +2048,7 @@ def _apply_srpm_result(self, work_item, result): # set, because its reference is shared between the koji_srpms and the srpm sections self.data["buildroot"]["koji_srpms"][koji_id][arch][srpm_id]["directly_required_pkg_names"].update(deps) + def _analyze_build_groups(self): log("") @@ -2468,9 +2062,12 @@ def _analyze_build_groups(self): for arch in self.data["buildroot"]["srpms"][repo_id]: - generated_id = f"CR-buildroot-base-env-{repo_id}-{arch}" + generated_id = "CR-buildroot-base-env-{repo_id}-{arch}".format( + repo_id=repo_id, + arch=arch + ) - # Using the _analyze_env function! + # Using the _analyze_env function! # So I need to reconstruct a fake env_conf fake_env_conf = {} fake_env_conf["id"] = generated_id @@ -2484,7 +2081,10 @@ def _analyze_build_groups(self): fake_env_conf["arch_packages"] = {} fake_env_conf["arch_packages"][arch] = [] - log(f"Resolving build group: {repo_id} {arch}") + log("Resolving build group: {repo_id} {arch}".format( + repo_id=repo_id, + arch=arch + )) repo = self.configs["repos"][repo_id] fake_env = self._analyze_env(fake_env_conf, repo, arch) @@ -2503,6 +2103,7 @@ def _analyze_build_groups(self): def _expand_buildroot_srpms(self): # This function is idempotent! + # # That means it can be run many times without affecting the old results. log("Expanding the SRPM set...") @@ -2523,7 +2124,7 @@ def _expand_buildroot_srpms(self): # Adding a new one! counter += 1 - + srpm_reponame = self.data["pkgs"][repo_id][arch][pkg_id]["reponame"] # This is the same set in both koji_srpms and srpms @@ -2564,6 +2165,7 @@ def _expand_buildroot_srpms(self): def _analyze_srpm_buildroots(self, pass_counter): # This function is idempotent! + # # That means it can be run many times without affecting the old results. log("") @@ -2613,12 +2215,20 @@ def _analyze_srpm_buildroots(self, pass_counter): fake_env_conf["arch_packages"][arch] = [] srpms_to_resolve_counter += 1 + + #log("[ Buildroot - pass {} - {} of {} ]".format(pass_counter, srpms_to_resolve_counter, total_srpms_to_resolve)) + #log("Resolving SRPM buildroot: {repo_id} {arch} {srpm_id}".format( + # repo_id=repo_id, + # arch=arch, + # srpm_id=srpm_id + #)) repo = self.configs["repos"][repo_id] + + #fake_workload = self._analyze_workload(fake_workload_conf, fake_env_conf, repo, arch) self._queue_workload_processing(fake_workload_conf, fake_env_conf, repo, arch) # Save the buildroot data self.data["buildroot"]["srpms"][repo_id][arch][srpm_id]["queued"] = True - log(f"fake_workload_results -> {fake_workload_results}") asyncio.run(self._analyze_workloads_async(fake_workload_results)) @@ -2629,7 +2239,12 @@ def _analyze_srpm_buildroots(self, pass_counter): if srpm["processed"]: continue - fake_workload_id = f"{srpm_id}:{self.data['buildroot']['build_groups'][repo_id][arch]['generated_id']}:{repo_id}:{arch}" + fake_workload_id = "{workload_conf_id}:{env_conf_id}:{repo_id}:{arch}".format( + workload_conf_id=srpm_id, + env_conf_id=self.data["buildroot"]["build_groups"][repo_id][arch]["generated_id"], + repo_id=repo_id, + arch=arch + ) fake_workload = fake_workload_results[fake_workload_id] @@ -2677,6 +2292,7 @@ def _analyze_buildroot(self): self._populate_buildroot_with_view_srpms(view_conf, arch) # Time to resolve the build groups! + # # This initialises and populates: # buildroot["build_groups"] self._analyze_build_groups() @@ -2692,6 +2308,7 @@ def _analyze_buildroot(self): log("") log("") # Get the directly_required_pkg_names from koji root logs + # # Adds stuff to existing: # data["buildroot"]["koji_srpms"]... # ... which also updates: @@ -2718,8 +2335,9 @@ def _analyze_buildroot(self): log("") break + def _add_missing_levels_to_pkg_or_srpm(self, pkg_or_srpm, level): - """Add missing level entries to reach the target level.""" + pkg_current_max_level = len(pkg_or_srpm["level"]) - 1 for _ in range(level - pkg_current_max_level): pkg_or_srpm["level"].append({ @@ -2734,12 +2352,16 @@ def _add_buildroot_to_view(self, view_conf, arch): view_conf_id = view_conf["id"] - view_id = f"{view_conf_id}:{arch}" + view_id = "{view_conf_id}:{arch}".format( + view_conf_id=view_conf_id, + arch=arch + ) repo_id = view_conf["repository"] view = self.data["views"][view_id] + log("") log(f"Adding buildroot to view {view_id}...") @@ -2760,6 +2382,7 @@ def _add_buildroot_to_view(self, view_conf, arch): for buildroot_srpm_id in srpm_ids_to_process: buildroot_srpm = self.data["buildroot"]["srpms"][repo_id][arch][buildroot_srpm_id] + # Packages in the base buildroot (which would be the environment in workloads) for pkg_id in buildroot_srpm["pkg_env_ids"]: added_pkg_ids.add(pkg_id) @@ -2768,7 +2391,7 @@ def _add_buildroot_to_view(self, view_conf, arch): if pkg_id not in view["pkgs"]: pkg = self.data["pkgs"][repo_id][arch][pkg_id] view["pkgs"][pkg_id] = self._init_view_pkg(pkg, arch, level=level) - + # Add missing levels to the pkg self._add_missing_levels_to_pkg_or_srpm(view["pkgs"][pkg_id], level) @@ -2784,7 +2407,7 @@ def _add_buildroot_to_view(self, view_conf, arch): if view["pkgs"][pkg_id]["name"] in buildroot_srpm["directly_required_pkg_names"]: view["pkgs"][pkg_id]["in_buildroot_of_srpm_id_req"].add(buildroot_srpm_id) view["pkgs"][pkg_id]["level"][level]["req"].add(buildroot_srpm_id) - + # pkg_relations view["pkgs"][pkg_id]["required_by"].update(buildroot_srpm["pkg_relations"][pkg_id]["required_by"]) view["pkgs"][pkg_id]["recommended_by"].update(buildroot_srpm["pkg_relations"][pkg_id]["recommended_by"]) @@ -2799,10 +2422,10 @@ def _add_buildroot_to_view(self, view_conf, arch): if pkg_id not in view["pkgs"]: pkg = self.data["pkgs"][repo_id][arch][pkg_id] view["pkgs"][pkg_id] = self._init_view_pkg(pkg, arch, level=level) - + # Add missing levels to the pkg self._add_missing_levels_to_pkg_or_srpm(view["pkgs"][pkg_id], level) - + # It's in this buildroot view["pkgs"][pkg_id]["in_buildroot_of_srpm_id_all"].add(buildroot_srpm_id) view["pkgs"][pkg_id]["level"][level]["all"].add(buildroot_srpm_id) @@ -2811,18 +2434,18 @@ def _add_buildroot_to_view(self, view_conf, arch): if view["pkgs"][pkg_id]["name"] in buildroot_srpm["directly_required_pkg_names"]: view["pkgs"][pkg_id]["in_buildroot_of_srpm_id_req"].add(buildroot_srpm_id) view["pkgs"][pkg_id]["level"][level]["req"].add(buildroot_srpm_id) - + # Or a dependency? else: view["pkgs"][pkg_id]["in_buildroot_of_srpm_id_dep"].add(buildroot_srpm_id) view["pkgs"][pkg_id]["level"][level]["dep"].add(buildroot_srpm_id) - + # pkg_relations view["pkgs"][pkg_id]["required_by"].update(buildroot_srpm["pkg_relations"][pkg_id]["required_by"]) view["pkgs"][pkg_id]["recommended_by"].update(buildroot_srpm["pkg_relations"][pkg_id]["recommended_by"]) view["pkgs"][pkg_id]["suggested_by"].update(buildroot_srpm["pkg_relations"][pkg_id]["suggested_by"]) view["pkgs"][pkg_id]["supplements"].update(buildroot_srpm["pkg_relations"][pkg_id]["supplements"]) - + # Resetting the SRPMs, so only the new ones can be added srpm_ids_to_process = set() @@ -2835,7 +2458,7 @@ def _add_buildroot_to_view(self, view_conf, arch): if srpm_id not in view["source_pkgs"]: view["source_pkgs"][srpm_id] = self._init_view_srpm(pkg, level=level) srpm_ids_to_process.add(srpm_id) - + # Add missing levels to the pkg self._add_missing_levels_to_pkg_or_srpm(view["source_pkgs"][srpm_id], level) @@ -2851,8 +2474,8 @@ def _add_buildroot_to_view(self, view_conf, arch): view["source_pkgs"][srpm_id]["level"][level]["dep"].update(pkg["level"][level]["dep"]) view["source_pkgs"][srpm_id]["level"][level]["env"].update(pkg["level"][level]["env"]) - log(f" added {len(added_pkg_ids)} RPMs") - log(f" added {len(srpm_ids_to_process)} SRPMs") + log (f" added {len(added_pkg_ids)} RPMs") + log (f" added {len(srpm_ids_to_process)} SRPMs") # More iterations needed? if not srpm_ids_to_process: @@ -2860,6 +2483,7 @@ def _add_buildroot_to_view(self, view_conf, arch): log("") break + def _add_buildroot_to_views(self): log("") @@ -2881,7 +2505,8 @@ def _add_buildroot_to_views(self): log(" DONE!") log("") - def _init_pkg_or_srpm_relations_fields(self, target_pkg, type=None): + + def _init_pkg_or_srpm_relations_fields(self, target_pkg, type = None): # I kept them all listed so they're easy to copy # Workload IDs @@ -2889,7 +2514,7 @@ def _init_pkg_or_srpm_relations_fields(self, target_pkg, type=None): target_pkg["in_workload_ids_req"] = set() target_pkg["in_workload_ids_dep"] = set() target_pkg["in_workload_ids_env"] = set() - + # Workload Conf IDs target_pkg["in_workload_conf_ids_all"] = set() target_pkg["in_workload_conf_ids_req"] = set() @@ -2903,10 +2528,10 @@ def _init_pkg_or_srpm_relations_fields(self, target_pkg, type=None): target_pkg["in_buildroot_of_srpm_id_env"] = set() # Buildroot SRPM Names - target_pkg["in_buildroot_of_srpm_name_all"] = {} # of set() of srpm_ids - target_pkg["in_buildroot_of_srpm_name_req"] = {} # of set() of srpm_ids - target_pkg["in_buildroot_of_srpm_name_dep"] = {} # of set() of srpm_ids - target_pkg["in_buildroot_of_srpm_name_env"] = {} # of set() of srpm_ids + target_pkg["in_buildroot_of_srpm_name_all"] = {} # of set() of srpm_ids + target_pkg["in_buildroot_of_srpm_name_req"] = {} # of set() of srpm_ids + target_pkg["in_buildroot_of_srpm_name_dep"] = {} # of set() of srpm_ids + target_pkg["in_buildroot_of_srpm_name_env"] = {} # of set() of srpm_ids # Unwanted target_pkg["unwanted_completely_in_list_ids"] = set() @@ -2931,13 +2556,14 @@ def _init_pkg_or_srpm_relations_fields(self, target_pkg, type=None): target_pkg["weak_dependency_of_pkg_nevrs"] = set() target_pkg["reverse_weak_dependency_of_pkg_nevrs"] = set() + # Dependency of RPM Names target_pkg["dependency_of_pkg_names"] = {} # of set() of nevrs target_pkg["hard_dependency_of_pkg_names"] = {} # of set() of nevrs target_pkg["weak_dependency_of_pkg_names"] = {} # if set() of nevrs target_pkg["reverse_weak_dependency_of_pkg_names"] = {} # if set() of nevrs - def _populate_pkg_or_srpm_relations_fields(self, target_pkg, source_pkg, type=None, view=None): + def _populate_pkg_or_srpm_relations_fields(self, target_pkg, source_pkg, type = None, view = None): # source_pkg is the arch-specific binary package # target_pkg is a representation of that pages for all arches @@ -2953,6 +2579,7 @@ def _populate_pkg_or_srpm_relations_fields(self, target_pkg, source_pkg, type=No target_pkg["unwanted_completely_in_list_ids"].update(source_pkg["unwanted_completely_in_list_ids"]) target_pkg["unwanted_buildroot_in_list_ids"].update(source_pkg["unwanted_buildroot_in_list_ids"]) + # Dependency relationships for list_type in ["all", "req", "dep", "env"]: target_pkg[f"in_workload_ids_{list_type}"].update(source_pkg[f"in_workload_ids_{list_type}"]) @@ -2968,9 +2595,9 @@ def _populate_pkg_or_srpm_relations_fields(self, target_pkg, source_pkg, type=No if srpm_name not in target_pkg[f"in_buildroot_of_srpm_name_{list_type}"]: target_pkg[f"in_buildroot_of_srpm_name_{list_type}"][srpm_name] = set() - + target_pkg[f"in_buildroot_of_srpm_name_{list_type}"][srpm_name].add(srpm_id) - + # Level number level_number = 0 for level in source_pkg["level"]: @@ -2984,7 +2611,7 @@ def _populate_pkg_or_srpm_relations_fields(self, target_pkg, source_pkg, type=No for level_data in source_pkg["level"]: # 'level' is the number # 'level_data' is the ["all"][workload_id] or ["all"][srpm_id] or - # ["req"][workload_id] or ["req"][srpm_id] or + # ["req"][workload_id] or ["req"][srpm_id] or # ["dep"][workload_id] or ["dep"][srpm_id] or # ["env"][workload_id] or ["env"][srpm_id] @@ -2999,23 +2626,24 @@ def _populate_pkg_or_srpm_relations_fields(self, target_pkg, source_pkg, type=No if level_scope not in target_pkg["level"][level]: target_pkg["level"][level][level_scope] = set() - + target_pkg["level"][level][level_scope].update(those_ids) - - level += 1 - + + level +=1 + + if type == "rpm": # Hard dependency of for pkg_id in source_pkg["required_by"]: pkg_name = pkg_id_to_name(pkg_id) - + # This only happens in addon views, and only rarely. # Basically means that a package in the addon view is required # by a package in the base view. # Doesn't make sense? # Think of 'glibc-all-langpacks' being in the addon, # while the proper langpacks along with 'glibc' are in the base view. - # + # # In that case, 'glibc' is not in the addon, but 'glibc-all-langpacks' # requires it. # @@ -3030,7 +2658,10 @@ def _populate_pkg_or_srpm_relations_fields(self, target_pkg, source_pkg, type=No continue pkg = view["pkgs"][pkg_id] - pkg_nevr = f"{pkg['name']}-{pkg['evr']}" + pkg_nevr = "{name}-{evr}".format( + name=pkg["name"], + evr=pkg["evr"] + ) target_pkg["hard_dependency_of_pkg_nevrs"].add(pkg_nevr) if pkg_name not in target_pkg["hard_dependency_of_pkg_names"]: @@ -3051,7 +2682,10 @@ def _populate_pkg_or_srpm_relations_fields(self, target_pkg, source_pkg, type=No continue pkg = view["pkgs"][pkg_id] - pkg_nevr = f"{pkg['name']}-{pkg['evr']}" + pkg_nevr = "{name}-{evr}".format( + name=pkg["name"], + evr=pkg["evr"] + ) target_pkg["weak_dependency_of_pkg_nevrs"].add(pkg_nevr) if pkg_name not in target_pkg["weak_dependency_of_pkg_names"]: @@ -3071,13 +2705,16 @@ def _populate_pkg_or_srpm_relations_fields(self, target_pkg, source_pkg, type=No continue pkg = view["pkgs"][pkg_id] - pkg_nevr = f"{pkg['name']}-{pkg['evr']}" + pkg_nevr = "{name}-{evr}".format( + name=pkg["name"], + evr=pkg["evr"] + ) target_pkg["reverse_weak_dependency_of_pkg_nevrs"].add(pkg_nevr) if pkg_name not in target_pkg["reverse_weak_dependency_of_pkg_names"]: target_pkg["reverse_weak_dependency_of_pkg_names"][pkg_name] = set() target_pkg["reverse_weak_dependency_of_pkg_names"][pkg_name].add(pkg_nevr) - + # All types of dependency target_pkg["dependency_of_pkg_nevrs"].update(target_pkg["hard_dependency_of_pkg_nevrs"]) target_pkg["dependency_of_pkg_nevrs"].update(target_pkg["weak_dependency_of_pkg_nevrs"]) @@ -3086,13 +2723,13 @@ def _populate_pkg_or_srpm_relations_fields(self, target_pkg, source_pkg, type=No for pkg_name, pkg_nevrs in target_pkg["hard_dependency_of_pkg_names"].items(): if pkg_name not in target_pkg["dependency_of_pkg_names"]: target_pkg["dependency_of_pkg_names"][pkg_name] = set() - + target_pkg["dependency_of_pkg_names"][pkg_name].update(pkg_nevrs) for pkg_name, pkg_nevrs in target_pkg["weak_dependency_of_pkg_names"].items(): if pkg_name not in target_pkg["dependency_of_pkg_names"]: target_pkg["dependency_of_pkg_names"][pkg_name] = set() - + target_pkg["dependency_of_pkg_names"][pkg_name].update(pkg_nevrs) for pkg_name, pkg_nevrs in target_pkg["reverse_weak_dependency_of_pkg_names"].items(): @@ -3100,9 +2737,11 @@ def _populate_pkg_or_srpm_relations_fields(self, target_pkg, source_pkg, type=No target_pkg["dependency_of_pkg_names"][pkg_name] = set() target_pkg["dependency_of_pkg_names"][pkg_name].update(pkg_nevrs) + # TODO: add the levels + def _generate_views_all_arches(self): views_all_arches = {} @@ -3111,6 +2750,7 @@ def _generate_views_all_arches(self): #if view_conf["type"] == "compose": if True: + repo_id = view_conf["repository"] view_all_arches = {} @@ -3154,8 +2794,13 @@ def _generate_views_all_arches(self): view_all_arches["numbers"]["srpms"]["build_level_1"] = 0 view_all_arches["numbers"]["srpms"]["build_level_2_plus"] = 0 + for arch in view_conf["architectures"]: - view_id = f"{view_conf_id}:{arch}" + view_id = "{view_conf_id}:{arch}".format( + view_conf_id=view_conf_id, + arch=arch + ) + view = self.data["views"][view_id] # Workloads @@ -3172,17 +2817,19 @@ def _generate_views_all_arches(self): view_all_arches["workloads"][workload_conf_id]["succeeded"] = True view_all_arches["workloads"][workload_conf_id]["no_warnings"] = True # ... - + if not workload["succeeded"]: view_all_arches["workloads"][workload_conf_id]["succeeded"] = False view_all_arches["everything_succeeded"] = False - + if workload["warnings"]["message"]: view_all_arches["workloads"][workload_conf_id]["no_warnings"] = False view_all_arches["no_warnings"] = False + # Binary Packages for package in view["pkgs"].values(): + # Binary Packages by name key = "pkgs_by_name" identifier = package["name"] @@ -3228,7 +2875,7 @@ def _generate_views_all_arches(self): view_all_arches[key][identifier]["category"] = None self._init_pkg_or_srpm_relations_fields(view_all_arches[key][identifier], type="rpm") - + view_all_arches[key][identifier]["arches"].add(arch) view_all_arches[key][identifier]["reponame_per_arch"][arch] = package["reponame"] view_all_arches[key][identifier]["highest_priority_reponames_per_arch"][arch] = package["highest_priority_reponames"] @@ -3239,9 +2886,10 @@ def _generate_views_all_arches(self): self._populate_pkg_or_srpm_relations_fields(view_all_arches[key][identifier], package, type="rpm", view=view) - + # Source Packages for package in view["source_pkgs"].values(): + # Source Packages by name key = "source_pkgs_by_name" identifier = package["name"] @@ -3261,6 +2909,7 @@ def _generate_views_all_arches(self): view_all_arches[key][identifier]["category"] = None self._init_pkg_or_srpm_relations_fields(view_all_arches[key][identifier]) + if view_all_arches["has_buildroot"]: if not self.data["buildroot"]["srpms"][repo_id][arch][package["id"]]["succeeded"]: @@ -3272,11 +2921,11 @@ def _generate_views_all_arches(self): view_all_arches[key][identifier]["buildroot_no_warnings"] = False view_all_arches[key][identifier]["warnings"][arch] = self.data["buildroot"]["srpms"][repo_id][arch][package["id"]]["warnings"] - + view_all_arches[key][identifier]["arches"].add(arch) self._populate_pkg_or_srpm_relations_fields(view_all_arches[key][identifier], package, type="srpm") - + # Add binary packages to source packages for pkg_id, pkg in view["pkgs"].items(): @@ -3287,8 +2936,13 @@ def _generate_views_all_arches(self): view_all_arches["source_pkgs_by_name"][source_name]["pkg_names"].add(pkg["name"]) # Add package nevrs - pkg_nevr = f"{pkg['name']}-{pkg['evr']}" + pkg_nevr = "{name}-{evr}".format( + name=pkg["name"], + evr=pkg["evr"] + ) view_all_arches["source_pkgs_by_name"][source_name]["pkg_nevrs"].add(pkg_nevr) + + # RPMs for pkg in view_all_arches["pkgs_by_nevr"].values(): @@ -3306,19 +2960,12 @@ def _generate_views_all_arches(self): category = "build_level_1" elif pkg["level_number"] > 1: category = "build_level_2_plus" - - if category is not None: - view_all_arches["numbers"]["pkgs"][category] += 1 - - view_all_arches["numbers"]["pkgs"]["runtime"] = \ - view_all_arches["numbers"]["pkgs"]["env"] + \ - view_all_arches["numbers"]["pkgs"]["req"] + \ - view_all_arches["numbers"]["pkgs"]["dep"] - view_all_arches["numbers"]["pkgs"]["build"] = \ - view_all_arches["numbers"]["pkgs"]["build_base"] + \ - view_all_arches["numbers"]["pkgs"]["build_level_1"] + \ - view_all_arches["numbers"]["pkgs"]["build_level_2_plus"] - + + view_all_arches["numbers"]["pkgs"][category] += 1 + + view_all_arches["numbers"]["pkgs"]["runtime"] = view_all_arches["numbers"]["pkgs"]["env"] + view_all_arches["numbers"]["pkgs"]["req"] + view_all_arches["numbers"]["pkgs"]["dep"] + view_all_arches["numbers"]["pkgs"]["build"] = view_all_arches["numbers"]["pkgs"]["build_base"] + view_all_arches["numbers"]["pkgs"]["build_level_1"] + view_all_arches["numbers"]["pkgs"]["build_level_2_plus"] + # SRPMs for pkg in view_all_arches["source_pkgs_by_name"].values(): category = None @@ -3335,10 +2982,9 @@ def _generate_views_all_arches(self): category = "build_level_1" elif pkg["level_number"] > 1: category = "build_level_2_plus" - - if category is not None: - view_all_arches["numbers"]["srpms"][category] += 1 - + + view_all_arches["numbers"]["srpms"][category] += 1 + view_all_arches["numbers"]["srpms"]["runtime"] = \ view_all_arches["numbers"]["srpms"]["env"] + \ view_all_arches["numbers"]["srpms"]["req"] + \ @@ -3349,11 +2995,16 @@ def _generate_views_all_arches(self): view_all_arches["numbers"]["srpms"]["build_level_1"] + \ view_all_arches["numbers"]["srpms"]["build_level_2_plus"] + + + + # Done views_all_arches[view_conf_id] = view_all_arches - + self.data["views_all_arches"] = views_all_arches + def _add_unwanted_packages_to_view(self, view, view_conf): arch = view["arch"] @@ -3365,7 +3016,7 @@ def _add_unwanted_packages_to_view(self, view, view_conf): for unwanted_label in unwanted["labels"]: if view_label == unwanted_label: unwanted_conf_ids.add(unwanted_conf_id) - + # Dicts pkgs_unwanted_buildroot = {} pkgs_unwanted_completely = {} @@ -3406,7 +3057,7 @@ def _add_unwanted_packages_to_view(self, view, view_conf): if srpm_name in srpms_unwanted_completely: list_ids = srpms_unwanted_completely[srpm_name] view["pkgs"][pkg_id]["unwanted_completely_in_list_ids"].update(list_ids) - + # Add it to the srpms for srpm_id, srpm in view["source_pkgs"].items(): srpm_name = srpm["name"] @@ -3415,6 +3066,7 @@ def _add_unwanted_packages_to_view(self, view, view_conf): list_ids = srpms_unwanted_completely[srpm_name] view["source_pkgs"][srpm_id]["unwanted_completely_in_list_ids"].update(list_ids) + def _add_unwanted_packages_to_views(self): log("") @@ -3429,7 +3081,11 @@ def _add_unwanted_packages_to_views(self): if view_conf["buildroot_strategy"] == "root_logs": for arch in view_conf["architectures"]: - view_id = f"{view_conf_id}:{arch}" + view_id = "{view_conf_id}:{arch}".format( + view_conf_id=view_conf_id, + arch=arch + ) + view = self.data["views"][view_id] self._add_unwanted_packages_to_view(view, view_conf) @@ -3454,12 +3110,13 @@ def _recommend_maintainers(self): # ... etc # # I'll call a combination of these a _score_ because I can't think of - # anything better at this point. It's a tuple! - # + # anything better at this point. It's a tuple! + # # (0, 0) # | '-- sub-level 0 == explicitly required # '---- level 0 == runtime - # + # + for view_conf_id in self.configs["views"]: view_conf = self.configs["views"][view_conf_id] @@ -3488,7 +3145,7 @@ def _recommend_maintainers(self): # Take all explicitly required packages and assign them # to the maintainer of their workloads. # - # Or of this is the buildroot levels, + # Or of this is the buildroot levels, for pkg_name, pkg in view_all_arches["pkgs_by_name"].items(): source_name = pkg["source_name"] @@ -3503,32 +3160,43 @@ def _recommend_maintainers(self): # 1/ maintainer_recommendation if workload_maintainer not in pkg["maintainer_recommendation"]: + #pkg["maintainer_recommendation"][workload_maintainer] = set() self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation"][workload_maintainer] = set() - + + #pkg["maintainer_recommendation"][workload_maintainer].add(score) self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation"][workload_maintainer].add(score) # 2/ maintainer_recommendation_details if level not in pkg["maintainer_recommendation_details"]: + #pkg["maintainer_recommendation_details"][level] = {} self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation_details"][level] = {} - + if sublevel not in pkg["maintainer_recommendation_details"][level]: + #pkg["maintainer_recommendation_details"][level][sublevel] = {} self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation_details"][level][sublevel] = {} - + if workload_maintainer not in pkg["maintainer_recommendation_details"][level][sublevel]: + #pkg["maintainer_recommendation_details"][level][sublevel][workload_maintainer] = {} + #pkg["maintainer_recommendation_details"][level][sublevel][workload_maintainer]["reasons"] = {} + #pkg["maintainer_recommendation_details"][level][sublevel][workload_maintainer]["locations"] = {} self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation_details"][level][sublevel][workload_maintainer] = {} self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation_details"][level][sublevel][workload_maintainer]["reasons"] = set() self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation_details"][level][sublevel][workload_maintainer]["locations"] = set() + #pkg["maintainer_recommendation_details"][level][sublevel][workload_maintainer]["locations"].add(workload_conf_id) self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation_details"][level][sublevel][workload_maintainer]["locations"].add(workload_conf_id) # Lie to the while loop so it runs at least once level_changes_made = True level_change_detection = set() + while level_changes_made: + # Level 1 and higher if int(level) > 0: + level_changes_made = False log(f" {score}") @@ -3574,23 +3242,31 @@ def _recommend_maintainers(self): # 1/ maintainer_recommendation if buildroot_srpm_maintainer not in pkg["maintainer_recommendation"]: + #pkg["maintainer_recommendation"][workload_maintainer] = set() self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation"][buildroot_srpm_maintainer] = set() + #pkg["maintainer_recommendation"][buildroot_srpm_maintainer].add(score) self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation"][buildroot_srpm_maintainer].add(score) # 2/ maintainer_recommendation_details if level not in pkg["maintainer_recommendation_details"]: + #pkg["maintainer_recommendation_details"][level] = {} self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation_details"][level] = {} - + if sublevel not in pkg["maintainer_recommendation_details"][level]: + #pkg["maintainer_recommendation_details"][level][sublevel] = {} self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation_details"][level][sublevel] = {} - + if buildroot_srpm_maintainer not in pkg["maintainer_recommendation_details"][level][sublevel]: + #pkg["maintainer_recommendation_details"][level][sublevel][buildroot_srpm_maintainer] = {} + #pkg["maintainer_recommendation_details"][level][sublevel][buildroot_srpm_maintainer]["reasons"] = {} + #pkg["maintainer_recommendation_details"][level][sublevel][buildroot_srpm_maintainer]["locations"] = {} self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation_details"][level][sublevel][buildroot_srpm_maintainer] = {} self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation_details"][level][sublevel][buildroot_srpm_maintainer]["reasons"] = set() self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation_details"][level][sublevel][buildroot_srpm_maintainer]["locations"] = set() + #pkg["maintainer_recommendation_details"][level][sublevel][buildroot_srpm_maintainer]["locations"].add(buildroot_srpm_name) self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation_details"][level][sublevel][buildroot_srpm_maintainer]["locations"].add(buildroot_srpm_name) @@ -3642,23 +3318,30 @@ def _recommend_maintainers(self): else: sublevel_changes_made = True sublevel_change_detection.add(sublevel_change_detection_tuple) - + # 1/ maintainer_recommendation if superior_pkg_maintainer not in pkg["maintainer_recommendation"]: + #pkg["maintainer_recommendation"][workload_maintainer] = set() self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation"][superior_pkg_maintainer] = set() + #pkg["maintainer_recommendation"][superior_pkg_maintainer].add(score) self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation"][superior_pkg_maintainer].add(score) # 2/ maintainer_recommendation_details if level not in pkg["maintainer_recommendation_details"]: + #pkg["maintainer_recommendation_details"][level] = {} self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation_details"][level] = {} - + if sublevel not in pkg["maintainer_recommendation_details"][level]: + #pkg["maintainer_recommendation_details"][level][sublevel] = {} self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation_details"][level][sublevel] = {} - + if superior_pkg_maintainer not in pkg["maintainer_recommendation_details"][level][sublevel]: + #pkg["maintainer_recommendation_details"][level][sublevel][superior_pkg_maintainer] = {} + #pkg["maintainer_recommendation_details"][level][sublevel][superior_pkg_maintainer]["reasons"] = {} + #pkg["maintainer_recommendation_details"][level][sublevel][superior_pkg_maintainer]["locations"] = {} self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation_details"][level][sublevel][superior_pkg_maintainer] = {} self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation_details"][level][sublevel][superior_pkg_maintainer]["reasons"] = set() self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation_details"][level][sublevel][superior_pkg_maintainer]["locations"] = set() @@ -3668,9 +3351,10 @@ def _recommend_maintainers(self): self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation_details"][level][sublevel][superior_pkg_maintainer]["locations"].update(locations) reason = (superior_pkg_name, superior_srpm_name, pkg_name) + #pkg["maintainer_recommendation_details"][level][sublevel][superior_pkg_maintainer]["reasons"].add(reason) self.data["views_all_arches"][view_conf_id]["pkgs_by_name"][pkg_name]["maintainer_recommendation_details"][level][sublevel][superior_pkg_maintainer]["reasons"].add(reason) - + # Now add this info to the source packages for pkg_name, pkg in view_all_arches["pkgs_by_name"].items(): source_name = pkg["source_name"] @@ -3681,13 +3365,13 @@ def _recommend_maintainers(self): if maintainer not in self.data["views_all_arches"][view_conf_id]["source_pkgs_by_name"][source_name]["maintainer_recommendation"]: self.data["views_all_arches"][view_conf_id]["source_pkgs_by_name"][source_name]["maintainer_recommendation"][maintainer] = set() - + self.data["views_all_arches"][view_conf_id]["source_pkgs_by_name"][source_name]["maintainer_recommendation"][maintainer].update(maintainer_scores) # Add it here so it's not processed again in the another level this_level_srpms.add(source_name) - + # 2/ maintainer_recommendation_details for loop_level, loop_sublevels in pkg["maintainer_recommendation_details"].items(): @@ -3719,12 +3403,14 @@ def _recommend_maintainers(self): # And set stuff for the next level prev_level = level level = str(int(level) + 1) - # level += 1 + #level += 1 sublevel = str(0) score = (level, sublevel) previous_level_srpms.update(this_level_srpms) this_level_srpms = set() + + # And elect the best owners for each srpm for source_name, srpm in view_all_arches["source_pkgs_by_name"].items(): @@ -3769,10 +3455,14 @@ def _recommend_maintainers(self): self.data["views_all_arches"][view_conf_id]["source_pkgs_by_name"][source_name]["best_maintainers"].update(best_maintainers) + + + log("") log(" DONE!") log("") + def analyze_things(self): log("") log("###############################################################################") @@ -3795,6 +3485,9 @@ def analyze_things(self): self.tmp_dnf_cachedir = os.path.join(tmp, "dnf_cachedir") self.tmp_installroots = os.path.join(tmp, "installroots") + # List of supported arches + all_arches = self.settings["allowed_arches"] + # Repos log("") log("===== Analyzing Repos =====") @@ -3848,7 +3541,7 @@ def analyze_things(self): # This completely creates: # data["buildroot"]["koji_srpms"][koji_id][arch][srpm_id]... # data["buildroot"]["srpms"][repo_id][arch][srpm_id]... - + # log("") log("===== Analyzing Buildroot =====") log("") @@ -3857,6 +3550,7 @@ def analyze_things(self): self._record_metric("finished _analyze_buildroot()") # Add buildroot packages to views + # # Further extends the following with buildroot packages: # data["views"][view_id]["pkgs"] # data["views"][view_id]["source_pkgs"] @@ -3892,11 +3586,13 @@ def analyze_things(self): self._record_metric("finished _recommend_maintainers()") + # Finally, save the cache for next time dump_data(self.settings["root_log_deps_cache_path"], self.cache["root_log_deps"]["next"]) self._record_metric("finished dumping the root log data cache") - self._record_metric("finished analyze_things()") + + self._record_metric("finished analyze_things()") return self.data diff --git a/content_resolver/config_manager.py b/content_resolver/config_manager.py index eaad42e..bdc12ed 100644 --- a/content_resolver/config_manager.py +++ b/content_resolver/config_manager.py @@ -98,7 +98,9 @@ def _load_config_repo_v2(self, document_id, document, settings): for arch_raw in document["data"]["source"]["architectures"]: arch = str(arch_raw) if arch not in settings["allowed_arches"]: - log(f" Warning: {document_id}.yaml lists an unsupported architecture: {arch}. Moving on...") + log(" Warning: {file}.yaml lists an unsupported architecture: {arch}. Moving on...".format( + file=document_id, + arch=arch)) continue config["source"]["architectures"].append(str(arch)) except KeyError: @@ -118,8 +120,11 @@ def _load_config_repo_v2(self, document_id, document, settings): config["source"]["repos"][id]["name"] = name try: config["source"]["repos"][id]["baseurl"] = repo_data["baseurl"] - except KeyError as err: - raise ConfigError(f"'{document_id}.yaml' - is invalid. Repo {id} doesn't list baseurl.") from err + except KeyError: + # FIXME: No reference for `yaml_file` + raise ConfigError("'{file}.yaml' - is invalid. Repo {id} doesn't list baseurl.".format( + file=yml_file, + id=id)) config["source"]["repos"][id]["priority"] = priority config["source"]["repos"][id]["exclude"] = exclude config["source"]["repos"][id]["limit_arches"] = limit_arches @@ -185,7 +190,10 @@ def _load_config_env(self, document_id, document, settings): if "arch_packages" in document["data"]: for arch, pkgs in document["data"]["arch_packages"].items(): if arch not in settings["allowed_arches"]: - log(f" Warning: {document_id}.yaml lists an unsupported architecture: {arch}. Moving on...") + log(" Warning: {file}.yaml lists an unsupported architecture: {arch}. Moving on...".format( + file=document_id, + arch=arch + )) continue for pkg_raw in pkgs: pkg = str(pkg_raw) @@ -266,7 +274,9 @@ def _load_config_workload(self, document_id, document, settings): config["packages"].append(str(pkg)) except (TypeError, KeyError): pass # Because it's now valid - #log(f" Warning: {document_id} has an empty 'packages' field defined which is invalid. Moving on...") + #log(" Warning: {file} has an empty 'packages' field defined which is invalid. Moving on...".format( + # file=document_id + #)) # Architecture-specific packages. config["arch_packages"] = {} @@ -275,7 +285,10 @@ def _load_config_workload(self, document_id, document, settings): if "arch_packages" in document["data"]: for arch, pkgs in document["data"]["arch_packages"].items(): if arch not in settings["allowed_arches"]: - log(f" Warning: {document_id}.yaml lists an unsupported architecture: {arch}. Moving on...") + log(" Warning: {file}.yaml lists an unsupported architecture: {arch}. Moving on...".format( + file=document_id, + arch=arch + )) continue # This workaround allows for "arch_packages/ARCH" to be left empty in the config try: @@ -283,7 +296,10 @@ def _load_config_workload(self, document_id, document, settings): pkg = str(pkg_raw) config["arch_packages"][arch].append(pkg) except TypeError: - log(f" Warning: {document_id} has an empty 'arch_packages/{arch}' field defined which is invalid. Moving on...") + log(" Warning: {file} has an empty 'arch_packages/{arch}' field defined which is invalid. Moving on...".format( + file=document_id, + arch=arch + )) # Extra installation options. # The following are now supported: @@ -444,7 +460,10 @@ def _load_config_compose_view(self, document_id, document, settings): if "unwanted_arch_packages" in document["data"]: for arch, pkgs in document["data"]["unwanted_arch_packages"].items(): if arch not in settings["allowed_arches"]: - log(f" Warning: {document_id}.yaml lists an unsupported architecture: {arch}. Moving on...") + log(" Warning: {file}.yaml lists an unsupported architecture: {arch}. Moving on...".format( + file=document_id, + arch=arch + )) continue for pkg_raw in pkgs: pkg = str(pkg_raw) @@ -505,7 +524,10 @@ def _load_config_addon_view(self, document_id, document, settings): if "unwanted_arch_packages" in document["data"]: for arch, pkgs in document["data"]["unwanted_arch_packages"].items(): if arch not in settings["allowed_arches"]: - log(f" Warning: {document_id}.yaml lists an unsupported architecture: {arch}. Moving on...") + log(" Warning: {file}.yaml lists an unsupported architecture: {arch}. Moving on...".format( + file=document_id, + arch=arch + )) continue for pkg_raw in pkgs: pkg = str(pkg_raw) @@ -578,7 +600,10 @@ def _load_config_unwanted(self, document_id, document, settings): if "unwanted_arch_packages" in document["data"]: for arch, pkgs in document["data"]["unwanted_arch_packages"].items(): if arch not in settings["allowed_arches"]: - log(f" Warning: {document_id}.yaml lists an unsupported architecture: {arch}. Moving on...") + log(" Warning: {file}.yaml lists an unsupported architecture: {arch}. Moving on...".format( + file=document_id, + arch=arch + )) continue for pkg_raw in pkgs: pkg = str(pkg_raw) @@ -597,7 +622,10 @@ def _load_config_unwanted(self, document_id, document, settings): if "unwanted_arch_source_packages" in document["data"]: for arch, pkgs in document["data"]["unwanted_arch_source_packages"].items(): if arch not in settings["allowed_arches"]: - log(f" Warning: {document_id}.yaml lists an unsupported architecture: {arch}. Moving on...") + log(" Warning: {file}.yaml lists an unsupported architecture: {arch}. Moving on...".format( + file=document_id, + arch=arch + )) continue for pkg_raw in pkgs: pkg = str(pkg_raw) @@ -628,7 +656,10 @@ def _load_config_buildroot(self, document_id, document, settings): if "base_buildroot" in document["data"]: for arch, pkgs in document["data"]["base_buildroot"].items(): if arch not in settings["allowed_arches"]: - log(f" Warning: {document_id}.yaml lists an unsupported architecture: {arch}. Moving on...") + log(" Warning: {file}.yaml lists an unsupported architecture: {arch}. Moving on...".format( + file=document_id, + arch=arch + )) continue if pkgs: for pkg_raw in pkgs: @@ -641,7 +672,10 @@ def _load_config_buildroot(self, document_id, document, settings): if "source_packages" in document["data"]: for arch, srpms_dict in document["data"]["source_packages"].items(): if arch not in settings["allowed_arches"]: - log(f" Warning: {document_id}.yaml lists an unsupported architecture: {arch}. Moving on...") + log(" Warning: {file}.yaml lists an unsupported architecture: {arch}. Moving on...".format( + file=document_id, + arch=arch + )) continue if not srpms_dict: continue @@ -652,7 +686,9 @@ def _load_config_buildroot(self, document_id, document, settings): for pkg_raw in srpm_data["requires"]: requires.append(str(pkg_raw)) except TypeError: - log(f" Warning: {document_id} has an empty 'requires' field defined which is invalid. Moving on...") + log(" Warning: {file} has an empty 'requires' field defined which is invalid. Moving on...".format( + file=document_id + )) continue config["source_packages"][arch][str(srpm_name)] = {} @@ -672,7 +708,10 @@ def _load_json_data_buildroot_pkg_relations(self, document_id, document, setting # Arch arch = document["data"]["arch"] if arch not in settings["allowed_arches"]: - raise ConfigError(f"Error: '{document_id}.json' lists an unsupported architecture: {arch}.") + raise ConfigError("Error: '{file}.json' lists an unsupported architecture: {arch}.".format( + file=document_id, + arch=arch + )) config["arch"] = arch #pkg_relations @@ -822,7 +861,9 @@ def get_configs(self): try: document = yaml.safe_load(file) except yaml.YAMLError as err: - raise ConfigError(f"Error loading a config '{yml_file}': {err}") from err + raise ConfigError("Error loading a config '{filename}': {err}".format( + filename=yml_file, + err=err)) # Only accept yaml files stating their purpose! if not ("document" in document and "version" in document): @@ -929,8 +970,10 @@ def get_configs(self): try: try: json_data = self.load_data(os.path.join(directory, json_file)) - except Exception as err: - raise ConfigError(f"Error loading a JSON data file '{json_file}': {err}") from err + except: + raise ConfigError("Error loading a JSON data file '{filename}': {err}".format( + filename=json_file, + err=err)) # Only accept json files stating their purpose! if not ("document_type" in json_data and "version" in json_data): diff --git a/content_resolver/historia_data.py b/content_resolver/historia_data.py index 3e15ef8..6784bf7 100644 --- a/content_resolver/historia_data.py +++ b/content_resolver/historia_data.py @@ -15,7 +15,10 @@ def _save_current_historic_data(query): # Where to save it year = datetime.datetime.now().strftime("%Y") week = datetime.datetime.now().strftime("%W") - filename = f"historic_data-{year}-week_{week}.json" + filename = "historic_data-{year}-week_{week}.json".format( + year=year, + week=week + ) output_dir = os.path.join(query.settings["output"], "history") os.makedirs(output_dir, exist_ok=True) file_path = os.path.join(output_dir, filename) @@ -81,7 +84,9 @@ def _save_current_historic_data(query): history_data["views"][view_conf_id]["srpm_count_build_level_2_plus"] = view_all_arches["numbers"]["srpms"]["build_level_2_plus"] # And save it - log(f" Saving in: {file_path}") + log(" Saving in: {file_path}".format( + file_path=file_path + )) dump_data(file_path, history_data) log(" Done!") @@ -112,15 +117,23 @@ def _read_historic_data(query): date = datetime.datetime.strptime(document["date"],"%Y-%m-%d") year = date.strftime("%Y") week = date.strftime("%W") - key = f"{year}-week_{week}" + key = "{year}-week_{week}".format( + year=year, + week=week + ) except (KeyError, ValueError): - err_log(f"Invalid file in historic data: {filename}. Ignoring.") + err_log("Invalid file in historic data: {filename}. Ignoring.".format( + filename=filename + )) continue historic_data[key] = document return historic_data + log(" Done!") + log("") + def _generate_chartjs_data(historic_data, query): @@ -159,7 +172,9 @@ def _generate_chartjs_data(historic_data, query): entry_data["datasets"].append(dataset) - entry_name = f"chartjs-data--workload--{workload_id}" + entry_name = "chartjs-data--workload--{workload_id}".format( + workload_id=workload_id + ) _generate_json_file(entry_data, entry_name, query.settings) # Data for workload overview pages @@ -204,7 +219,10 @@ def _generate_chartjs_data(historic_data, query): entry_data["datasets"].append(dataset) - entry_name = f"chartjs-data--workload-overview--{workload_conf_id}--{repo_id}" + entry_name = "chartjs-data--workload-overview--{workload_conf_id}--{repo_id}".format( + workload_conf_id=workload_conf_id, + repo_id=repo_id + ) _generate_json_file(entry_data, entry_name, query.settings) # Data for workload cmp arches pages @@ -252,7 +270,11 @@ def _generate_chartjs_data(historic_data, query): entry_data["datasets"].append(dataset) - entry_name = f"chartjs-data--workload-cmp-arches--{workload_conf_id}--{env_conf_id}--{repo_id}" + entry_name = "chartjs-data--workload-cmp-arches--{workload_conf_id}--{env_conf_id}--{repo_id}".format( + workload_conf_id=workload_conf_id, + env_conf_id=env_conf_id, + repo_id=repo_id + ) _generate_json_file(entry_data, entry_name, query.settings) # Data for workload cmp envs pages @@ -300,7 +322,11 @@ def _generate_chartjs_data(historic_data, query): entry_data["datasets"].append(dataset) - entry_name = f"chartjs-data--workload-cmp-envs--{workload_conf_id}--{repo_id}--{arch}" + entry_name = "chartjs-data--workload-cmp-envs--{workload_conf_id}--{repo_id}--{arch}".format( + workload_conf_id=workload_conf_id, + repo_id=repo_id, + arch=arch + ) _generate_json_file(entry_data, entry_name, query.settings) # Data for env pages @@ -339,7 +365,9 @@ def _generate_chartjs_data(historic_data, query): entry_data["datasets"].append(dataset) - entry_name = f"chartjs-data--env--{env_id}" + entry_name = "chartjs-data--env--{env_id}".format( + env_id=env_id + ) _generate_json_file(entry_data, entry_name, query.settings) # Data for env overview pages @@ -384,7 +412,10 @@ def _generate_chartjs_data(historic_data, query): entry_data["datasets"].append(dataset) - entry_name = f"chartjs-data--env-overview--{env_conf_id}--{repo_id}" + entry_name = "chartjs-data--env-overview--{env_conf_id}--{repo_id}".format( + env_conf_id=env_conf_id, + repo_id=repo_id + ) _generate_json_file(entry_data, entry_name, query.settings) # Data for env cmp arches pages @@ -429,7 +460,10 @@ def _generate_chartjs_data(historic_data, query): entry_data["datasets"].append(dataset) - entry_name = f"chartjs-data--env-cmp-arches--{env_conf_id}--{repo_id}" + entry_name = "chartjs-data--env-cmp-arches--{env_conf_id}--{repo_id}".format( + env_conf_id=env_conf_id, + repo_id=repo_id + ) _generate_json_file(entry_data, entry_name, query.settings) # Data for view pages @@ -448,9 +482,21 @@ def _generate_chartjs_data(historic_data, query): # Second, get the actual data for everything that's needed entry_data["datasets"] = [] - dataset_names = ["env", "req", "dep"] if view_all_arches["has_buildroot"]: - dataset_names.extend(["build_base", "build_level_1", "build_level_2_plus"]) + dataset_names = [ + "env", + "req", + "dep", + "build_base", + "build_level_1", + "build_level_2_plus" + ] + else: + dataset_names = [ + "env", + "req", + "dep" + ] dataset_metadata = { "env": { @@ -506,7 +552,9 @@ def _generate_chartjs_data(historic_data, query): entry_data["datasets"].append(dataset) - entry_name = f"chartjs-data--view--{view_conf_id}" + entry_name = "chartjs-data--view--{view_conf_id}".format( + view_conf_id=view_conf_id + ) _generate_json_file(entry_data, entry_name, query.settings) diff --git a/content_resolver/page_generation.py b/content_resolver/page_generation.py index dfce6ab..0950f36 100644 --- a/content_resolver/page_generation.py +++ b/content_resolver/page_generation.py @@ -6,13 +6,17 @@ def _generate_html_page(template_name, template_data, page_name, settings): - log(f"Generating the '{page_name}' page...") + log("Generating the '{page_name}' page...".format( + page_name=page_name + )) output = settings["output"] template_env = settings["jinja2_template_env"] - template = template_env.get_template(f"{template_name}.html") + template = template_env.get_template("{template_name}.html".format( + template_name=template_name + )) if not template_data: template_data = {} @@ -20,14 +24,16 @@ def _generate_html_page(template_name, template_data, page_name, settings): page = template.render(**template_data) - # Prevent path traversal by extracting only the basename - safe_page_name = os.path.basename(page_name.replace(":", "--")) - filename = f"{safe_page_name}.html" + filename = ("{page_name}.html".format( + page_name=page_name.replace(":", "--") + )) - log(f" Writing file... ({filename})") + log(" Writing file... ({filename})".format( + filename=filename + )) with open(os.path.join(output, filename), "w") as file: file.write(page) - + log(" Done!") log("") @@ -44,7 +50,10 @@ def _generate_workload_pages(query): "repo_id": repo_id } - page_name = f"workload-overview--{workload_conf_id}--{repo_id}" + page_name = "workload-overview--{workload_conf_id}--{repo_id}".format( + workload_conf_id=workload_conf_id, + repo_id=repo_id + ) _generate_html_page("workload_overview", template_data, page_name, query.settings) # Workload detail pages @@ -70,9 +79,13 @@ def _generate_workload_pages(query): "repo": repo } - page_name = f"workload--{workload_id}" + page_name = "workload--{workload_id}".format( + workload_id=workload_id + ) _generate_html_page("workload", template_data, page_name, query.settings) - page_name = f"workload-dependencies--{workload_id}" + page_name = "workload-dependencies--{workload_id}".format( + workload_id=workload_id + ) _generate_html_page("workload_dependencies", template_data, page_name, query.settings) # Workload compare arches pages @@ -109,7 +122,11 @@ def _generate_workload_pages(query): "rows": rows } - page_name = f"workload-cmp-arches--{workload_conf_id}--{env_conf_id}--{repo_id}" + page_name = "workload-cmp-arches--{workload_conf_id}--{env_conf_id}--{repo_id}".format( + workload_conf_id=workload_conf_id, + env_conf_id=env_conf_id, + repo_id=repo_id + ) _generate_html_page("workload_cmp_arches", template_data, page_name, query.settings) @@ -145,7 +162,11 @@ def _generate_workload_pages(query): "rows": rows } - page_name = f"workload-cmp-envs--{workload_conf_id}--{repo_id}--{arch}" + page_name = "workload-cmp-envs--{workload_conf_id}--{repo_id}--{arch}".format( + workload_conf_id=workload_conf_id, + repo_id=repo_id, + arch=arch + ) _generate_html_page("workload_cmp_envs", template_data, page_name, query.settings) @@ -164,7 +185,10 @@ def _generate_env_pages(query): "repo_id": repo_id } - page_name = f"env-overview--{env_conf_id}--{repo_id}" + page_name = "env-overview--{env_conf_id}--{repo_id}".format( + env_conf_id=env_conf_id, + repo_id=repo_id + ) _generate_html_page("env_overview", template_data, page_name, query.settings) # env detail pages @@ -185,10 +209,14 @@ def _generate_env_pages(query): "repo": repo } - page_name = f"env--{env_id}" + page_name = "env--{env_id}".format( + env_id=env_id + ) _generate_html_page("env", template_data, page_name, query.settings) - page_name = f"env-dependencies--{env_id}" + page_name = "env-dependencies--{env_id}".format( + env_id=env_id + ) _generate_html_page("env_dependencies", template_data, page_name, query.settings) # env compare arches pages @@ -221,7 +249,10 @@ def _generate_env_pages(query): "rows": rows } - page_name = f"env-cmp-arches--{env_conf_id}--{repo_id}" + page_name = "env-cmp-arches--{env_conf_id}--{repo_id}".format( + env_conf_id=env_conf_id, + repo_id=repo_id + ) _generate_html_page("env_cmp_arches", template_data, page_name, query.settings) @@ -240,11 +271,15 @@ def _generate_maintainer_pages(query): } # Overview page - page_name = f"maintainer--{maintainer}" + page_name = "maintainer--{maintainer}".format( + maintainer=maintainer + ) _generate_html_page("maintainer_overview", template_data, page_name, query.settings) # My Workloads page - page_name = f"maintainer-workloads--{maintainer}" + page_name = "maintainer-workloads--{maintainer}".format( + maintainer=maintainer + ) _generate_html_page("maintainer_workloads", template_data, page_name, query.settings) log(" Done!") @@ -259,7 +294,9 @@ def _generate_config_pages(query): "query": query, "conf_type": conf_type } - page_name = f"configs_{conf_type}" + page_name = "configs_{conf_type}".format( + conf_type=conf_type + ) _generate_html_page("configs", template_data, page_name, query.settings) # Config repo pages @@ -268,7 +305,9 @@ def _generate_config_pages(query): "query": query, "repo_conf": repo_conf } - page_name = f"config-repo--{repo_id}" + page_name = "config-repo--{repo_id}".format( + repo_id=repo_id + ) _generate_html_page("config_repo", template_data, page_name, query.settings) # Config env pages @@ -277,7 +316,9 @@ def _generate_config_pages(query): "query": query, "env_conf": env_conf } - page_name = f"config-env--{env_conf_id}" + page_name = "config-env--{env_conf_id}".format( + env_conf_id=env_conf_id + ) _generate_html_page("config_env", template_data, page_name, query.settings) # Config workload pages @@ -286,7 +327,9 @@ def _generate_config_pages(query): "query": query, "workload_conf": workload_conf } - page_name = f"config-workload--{workload_conf_id}" + page_name = "config-workload--{workload_conf_id}".format( + workload_conf_id=workload_conf_id + ) _generate_html_page("config_workload", template_data, page_name, query.settings) # Config label pages @@ -295,7 +338,9 @@ def _generate_config_pages(query): "query": query, "label_conf": label_conf } - page_name = f"config-label--{label_conf_id}" + page_name = "config-label--{label_conf_id}".format( + label_conf_id=label_conf_id + ) _generate_html_page("config_label", template_data, page_name, query.settings) # Config view pages @@ -304,7 +349,9 @@ def _generate_config_pages(query): "query": query, "view_conf": view_conf } - page_name = f"config-view--{view_conf_id}" + page_name = "config-view--{view_conf_id}".format( + view_conf_id=view_conf_id + ) _generate_html_page("config_view", template_data, page_name, query.settings) # Config unwanted pages @@ -313,7 +360,9 @@ def _generate_config_pages(query): "query": query, "unwanted_conf": unwanted_conf } - page_name = f"config-unwanted--{unwanted_conf_id}" + page_name = "config-unwanted--{unwanted_conf_id}".format( + unwanted_conf_id=unwanted_conf_id + ) _generate_html_page("config_unwanted", template_data, page_name, query.settings) log(" Done!") @@ -330,7 +379,10 @@ def _generate_repo_pages(query): "repo": repo, "arch": arch } - page_name = f"repo--{repo_id}--{arch}" + page_name = "repo--{repo_id}--{arch}".format( + repo_id=repo_id, + arch=arch + ) _generate_html_page("repo", template_data, page_name, query.settings) @@ -352,27 +404,39 @@ def _generate_view_pages(query): } # Generate the overview page - page_name = f"view--{view_conf_id}" + page_name = "view--{view_conf_id}".format( + view_conf_id=view_conf_id + ) _generate_html_page("view_overview", template_data, page_name, query.settings) # Generate the packages page - page_name = f"view-packages--{view_conf_id}" + page_name = "view-packages--{view_conf_id}".format( + view_conf_id=view_conf_id + ) _generate_html_page("view_packages", template_data, page_name, query.settings) # Generate the source packages page - page_name = f"view-sources--{view_conf_id}" + page_name = "view-sources--{view_conf_id}".format( + view_conf_id=view_conf_id + ) _generate_html_page("view_sources", template_data, page_name, query.settings) # Generate the unwanted packages page - page_name = f"view-unwanted--{view_conf_id}" + page_name = "view-unwanted--{view_conf_id}".format( + view_conf_id=view_conf_id + ) _generate_html_page("view_unwanted", template_data, page_name, query.settings) # Generate the workloads page - page_name = f"view-workloads--{view_conf_id}" + page_name = "view-workloads--{view_conf_id}".format( + view_conf_id=view_conf_id + ) _generate_html_page("view_workloads", template_data, page_name, query.settings) # Generate the errors page - page_name = f"view-errors--{view_conf_id}" + page_name = "view-errors--{view_conf_id}".format( + view_conf_id=view_conf_id + ) _generate_html_page("view_errors", template_data, page_name, query.settings) @@ -381,7 +445,10 @@ def _generate_view_pages(query): # Generate the arch lists for arch in view_conf["architectures"]: - view_id = f"{view_conf_id}:{arch}" + view_id = "{view_conf_id}:{arch}".format( + view_conf_id=view_conf_id, + arch=arch + ) view = query.data["views"][view_id] @@ -391,7 +458,10 @@ def _generate_view_pages(query): "view": view, "arch": arch, } - page_name = f"view--{view_conf_id}--{arch}" + page_name = "view--{view_conf_id}--{arch}".format( + view_conf_id=view_conf_id, + arch=arch + ) #_generate_html_page("view_packages", template_data, page_name, query.settings) # ... @@ -405,7 +475,10 @@ def _generate_view_pages(query): "view_all_arches": view_all_arches, "pkg": pkg, } - page_name = f"view-rpm--{view_conf_id}--{pkg_name}" + page_name = "view-rpm--{view_conf_id}--{pkg_name}".format( + view_conf_id=view_conf_id, + pkg_name=pkg_name + ) _generate_html_page("view_rpm", template_data, page_name, query.settings) _generate_json_file(pkg, page_name, query.settings) @@ -419,7 +492,10 @@ def _generate_view_pages(query): "view_all_arches": view_all_arches, "srpm": srpm, } - page_name = f"view-srpm--{view_conf_id}--{srpm_name}" + page_name = "view-srpm--{view_conf_id}--{srpm_name}".format( + view_conf_id=view_conf_id, + srpm_name=srpm_name + ) _generate_html_page("view_srpm", template_data, page_name, query.settings) _generate_json_file(srpm, page_name, query.settings) diff --git a/content_resolver/utils.py b/content_resolver/utils.py index b80d1d2..c4e4b91 100644 --- a/content_resolver/utils.py +++ b/content_resolver/utils.py @@ -2,12 +2,7 @@ import json import re import sys -from contextlib import contextmanager -from typing import Iterator - import jinja2 -import libdnf5 - class SetEncoder(json.JSONEncoder): def default(self, obj): @@ -71,27 +66,4 @@ def url_to_id(url): def datetime_now_string(): - return datetime.datetime.now().strftime("%m/%d/%Y, %H:%M:%S") - - -@contextmanager -def dnf5_base() -> Iterator[libdnf5.base.Base]: - """ - Context manager wrapper for libdnf5.base.Base(). - - DNF5's Base class doesn't natively support the context manager protocol, - so this wrapper provides proper resource management. - - Usage: - with dnf5_base() as base: - config = base.get_config() - # ... use base ... - """ - base = libdnf5.base.Base() - try: - yield base - finally: - # DNF5 Base cleanup is handled by Python's garbage collector - # No explicit cleanup needed, but the finally block ensures - # proper exception handling and resource cleanup if needed in future - pass + return datetime.datetime.now().strftime("%m/%d/%Y, %H:%M:%S") \ No newline at end of file