diff --git a/nf_core/commands_modules.py b/nf_core/commands_modules.py index c2717b4852..21c5791c73 100644 --- a/nf_core/commands_modules.py +++ b/nf_core/commands_modules.py @@ -4,7 +4,6 @@ import rich -from nf_core.pipelines.containers_utils import try_generate_container_configs from nf_core.utils import CONTAINER_PLATFORMS, rich_force_colors log = logging.getLogger(__name__) @@ -328,7 +327,13 @@ def modules_bump_versions(ctx, tool, directory, all_modules, show_all, dry_run): ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"], ) - version_bumper.bump_versions(module=tool, all_modules=all_modules, show_up_to_date=show_all, dry_run=dry_run) + version_bumper.bump_versions( + module=tool, + all_modules=all_modules, + show_up_to_date=show_all, + dry_run=dry_run, + hide_progress=ctx.obj["hide_progress"], + ) except ModuleExceptionError as e: log.error(e) sys.exit(1) @@ -341,92 +346,31 @@ def modules_containers_create(ctx, module: str, directory: Path, force: bool) -> """ Build docker and singularity containers for linux/arm64 and linux/amd64 using wave. """ - from rich.console import Group - from rich.live import Live - from rich.progress import BarColumn, MofNCompleteColumn, Progress, SpinnerColumn, TextColumn - from nf_core.modules.containers import ModuleContainers - from nf_core.pipelines.lint_utils import console try: - manager = ModuleContainers(module=module, directory=directory, verbose=ctx.obj["verbose"]) + module_containers = ModuleContainers(module=module, directory=directory, verbose=ctx.obj["verbose"]) ModuleContainers.check_tower_token() - # Normalize to a list of components: batch mode processes all available - # modules, single-module mode is just a list of one. - if manager.all_modules: - if not manager.available_modules: + if module_containers.all_modules: + if not module_containers.available_modules: log.error("No modules found to build containers for") sys.exit(1) - log.info(f"Building containers for {len(manager.available_modules)} module(s)") - components = manager.available_modules - else: - assert manager.nfcore_component is not None - components = [manager.nfcore_component] - - failed_modules = [] + log.info(f"Building containers for {len(module_containers.available_modules)} module(s)") - overall_progress = Progress( - "[bold blue]{task.description}", - BarColumn(), - MofNCompleteColumn(), - console=console, - disable=ctx.obj["hide_progress"], - ) - module_progress = Progress( - SpinnerColumn(finished_text="[green]✓[/green]"), - "[bold blue]{task.description}", - TextColumn("{task.fields[status]}"), - console=console, - disable=ctx.obj["hide_progress"], + failed_modules = module_containers.build_containers_with_progress( + force=force, hide_progress=ctx.obj["hide_progress"] ) - with Live(Group(overall_progress, module_progress), console=console, transient=True): - # The overall bar is only useful when processing more than one module - overall_task_id = overall_progress.add_task("modules", total=len(components), visible=len(components) > 1) - for component in components: - module_name = component.component_name - module_task_id = module_progress.add_task( - f"[cyan]{module_name}[/cyan]", - total=None, - status="building containers...", - ) - try: - if manager.all_modules: - # Per-module instance, reusing the already-scanned component list - module_manager = ModuleContainers( - module=module_name, - directory=directory, - verbose=ctx.obj["verbose"], - components=manager.available_modules, - ) - else: - module_manager = manager - _, success = module_manager.create( - progress_bar=module_progress, task_id=module_task_id, force=force - ) - if success: - module_manager.update_containers_in_meta() - if module_manager.repo_type == "pipeline": - try_generate_container_configs(directory, module_manager.module_directory) - else: - failed_modules.append(module_name) - except (ValueError, RuntimeError, OSError) as e: - log.error(f"✗ Failed to build containers for {module_name}: {e}") - failed_modules.append(module_name) - finally: - overall_progress.advance(overall_task_id) - module_progress.remove_task(module_task_id) - if failed_modules: - if manager.all_modules: + if module_containers.all_modules: log.warning( f"Failed to build containers for {len(failed_modules)} module(s): {', '.join(failed_modules)}" ) else: - log.error(f"✗ Some container builds failed for {manager.module}") + log.error(f"✗ Some container builds failed for {module_containers.module}") sys.exit(1) - elif manager.all_modules: + elif module_containers.all_modules: log.info("Successfully built containers for all modules") except (UserWarning, LookupError, FileNotFoundError, ValueError, RuntimeError) as e: @@ -441,8 +385,8 @@ def modules_containers_conda_lock(ctx, module, platform=CONTAINER_PLATFORMS[0]): from nf_core.modules.containers import ModuleContainers try: - manager = ModuleContainers(module, ".", verbose=ctx.obj["verbose"]) - lock_file = manager.get_conda_lock_file(platform) + module_containers = ModuleContainers(module, ".", verbose=ctx.obj["verbose"]) + lock_file = module_containers.get_conda_lock_file(platform) stdout.print(lock_file) except (UserWarning, LookupError, FileNotFoundError, ValueError, RuntimeError) as e: log.error(e) @@ -456,8 +400,8 @@ def modules_containers_list(ctx, module, plain_text=False): from nf_core.modules.containers import ModuleContainers try: - manager = ModuleContainers(module, ".", verbose=ctx.obj["verbose"]) - containers = manager.list_containers() + module_containers = ModuleContainers(module, ".", verbose=ctx.obj["verbose"]) + containers = module_containers.list_containers() if plain_text: for cs, p, img in containers: diff --git a/nf_core/components/create.py b/nf_core/components/create.py index 78e1c5f7d7..a23ea4b9bf 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -177,6 +177,13 @@ def create(self) -> bool: run_prettier_on_file(new_files) log.info("Created following files:\n " + "\n ".join(new_files)) + + if self.component_type == "modules": + log.info( + f"Build Docker & Singularity images (Seqera/Wave) and update [magenta]main.nf[/] with the container URLs:\n" + f" [blue]nf-core modules containers create {self.component_name}[/]" + ) + return True def _get_bioconda_tool(self): diff --git a/nf_core/modules/bump_versions.py b/nf_core/modules/bump_versions.py index eb0e923324..ba5e2920be 100644 --- a/nf_core/modules/bump_versions.py +++ b/nf_core/modules/bump_versions.py @@ -21,7 +21,7 @@ import nf_core.utils from nf_core.components.components_command import ComponentCommand from nf_core.components.nfcore_component import NFCoreComponent -from nf_core.utils import NFCoreYamlConfig, custom_yaml_dumper, rich_force_colors +from nf_core.utils import NFCoreYamlConfig, rich_force_colors from nf_core.utils import plural_s as _s log = logging.getLogger(__name__) @@ -50,26 +50,29 @@ def bump_versions( all_modules: bool = False, show_up_to_date: bool = False, dry_run: bool = False, + hide_progress: bool = False, ) -> list[NFCoreComponent]: """ Bump the container and conda version of single module or all modules. If module is the name of a directory in the modules directory, all modules in that directory will be bumped. - Looks for a bioconda tool version in the `main.nf` file of the module and checks whether - are more recent version is available. If yes, then tries to get docker/singularity - container links and replace the bioconda version and the container links in the main.nf file - of the respective module. + Looks for a bioconda tool version in the `environment.yml` file of the module and checks + whether a more recent version is available. If yes, the pinned version + in `environment.yml` is bumped and the Docker/Singularity containers are rebuilt from it + with Wave. Args: module: a specific module to update all_modules: whether to bump versions for all modules show_up_to_date: whether to show up-to-date modules as well dry_run: whether to dry run the command + hide_progress: whether to hide the progress bars Returns: list[NFCoreComponent]: the updated modules """ + hide_progress = hide_progress or os.environ.get("HIDE_PROGRESS", None) is not None self.up_to_date = [] self.updated = [] self.failed = [] @@ -121,8 +124,9 @@ def bump_versions( BarColumn(bar_width=None), "[magenta]{task.completed} of {task.total}[reset] » [bold yellow]{task.fields[test_name]}", transient=True, - disable=os.environ.get("HIDE_PROGRESS", None) is not None, + disable=hide_progress, ) + modules_to_rebuild: list[NFCoreComponent] = [] with progress_bar: bump_progress = progress_bar.add_task( "Bumping nf-core modules versions", @@ -131,39 +135,72 @@ def bump_versions( ) for mod in nfcore_modules: progress_bar.update(bump_progress, advance=1, test_name=mod.component_name) - self.bump_module_version(mod) + if self.bump_module_version(mod): + modules_to_rebuild.append(mod) + + if modules_to_rebuild: + self._build_wave_containers(modules_to_rebuild, hide_progress) self._print_results() return nfcore_modules + def _build_wave_containers(self, modules: list[NFCoreComponent], hide_progress: bool = False) -> None: + """Rebuild the Docker/Singularity containers for ``modules`` with Seqera Wave. + + Builds from each module's (already bumped) environment.yml and updates main.nf, + meta.yml and the conda lock files. Modules whose build fails are added to + ``self.failed``. + """ + from nf_core.modules.containers import ModuleContainers + + log.info(f"Building Seqera containers for {len(modules)} bumped module{_s(modules)} with Wave...") + module_containers = ModuleContainers( + module=None, + directory=self.directory, + all_modules=True, + components=modules, + ) + ModuleContainers.check_tower_token() + failed_modules = module_containers.build_containers_with_progress(hide_progress=hide_progress) + for module_name in failed_modules: + self.failed.append(("Container build with Wave failed", module_name)) + def bump_module_version(self, module: NFCoreComponent) -> bool: """ Bump the bioconda and container version of a single NFCoreComponent Args: module: NFCoreComponent + + Returns: + True if the module's ``environment.yml`` was bumped and its containers therefore + need rebuilding; False if it was already up to date, ignored or failed. """ config_version = None - bioconda_packages = [] - try: - # Extract bioconda version from `environment.yml` - bioconda_packages = self.get_bioconda_version(module) - except FileNotFoundError: - # try it in the main.nf instead - try: - with open(module.main_nf) as fh: - for line in fh: - if "bioconda::" in line: - bioconda_packages = [b for b in line.split() if "bioconda::" in b] - except FileNotFoundError: - log.error( - f"Neither `environment.yml` nor `main.nf` of {module.component_name} module could be read to get bioconada version of used tools." - ) - - # If multiple versions - don't update! (can't update mulled containers) - if not bioconda_packages or len(bioconda_packages) > 1: - self.failed.append(("Ignoring mulled container", module.component_name)) + # Extract the pinned conda dependencies from `environment.yml`. Wave rebuilds the containers + # straight from this file, so every pinned package can be bumped - including multi-package + # ("mulled") modules, which earlier versions had to skip because the old mulled container + # names could not be regenerated. + dependencies = self.get_bioconda_version(module) + + # Only channel-pinned, versioned deps (e.g. `bioconda::samtools=1.21`) can be bumped; skip + # pip sub-dicts, unpinned deps and anything without a `channel::name=version` shape. + conda_packages: list[tuple[str, str, str]] = [] # (channel, tool_name, current_version) + for dep in dependencies: + if not isinstance(dep, str): + continue + spec = dep.strip("'").strip('"') + if "::" not in spec or "=" not in spec: + continue + channel, name_version = spec.split("::", 1) + tool_name, current_version = name_version.split("=", 1) + # Drop an optional build string (`name=version=build`) + current_version = current_version.split("=", 1)[0] + conda_packages.append((channel, tool_name, current_version)) + + if not conda_packages: + self.failed.append(("No pinned conda dependencies to bump", module.component_name)) return False # Don't update if blocked in blacklist @@ -173,118 +210,139 @@ def bump_module_version(self, module: NFCoreComponent) -> bool: if not config_version: self.ignored.append(("Omitting module due to config.", module.component_name)) return False - - # check for correct version and newer versions - bioconda_tool_name = bioconda_packages[0].split("=")[0].replace("bioconda::", "").strip("'").strip('"') - bp = bioconda_packages[0] - bp = bp.strip("'").strip('"') - bioconda_version = bp.split("=")[1] - - if not config_version: - try: - response = nf_core.utils.anaconda_package(bp) - except (LookupError, ValueError): + if len(conda_packages) > 1: + # A single pinned version in the config can't be mapped onto a multi-package module self.failed.append( - ( - f"Conda version not specified correctly: {Path(module.main_nf).relative_to(self.directory)}", - module.component_name, - ) + ("Cannot pin a version via config for a multi-package module", module.component_name) ) return False - # Check that required version is available at all - if bioconda_version not in response.get("versions"): - self.failed.append((f"Conda package had unknown version: `{module.main_nf}`", module.component_name)) - return False - - # Check version is latest available - last_ver = response.get("latest_version") - else: - last_ver = config_version - - if last_ver is not None and last_ver != bioconda_version: - log.debug(f"Updating version for {module.component_name}") - # Get docker and singularity container links - try: - docker_img, singularity_img = nf_core.utils.get_biocontainer_tag(bioconda_tool_name, last_ver) - except LookupError as e: - self.failed.append((f"Could not download container tags: {e}", module.component_name)) - return False - - patterns = [ - (rf"biocontainers/{bioconda_tool_name}:[^'\"\s]+", docker_img), - ( - rf"https://depot.galaxyproject.org/singularity/{bioconda_tool_name}:[^'\"\s]+", - singularity_img, - ), - ] - - with open(module.main_nf) as fh: - content = fh.read() + # Resolve the target version for every package: the latest available on Anaconda, or the + # version pinned in the config blacklist. + targets: list[tuple[str, str, str, str]] = [] # (channel, tool_name, current_version, target_version) + for channel, tool_name, current_version in conda_packages: + if config_version: + target_version = config_version + else: + dep_spec = f"{channel}::{tool_name}={current_version}" + try: + response = nf_core.utils.anaconda_package(dep_spec) + except (LookupError, ValueError): + self.failed.append( + ( + f"Conda version not specified correctly: {Path(module.environment_yml).relative_to(self.directory) if module.environment_yml else module.component_name}", + module.component_name, + ) + ) + return False - # Go over file content of main.nf and find replacements - for pattern in patterns: - found_match = False - newcontent = [] - for line in content.splitlines(): - # Match the pattern - matches_pattern = re.findall(rf"^.*{pattern[0]}.*$", line) - if matches_pattern: - found_match = True - - # Replace the match - newline = re.sub(pattern[0], pattern[1], line) - newcontent.append(newline) - # No match, keep line as it is - else: - newcontent.append(line) - - if found_match: - content = "\n".join(newcontent) + "\n" - else: + # Check that the pinned version is available at all + if current_version not in response.get("versions"): self.failed.append( - (f"Did not find pattern {pattern[0]} in module {module.component_name}", module.component_name) + (f"Conda package had unknown version: `{module.environment_yml}`", module.component_name) ) return False - # Write new content to the file - with open(module.main_nf, "w") as fh: - fh.write(content) + target_version = response.get("latest_version") + + if target_version is None: + continue + targets.append((channel, tool_name, current_version, target_version)) - # change version in environment.yml + if not targets: + self.up_to_date.append((f"Module version up to date: {module.component_name}", module.component_name)) + return False + + # Check 1: bump any dependency behind the latest. Wave rebuilds the containers from + # environment.yml afterwards (see bump_versions()), updating main.nf, meta.yml and conda locks. + outdated = [t for t in targets if t[3] != t[2]] + if outdated: if not module.environment_yml: log.error(f"Could not read `environment.yml` of {module.component_name} module.") + self.failed.append(("No environment.yml found to update", module.component_name)) return False + # Rewrite the versions in the raw file text so YAML comments (e.g. renovate hints) and + # formatting survive - a yaml load/dump round-trip would silently drop them. with open(module.environment_yml) as fh: - env_yml = yaml.safe_load(fh) - env_yml["dependencies"][0] = re.sub( - bioconda_packages[0], f"bioconda::{bioconda_tool_name}={last_ver}", env_yml["dependencies"][0] - ) + content = fh.read() + for channel, tool_name, current_version, target_version in outdated: + log.debug(f"Updating {tool_name} version for {module.component_name}") + content = re.sub( + rf"({re.escape(channel)}::{re.escape(tool_name)})=[^\s'\"]+", + rf"\g<1>={target_version}", + content, + ) + self.updated.append( + (f"Module updated: {tool_name} {current_version} --> {target_version}", module.component_name) + ) with open(module.environment_yml, "w") as fh: - yaml.dump(env_yml, fh, default_flow_style=False, Dumper=custom_yaml_dumper()) + fh.write(content) + return True - self.updated.append( - ( - f"Module updated: {bioconda_version} --> {last_ver}", - module.component_name, + # Check 2: environment.yml is already current, but make sure the containers were actually + # built for every package. An earlier run may have bumped environment.yml and then been + # interrupted before the Wave build wrote the new containers / conda locks. + for _channel, tool_name, _current_version, target_version in targets: + if not self._containers_built_for_version(module, tool_name, target_version): + self.updated.append( + (f"Containers out of date - rebuilding for version {target_version}", module.component_name) ) + return True + + self.up_to_date.append((f"Module version up to date: {module.component_name}", module.component_name)) + return False + + def _containers_built_for_version(self, module: NFCoreComponent, tool_name: str, version: str) -> bool: + """Return True if the module's Wave conda-lock files already pin ``tool_name`` at ``version``. + + Image names are content hashes, so the version-bearing conda locks are checked instead, + assuming the bioconda dependency name matches the conda package name. No ``.conda-lock`` dir + means the module was never Wave-built (left alone); an empty dir, or a meta.yml without a + ``containers:`` section, means an interrupted build (rebuilt). + """ + from nf_core.components.components_utils import read_meta_yml + + conda_lock_dir = module.component_dir / ".conda-lock" + if not conda_lock_dir.is_dir(): + log.debug( + f"No conda-lock directory for {module.component_name}; treating as up to date " + "(Wave migration is out of scope for bump-versions)" ) return True + lock_files = list(conda_lock_dir.glob("*.txt")) + if not lock_files: + # Empty dir: a finished build always leaves locks, so this one was interrupted. + log.debug(f"Empty conda-lock directory for {module.component_name}; containers need rebuilding") + return False + # Lock entries look like `.../samtools-1.21-h50ea8bc_0.conda`; trailing `-` keeps 1.21 from + # matching 1.210. Stream each file so the search short-circuits. + needle = f"/{tool_name}-{version}-" + for lock_file in lock_files: + with lock_file.open() as fh: + if not any(needle in line for line in fh): + return False - else: - self.up_to_date.append((f"Module version up to date: {module.component_name}", module.component_name)) - return True + # The containers section is written last, so its absence means an interrupted build. Only + # require it to be present, not complete (arm64 builds may legitimately be missing). + if module.meta_yml is None or not Path(module.meta_yml).exists(): + log.debug(f"No meta.yml for {module.component_name}; containers need rebuilding") + return False + if not read_meta_yml(Path(module.meta_yml)).get("containers"): + log.debug(f"meta.yml for {module.component_name} has no containers section; containers need rebuilding") + return False + + return True def get_bioconda_version(self, module: NFCoreComponent) -> list[str]: """ Extract the bioconda version from a module """ # Check whether file exists and load it - bioconda_packages = [] + bioconda_packages: list[str] = [] if module.environment_yml is not None and module.environment_yml.exists(): with open(module.environment_yml) as fh: env_yml = yaml.safe_load(fh) - bioconda_packages = env_yml.get("dependencies", []) + bioconda_packages = (env_yml or {}).get("dependencies") or [] else: log.error(f"Could not read `environment.yml` of {module.component_name} module.") diff --git a/nf_core/modules/containers.py b/nf_core/modules/containers.py index 9e701617c0..f3788f1053 100644 --- a/nf_core/modules/containers.py +++ b/nf_core/modules/containers.py @@ -1,7 +1,9 @@ import base64 +import hashlib import logging import os import re +import shutil import threading import time from collections.abc import Callable @@ -86,6 +88,16 @@ class ModuleContainers: Helpers for building, linting and listing module containers. """ + # Wave build results, keyed by (environment.yml content hash, container_system, platform). + # Modules with an identical environment.yml (e.g. samtools/sort and samtools/view) resolve + # to the same container, so this avoids submitting duplicate builds within one process. + _build_cache: dict[tuple[str, str, str], ContainerEntry] = {} + # Path to an already-written conda lock file, keyed by Wave build_id. Modules that share + # a cached build (above) also share a build_id, so a later module can copy the file that + # an earlier one already downloaded instead of re-fetching identical lock contents. + _conda_lock_paths: dict[str, Path] = {} + _build_cache_lock = threading.Lock() + def __init__( self, module: str | None, @@ -452,11 +464,22 @@ def callback(build_id: str) -> None: conda_lock_path.parent.mkdir(parents=True, exist_ok=True) try: - # Download conda lock file (it will look up build_id from docker container) - log.debug(f"Downloading conda lock file for {platform} to {conda_lock_path}") - if progress_bar and task_id is not None: - progress_bar.update(task_id, status=f"conda lock {short_platform}...") - conda_lock_path.write_text(self.get_conda_lock_file(platform)) + with self._build_cache_lock: + cached_lock_path = self._conda_lock_paths.get(build_id) + if cached_lock_path is not None and cached_lock_path.exists(): + log.debug(f"Reusing conda lock file for build {build_id} (from {cached_lock_path})") + if progress_bar and task_id is not None: + progress_bar.update(task_id, status=f"conda lock {short_platform} (cached)...") + if cached_lock_path != conda_lock_path: + shutil.copyfile(cached_lock_path, conda_lock_path) + else: + # Download conda lock file (it will look up build_id from docker container) + log.debug(f"Downloading conda lock file for {platform} to {conda_lock_path}") + if progress_bar and task_id is not None: + progress_bar.update(task_id, status=f"conda lock {short_platform}...") + conda_lock_path.write_text(self.get_conda_lock_file(platform)) + with self._build_cache_lock: + self._conda_lock_paths[build_id] = conda_lock_path # Only register the entry once the lock has actually been written, so # meta.yml never points at a missing lock file. containers.conda[platform] = CondaEntry(lock_file=str(conda_lock_path)) @@ -509,6 +532,95 @@ def _raise_if_build_failed(result: dict, build_ref: str, details_uri: str = "") details = details_uri or result.get("detailsUri") or "" raise RuntimeError(f"Wave build {build_ref} failed: {reason}. Build log: {details}") + def build_containers_with_progress(self, force: bool = False, hide_progress: bool = False) -> list[str]: + """Build Seqera/Wave containers for the selected module(s) with a live progress display. + + Builds the single selected module, or every module in ``available_modules`` when + the instance was created with ``all_modules=True``. Shows an overall module bar + plus, per module, the per-build spinners (docker/singularity × platform) that + :meth:`create` renders when a progress bar is passed. + + Args: + force: Overwrite existing container directives even if already correct. + hide_progress: Disable the live progress display. + + Returns: + The names of the modules whose builds failed. + """ + from rich.console import Group + from rich.live import Live + from rich.progress import BarColumn, MofNCompleteColumn, Progress, SpinnerColumn, TextColumn + + from nf_core.pipelines.containers_utils import try_generate_container_configs + from nf_core.pipelines.lint_utils import console + + if self.all_modules: + components = self.available_modules + else: + assert self.nfcore_component is not None + components = [self.nfcore_component] + + failed_modules: list[str] = [] + + overall_progress = Progress( + "[bold blue]{task.description}", + BarColumn(), + MofNCompleteColumn(), + console=console, + disable=hide_progress, + ) + module_progress = Progress( + SpinnerColumn(finished_text="[green]✓[/green]"), + "[bold blue]{task.description}", + TextColumn("{task.fields[status]}"), + console=console, + disable=hide_progress, + ) + + # Batch builds defer config generation to one full scan after the loop. + batch = len(components) > 1 + + with Live(Group(overall_progress, module_progress), console=console, transient=True): + # The overall bar is only useful when processing more than one module + overall_task_id = overall_progress.add_task("modules", total=len(components), visible=len(components) > 1) + for component in components: + module_name = component.component_name + module_task_id = module_progress.add_task( + f"[cyan]{module_name}[/cyan]", + total=None, + status="building containers...", + ) + try: + if self.all_modules: + # Per-module instance, reusing the already-scanned component list + module_containers = ModuleContainers( + module=module_name, + directory=self.directory, + verbose=self.verbose, + components=self.available_modules, + ) + else: + module_containers = self + _, success = module_containers.create( + progress_bar=module_progress, task_id=module_task_id, force=force + ) + if success: + if module_containers.repo_type == "pipeline" and not batch: + try_generate_container_configs(self.directory, module_containers.module_directory) + else: + failed_modules.append(module_name) + except (ValueError, RuntimeError, OSError) as e: + log.error(f"✗ Failed to build containers for {module_name}: {e}") + failed_modules.append(module_name) + finally: + overall_progress.advance(overall_task_id) + module_progress.remove_task(module_task_id) + + if batch and self.repo_type == "pipeline" and len(failed_modules) < len(components): + try_generate_container_configs(self.directory) + + return failed_modules + @classmethod def _await_build( cls, @@ -573,12 +685,20 @@ def request_container( assert container_system in CONTAINER_SYSTEMS assert platform in CONTAINER_PLATFORMS + conda_bytes = conda_file.read_bytes() + cache_key = (hashlib.sha256(conda_bytes).hexdigest(), container_system, platform) + with cls._build_cache_lock: + cached_entry = cls._build_cache.get(cache_key) + if cached_entry is not None: + log.debug(f"Reusing cached {container_system}/{platform} build for identical environment.yml") + return cached_entry + # Submit the build via the Wave HTTP API (POST /v1alpha2/container). # `freeze` with no buildRepository pushes to the public community registry. payload: dict = { "packages": { "type": "CONDA", - "environment": base64.b64encode(conda_file.read_bytes()).decode(), + "environment": base64.b64encode(conda_bytes).decode(), }, "containerPlatform": platform, "freeze": True, @@ -642,7 +762,10 @@ def request_container( f"https://{ContainerRegistryUrls.SEQERA_SINGULARITY.value}/blobs/sha256/{digest[:2]}/{digest}/data" ) - return ContainerEntry(name=image, build_id=build_id, scan_id=scan_id, https=https_url) + entry = ContainerEntry(name=image, build_id=build_id, scan_id=scan_id, https=https_url) + with cls._build_cache_lock: + cls._build_cache[cache_key] = entry + return entry @classmethod def request_image_inspect(cls, image: str) -> dict: diff --git a/tests/modules/test_bump_versions.py b/tests/modules/test_bump_versions.py index 63e6eb9c2c..5736047c9b 100644 --- a/tests/modules/test_bump_versions.py +++ b/tests/modules/test_bump_versions.py @@ -1,20 +1,24 @@ import re import tempfile from pathlib import Path +from types import SimpleNamespace +from unittest import mock import pytest import ruamel.yaml import nf_core.modules.bump_versions from nf_core import __version__ -from nf_core.modules.modules_utils import ModuleExceptionError +from nf_core.modules.containers import ModuleContainers +from nf_core.modules.modules_utils import MetaYmlContainers, ModuleExceptionError from nf_core.utils import NFCoreYamlConfig from ..test_modules import TestModules class TestModulesBumpVersions(TestModules): - def test_modules_bump_versions_single_module(self): + @mock.patch.object(nf_core.modules.bump_versions.ModuleVersionBumper, "_build_wave_containers") + def test_modules_bump_versions_single_module(self, mock_build): """Test updating a single module""" # Change the bpipe/test version to an older version env_yml_path = Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "environment.yml") @@ -24,17 +28,35 @@ def test_modules_bump_versions_single_module(self): with open(env_yml_path, "w") as fh: fh.write(new_content) version_bumper = nf_core.modules.bump_versions.ModuleVersionBumper(pipeline_dir=self.nfcore_modules) + # mock_build keeps the Wave container rebuild from shelling out to the `wave` CLI modules = version_bumper.bump_versions(module="bpipe/test") assert len(version_bumper.failed) == 0 assert [m.component_name for m in modules] == ["bpipe/test"] - def test_modules_bump_versions_all_modules(self): + @mock.patch.object(nf_core.modules.bump_versions.ModuleVersionBumper, "_build_wave_containers") + def test_modules_bump_versions_all_modules(self, mock_build): """Test updating all modules""" version_bumper = nf_core.modules.bump_versions.ModuleVersionBumper(pipeline_dir=self.nfcore_modules) modules = version_bumper.bump_versions(all_modules=True) assert len(version_bumper.failed) == 0 assert [m.component_name for m in modules] == ["bpipe/test"] + @mock.patch.object(ModuleContainers, "create", return_value=(MetaYmlContainers(), False)) + def test_build_wave_containers(self, mock_create): + """Test building Wave containers and recording failed modules.""" + module_containers = ModuleContainers("bpipe/test", directory=self.nfcore_modules) + module = module_containers.nfcore_component + assert module is not None + version_bumper = nf_core.modules.bump_versions.ModuleVersionBumper(pipeline_dir=self.nfcore_modules) + + with mock.patch.dict("os.environ", {"TOWER_ACCESS_TOKEN": ""}): + version_bumper._build_wave_containers([module]) + + # ModuleContainers and its batch progress logic are real; only the external build is mocked. + mock_create.assert_called_once() + assert "TOWER_ACCESS_TOKEN is not set" in self.caplog.text + assert version_bumper.failed == [("Container build with Wave failed", "bpipe/test")] + @staticmethod def _mock_nf_core_yml(root_dir: Path) -> None: """Mock the .nf_core.yml""" @@ -87,6 +109,191 @@ def test_modules_bump_versions_submodules(self): out_modules = version_bumper.bump_versions(module="fgbio", dry_run=True) assert sorted([m.component_name for m in out_modules]) == sorted(in_modules) + def test_containers_built_for_version(self): + """Check-2: detect whether a module's conda locks already pin the current version.""" + # Bypass __init__ - the method only uses the passed module's component_dir + bumper = nf_core.modules.bump_versions.ModuleVersionBumper.__new__( + nf_core.modules.bump_versions.ModuleVersionBumper + ) + mod_dir = Path(tempfile.mkdtemp()) + meta_yml = mod_dir / "meta.yml" + module = SimpleNamespace(component_dir=mod_dir, component_name="samtools/sort", meta_yml=meta_yml) + + # No .conda-lock dir at all -> not Wave-managed, treated as up to date (no rebuild) + assert bumper._containers_built_for_version(module, "samtools", "1.21") is True + + lock_dir = mod_dir / ".conda-lock" + lock_dir.mkdir() + # Empty .conda-lock dir -> a Wave build was started but interrupted before writing locks + assert bumper._containers_built_for_version(module, "samtools", "1.21") is False + + (lock_dir / "linux_amd64-bd-abc_1.txt").write_text( + "- conda: https://conda.anaconda.org/bioconda/noarch/samtools-1.21-h50ea8bc_0.conda\n" + ) + # Lock current but meta.yml missing its containers section (build interrupted before the + # final meta.yml write) -> rebuild + meta_yml.write_text("name: samtools_sort\n") + assert bumper._containers_built_for_version(module, "samtools", "1.21") is False + + # Lock current AND meta.yml carries a containers section -> already built, no rebuild + meta_yml.write_text("name: samtools_sort\ncontainers:\n docker:\n linux/amd64:\n name: img\n") + assert bumper._containers_built_for_version(module, "samtools", "1.21") is True + # Trailing dash anchors the match so 1.21 does not also match 1.210 + assert bumper._containers_built_for_version(module, "samtools", "1.2") is False + # Lock pins an older version (e.g. interrupted build left stale locks) -> rebuild + assert bumper._containers_built_for_version(module, "samtools", "1.22") is False + + @mock.patch.object(nf_core.modules.bump_versions.ModuleVersionBumper, "_build_wave_containers") + @mock.patch("nf_core.utils.anaconda_package") + def test_modules_bump_versions_stale_containers_rebuild(self, mock_anaconda, mock_build): + """Check-2 integration: version is current but the conda locks are stale -> rebuild queued. + + Mirrors an earlier run that bumped environment.yml but was interrupted before the Wave + build finished, leaving conda locks pinned to the previous version. + """ + # bpipe/test pins bioconda::bpipe=0.9.13; report that as the latest so check-1 is a no-op + mock_anaconda.return_value = {"versions": ["0.9.13"], "latest_version": "0.9.13"} + + # Leave behind a stale conda lock (older version) for the module + lock_dir = Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", ".conda-lock") + lock_dir.mkdir(parents=True, exist_ok=True) + (lock_dir / "linux_amd64-bd-stale_1.txt").write_text( + "- conda: https://conda.anaconda.org/bioconda/noarch/bpipe-0.9.12-hdfd78af_0.conda\n" + ) + + version_bumper = nf_core.modules.bump_versions.ModuleVersionBumper(pipeline_dir=self.nfcore_modules) + version_bumper.bump_versions(module="bpipe/test") + + assert len(version_bumper.failed) == 0 + mock_build.assert_called_once() + assert [m.component_name for m in mock_build.call_args.args[0]] == ["bpipe/test"] + assert any("Containers out of date" in msg for msg, _ in version_bumper.updated) + + @mock.patch.object(nf_core.modules.bump_versions.ModuleVersionBumper, "_build_wave_containers") + @mock.patch("nf_core.utils.anaconda_package") + def test_modules_bump_versions_interrupted_build_rebuild(self, mock_anaconda, mock_build): + """Check-2 integration: version current, empty .conda-lock dir (interrupted build) -> rebuild.""" + mock_anaconda.return_value = {"versions": ["0.9.13"], "latest_version": "0.9.13"} + + # An interrupted build created the .conda-lock dir but wrote no locks + lock_dir = Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", ".conda-lock") + lock_dir.mkdir(parents=True, exist_ok=True) + + version_bumper = nf_core.modules.bump_versions.ModuleVersionBumper(pipeline_dir=self.nfcore_modules) + version_bumper.bump_versions(module="bpipe/test") + + assert len(version_bumper.failed) == 0 + mock_build.assert_called_once() + assert [m.component_name for m in mock_build.call_args.args[0]] == ["bpipe/test"] + + @mock.patch.object(nf_core.modules.bump_versions.ModuleVersionBumper, "_build_wave_containers") + @mock.patch("nf_core.utils.anaconda_package") + def test_modules_bump_versions_missing_meta_containers_rebuild(self, mock_anaconda, mock_build): + """Check-2 integration: locks current but meta.yml has no containers section -> rebuild. + + Mirrors a build interrupted after writing the conda locks but before meta.yml/main.nf. + """ + mock_anaconda.return_value = {"versions": ["0.9.13"], "latest_version": "0.9.13"} + + module_dir = Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test") + # Current-version lock present... + lock_dir = module_dir / ".conda-lock" + lock_dir.mkdir(parents=True, exist_ok=True) + (lock_dir / "linux_amd64-bd-current_1.txt").write_text( + "- conda: https://conda.anaconda.org/bioconda/noarch/bpipe-0.9.13-hdfd78af_0.conda\n" + ) + # ...but the containers section never got written (build interrupted before meta.yml) + meta_path = module_dir / "meta.yml" + yaml = ruamel.yaml.YAML() + with open(meta_path) as fh: + meta = yaml.load(fh) + meta.pop("containers", None) + with open(meta_path, "w") as fh: + yaml.dump(meta, fh) + + version_bumper = nf_core.modules.bump_versions.ModuleVersionBumper(pipeline_dir=self.nfcore_modules) + version_bumper.bump_versions(module="bpipe/test") + + assert len(version_bumper.failed) == 0 + mock_build.assert_called_once() + assert [m.component_name for m in mock_build.call_args.args[0]] == ["bpipe/test"] + + @mock.patch.object(nf_core.modules.bump_versions.ModuleVersionBumper, "_build_wave_containers") + @mock.patch("nf_core.utils.anaconda_package") + def test_modules_bump_versions_current_containers_no_rebuild(self, mock_anaconda, mock_build): + """Check-2 integration: version current, locks match AND meta has containers -> no rebuild.""" + mock_anaconda.return_value = {"versions": ["0.9.13"], "latest_version": "0.9.13"} + + module_dir = Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test") + # Conda lock already pins the current version + lock_dir = module_dir / ".conda-lock" + lock_dir.mkdir(parents=True, exist_ok=True) + (lock_dir / "linux_amd64-bd-current_1.txt").write_text( + "- conda: https://conda.anaconda.org/bioconda/noarch/bpipe-0.9.13-hdfd78af_0.conda\n" + ) + # ...and meta.yml carries a containers section, so the build is considered complete + meta_path = module_dir / "meta.yml" + yaml = ruamel.yaml.YAML() + with open(meta_path) as fh: + meta = yaml.load(fh) + meta["containers"] = {"docker": {"linux/amd64": {"name": "img"}}} + with open(meta_path, "w") as fh: + yaml.dump(meta, fh) + + version_bumper = nf_core.modules.bump_versions.ModuleVersionBumper(pipeline_dir=self.nfcore_modules) + version_bumper.bump_versions(module="bpipe/test") + + assert len(version_bumper.failed) == 0 + mock_build.assert_not_called() + assert any("up to date" in msg for msg, _ in version_bumper.up_to_date) + + @mock.patch("nf_core.utils.anaconda_package") + def test_modules_bump_versions_multi_package(self, mock_anaconda): + """Multi-package ('mulled') modules are now bumped, not skipped, and YAML comments survive.""" + mock_anaconda.return_value = {"versions": ["1.21", "1.23.1"], "latest_version": "1.23.1"} + + mod_dir = Path(tempfile.mkdtemp()) + env_yml = mod_dir / "environment.yml" + env_yml.write_text( + "channels:\n" + " - conda-forge\n" + " - bioconda\n" + "dependencies:\n" + " # renovate: datasource=conda depName=bioconda/htslib\n" + " - bioconda::htslib=1.21\n" + " # renovate: datasource=conda depName=bioconda/samtools\n" + " - bioconda::samtools=1.21\n" + ) + + # Bypass __init__ - bump_module_version only needs the result lists, tools_config and directory + bumper = nf_core.modules.bump_versions.ModuleVersionBumper.__new__( + nf_core.modules.bump_versions.ModuleVersionBumper + ) + bumper.failed = [] + bumper.ignored = [] + bumper.updated = [] + bumper.up_to_date = [] + bumper.directory = mod_dir + bumper.tools_config = SimpleNamespace() + module = SimpleNamespace( + component_name="samtools/cat", + component_dir=mod_dir, + environment_yml=env_yml, + meta_yml=None, + ) + + assert bumper.bump_module_version(module) is True + assert bumper.failed == [] + assert len(bumper.updated) == 2 + + new_content = env_yml.read_text() + # Both bioconda packages were bumped... + assert "bioconda::htslib=1.23.1" in new_content + assert "bioconda::samtools=1.23.1" in new_content + # ...and the renovate comments were preserved (no yaml round-trip) + assert "# renovate: datasource=conda depName=bioconda/htslib" in new_content + assert "# renovate: datasource=conda depName=bioconda/samtools" in new_content + def test_modules_bump_versions_fail(self): """Fail updating a module with wrong name""" version_bumper = nf_core.modules.bump_versions.ModuleVersionBumper(pipeline_dir=self.nfcore_modules) diff --git a/tests/modules/test_containers.py b/tests/modules/test_containers.py index 8b137a5196..b48b75b018 100644 --- a/tests/modules/test_containers.py +++ b/tests/modules/test_containers.py @@ -19,6 +19,10 @@ def setUp(self): super().setUp() self.environment_yml = self.bpipe_test_module_path / "environment.yml" self.module_containers = ModuleContainers("bpipe/test", directory=self.nfcore_modules) + # Avoid a build/lock cached by one test being reused by another (all fixtures share + # the same environment.yml content / build_id). + ModuleContainers._build_cache.clear() + ModuleContainers._conda_lock_paths.clear() def _write_meta(self, meta: dict) -> None: (self.bpipe_test_module_path / "meta.yml").write_text(yaml.safe_dump(meta), encoding="utf-8") @@ -377,33 +381,75 @@ def _create_local_module(self, module_name: str = "testmodule") -> Path: def test_init_pipeline_sets_local_module_paths(self): """ModuleContainers should resolve paths into modules/local/ for a pipeline repo""" module_dir = self._create_local_module("testmodule") - manager = ModuleContainers("testmodule", directory=self.pipeline_dir) + module_containers = ModuleContainers("testmodule", directory=self.pipeline_dir) - assert manager.repo_type == "pipeline" - assert manager.module_directory == module_dir - assert manager.environment_yml == module_dir / "environment.yml" - assert manager.meta_yml == module_dir / "meta.yml" + assert module_containers.repo_type == "pipeline" + assert module_containers.module_directory == module_dir + assert module_containers.environment_yml == module_dir / "environment.yml" + assert module_containers.meta_yml == module_dir / "meta.yml" def test_update_containers_in_meta_pipeline(self): """update_containers_in_meta writes containers to the local module's meta.yml""" module_dir = self._create_local_module("testmodule") - manager = ModuleContainers("testmodule", directory=self.pipeline_dir) + module_containers = ModuleContainers("testmodule", directory=self.pipeline_dir) containers = MetaYmlContainers( docker={p: ContainerEntry(name=f"docker-{p}") for p in CONTAINER_PLATFORMS}, singularity={p: ContainerEntry(name=f"sif-{p}") for p in CONTAINER_PLATFORMS}, conda={p: CondaEntry(lock_file=f"/lock/{p}.txt") for p in CONTAINER_PLATFORMS}, ) - manager.containers = containers + module_containers.containers = containers - with mock.patch.object(manager, "create") as mock_create: - manager.update_containers_in_meta() + with mock.patch.object(module_containers, "create") as mock_create: + module_containers.update_containers_in_meta() mock_create.assert_not_called() meta = yaml.safe_load((module_dir / "meta.yml").read_text(encoding="utf-8")) assert meta["containers"] == containers.dump_for_meta_yml() +class TestBuildContainersWithProgress(TestModules): + """Tests for the ``ModuleContainers.build_containers_with_progress`` method. + + ``create`` is mocked throughout so the tests cover the batch/aggregation logic + regardless of how containers are actually built (Wave API vs CLI). + """ + + def setUp(self): + super().setUp() + self.module_containers = ModuleContainers("bpipe/test", directory=self.nfcore_modules) + self.component = self.module_containers.nfcore_component + self.module_name = self.component.component_name + + def test_success_returns_no_failures(self): + with mock.patch.object(ModuleContainers, "create", return_value=(MetaYmlContainers(), True)) as mock_create: + failed = self.module_containers.build_containers_with_progress(hide_progress=True) + assert failed == [] + mock_create.assert_called_once() + + def test_build_failure_is_reported(self): + """A module whose create() returns success=False is reported as failed.""" + with mock.patch.object(ModuleContainers, "create", return_value=(MetaYmlContainers(), False)): + failed = self.module_containers.build_containers_with_progress(hide_progress=True) + assert failed == [self.module_name] + + def test_exception_during_build_is_caught(self): + """A RuntimeError from create() is caught and the module is reported as failed.""" + with mock.patch.object(ModuleContainers, "create", side_effect=RuntimeError("boom")): + failed = self.module_containers.build_containers_with_progress(hide_progress=True) + assert failed == [self.module_name] + + def test_all_modules_reconstructs_per_module_containers(self): + """In batch mode a fresh per-module instance is built, reusing the scanned component list.""" + all_module_containers = ModuleContainers( + module=None, directory=self.nfcore_modules, all_modules=True, components=[self.component] + ) + with mock.patch.object(ModuleContainers, "create", return_value=(MetaYmlContainers(), True)) as mock_create: + failed = all_module_containers.build_containers_with_progress(hide_progress=True) + assert failed == [] + mock_create.assert_called_once() + + class TestModuleContainersDockerfile(TestModuleContainersPipeline): """Tests for ModuleContainers behaviour when a module uses a Dockerfile instead of environment.yml"""