diff --git a/nf_core/commands_modules.py b/nf_core/commands_modules.py index c2717b4852..7c2c114894 100644 --- a/nf_core/commands_modules.py +++ b/nf_core/commands_modules.py @@ -4,7 +4,6 @@ import rich -from nf_core.pipelines.containers_utils import try_generate_container_configs from nf_core.utils import CONTAINER_PLATFORMS, rich_force_colors log = logging.getLogger(__name__) @@ -341,12 +340,7 @@ def modules_containers_create(ctx, module: str, directory: Path, force: bool) -> """ Build docker and singularity containers for linux/arm64 and linux/amd64 using wave. """ - from rich.console import Group - from rich.live import Live - from rich.progress import BarColumn, MofNCompleteColumn, Progress, SpinnerColumn, TextColumn - - from nf_core.modules.containers import ModuleContainers - from nf_core.pipelines.lint_utils import console + from nf_core.modules.containers import ModuleContainers, build_containers_with_progress try: manager = ModuleContainers(module=module, directory=directory, verbose=ctx.obj["verbose"]) @@ -364,60 +358,15 @@ def modules_containers_create(ctx, module: str, directory: Path, force: bool) -> assert manager.nfcore_component is not None components = [manager.nfcore_component] - failed_modules = [] - - overall_progress = Progress( - "[bold blue]{task.description}", - BarColumn(), - MofNCompleteColumn(), - console=console, - disable=ctx.obj["hide_progress"], - ) - module_progress = Progress( - SpinnerColumn(finished_text="[green]✓[/green]"), - "[bold blue]{task.description}", - TextColumn("{task.fields[status]}"), - console=console, - disable=ctx.obj["hide_progress"], + failed_modules = build_containers_with_progress( + manager, + components, + directory, + force=force, + hide_progress=ctx.obj["hide_progress"], + verbose=ctx.obj["verbose"], ) - with Live(Group(overall_progress, module_progress), console=console, transient=True): - # The overall bar is only useful when processing more than one module - overall_task_id = overall_progress.add_task("modules", total=len(components), visible=len(components) > 1) - for component in components: - module_name = component.component_name - module_task_id = module_progress.add_task( - f"[cyan]{module_name}[/cyan]", - total=None, - status="building containers...", - ) - try: - if manager.all_modules: - # Per-module instance, reusing the already-scanned component list - module_manager = ModuleContainers( - module=module_name, - directory=directory, - verbose=ctx.obj["verbose"], - components=manager.available_modules, - ) - else: - module_manager = manager - _, success = module_manager.create( - progress_bar=module_progress, task_id=module_task_id, force=force - ) - if success: - module_manager.update_containers_in_meta() - if module_manager.repo_type == "pipeline": - try_generate_container_configs(directory, module_manager.module_directory) - else: - failed_modules.append(module_name) - except (ValueError, RuntimeError, OSError) as e: - log.error(f"✗ Failed to build containers for {module_name}: {e}") - failed_modules.append(module_name) - finally: - overall_progress.advance(overall_task_id) - module_progress.remove_task(module_task_id) - if failed_modules: if manager.all_modules: log.warning( diff --git a/nf_core/components/create.py b/nf_core/components/create.py index 78e1c5f7d7..2efefe55ac 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -177,8 +177,37 @@ def create(self) -> bool: run_prettier_on_file(new_files) log.info("Created following files:\n " + "\n ".join(new_files)) + + if self.component_type == "modules": + self._build_seqera_containers() + return True + def _build_seqera_containers(self) -> None: + """ + Build Seqera/Wave containers for the freshly created module. + """ + from nf_core.modules.containers import ModuleContainers, build_containers_with_progress + + log.info("Building Seqera containers for the module with Wave...") + try: + manager = ModuleContainers( + module=self.component_name, + directory=self.directory, + ) + ModuleContainers.check_tower_token() + if manager.nfcore_component is None: + log.warning("Could not locate the new module to build Seqera containers.") + return + failed = build_containers_with_progress(manager, [manager.nfcore_component], self.directory) + if failed: + log.warning( + "Could not build all Seqera containers for the module. " + "Run `nf-core modules containers create` to retry." + ) + except (ValueError, RuntimeError, OSError) as e: + log.warning(f"Could not build Seqera containers for the module: {e}") + def _get_bioconda_tool(self): """ Try to find a bioconda package for 'tool' @@ -226,22 +255,6 @@ def _get_bioconda_tool(self): ) break - # Try to get the container tag (only if bioconda package was found) - if self.bioconda: - try: - if self.tool_conda_name: - self.docker_container, self.singularity_container = nf_core.utils.get_biocontainer_tag( - self.tool_conda_name, version - ) - else: - self.docker_container, self.singularity_container = nf_core.utils.get_biocontainer_tag( - self.component, version - ) - log.info(f"Using Docker container: '{self.docker_container}'") - log.info(f"Using Singularity container: '{self.singularity_container}'") - except (ValueError, LookupError) as e: - log.info(f"Could not find a Docker/Singularity container ({e})") - def _get_module_structure_components(self): process_label_defaults = [ "process_single", diff --git a/nf_core/modules/bump_versions.py b/nf_core/modules/bump_versions.py index eb0e923324..d1d57e8a0e 100644 --- a/nf_core/modules/bump_versions.py +++ b/nf_core/modules/bump_versions.py @@ -21,7 +21,7 @@ import nf_core.utils from nf_core.components.components_command import ComponentCommand from nf_core.components.nfcore_component import NFCoreComponent -from nf_core.utils import NFCoreYamlConfig, custom_yaml_dumper, rich_force_colors +from nf_core.utils import NFCoreYamlConfig, rich_force_colors from nf_core.utils import plural_s as _s log = logging.getLogger(__name__) @@ -56,10 +56,10 @@ def bump_versions( If module is the name of a directory in the modules directory, all modules in that directory will be bumped. - Looks for a bioconda tool version in the `main.nf` file of the module and checks whether - are more recent version is available. If yes, then tries to get docker/singularity - container links and replace the bioconda version and the container links in the main.nf file - of the respective module. + Looks for a bioconda tool version in the `environment.yml` file of the module and checks + whether a more recent version is available. If yes, the pinned version + in `environment.yml` is bumped and the Docker/Singularity containers are rebuilt from it + with Wave. Args: module: a specific module to update @@ -123,6 +123,7 @@ def bump_versions( transient=True, disable=os.environ.get("HIDE_PROGRESS", None) is not None, ) + modules_to_rebuild: list[NFCoreComponent] = [] with progress_bar: bump_progress = progress_bar.add_task( "Bumping nf-core modules versions", @@ -131,39 +132,70 @@ def bump_versions( ) for mod in nfcore_modules: progress_bar.update(bump_progress, advance=1, test_name=mod.component_name) - self.bump_module_version(mod) + if self.bump_module_version(mod): + modules_to_rebuild.append(mod) + + if modules_to_rebuild: + self._build_wave_containers(modules_to_rebuild) self._print_results() return nfcore_modules + def _build_wave_containers(self, modules: list[NFCoreComponent]) -> None: + """Rebuild the Docker/Singularity containers for ``modules`` with Seqera Wave. + + Builds from each module's (already bumped) environment.yml and updates main.nf, + meta.yml and the conda lock files. Modules whose build fails are added to + ``self.failed``. + """ + from nf_core.modules.containers import ModuleContainers, build_containers_with_progress + + log.info(f"Building Seqera containers for {len(modules)} bumped module{_s(modules)} with Wave...") + manager = ModuleContainers( + module=None, + directory=self.directory, + all_modules=True, + components=modules, + ) + ModuleContainers.check_tower_token() + failed_modules = build_containers_with_progress(manager, modules, self.directory) + for module_name in failed_modules: + self.failed.append(("Container build with Wave failed", module_name)) + def bump_module_version(self, module: NFCoreComponent) -> bool: """ Bump the bioconda and container version of a single NFCoreComponent Args: module: NFCoreComponent + + Returns: + True if the module's ``environment.yml`` was bumped and its containers therefore + need rebuilding; False if it was already up to date, ignored or failed. """ config_version = None - bioconda_packages = [] - try: - # Extract bioconda version from `environment.yml` - bioconda_packages = self.get_bioconda_version(module) - except FileNotFoundError: - # try it in the main.nf instead - try: - with open(module.main_nf) as fh: - for line in fh: - if "bioconda::" in line: - bioconda_packages = [b for b in line.split() if "bioconda::" in b] - except FileNotFoundError: - log.error( - f"Neither `environment.yml` nor `main.nf` of {module.component_name} module could be read to get bioconada version of used tools." - ) - - # If multiple versions - don't update! (can't update mulled containers) - if not bioconda_packages or len(bioconda_packages) > 1: - self.failed.append(("Ignoring mulled container", module.component_name)) + # Extract the pinned conda dependencies from `environment.yml`. Wave rebuilds the containers + # straight from this file, so every pinned package can be bumped - including multi-package + # ("mulled") modules, which earlier versions had to skip because the old mulled container + # names could not be regenerated. + dependencies = self.get_bioconda_version(module) + + # Only channel-pinned, versioned deps (e.g. `bioconda::samtools=1.21`) can be bumped; skip + # pip sub-dicts, unpinned deps and anything without a `channel::name=version` shape. + conda_packages: list[tuple[str, str, str]] = [] # (channel, tool_name, current_version) + for dep in dependencies: + if not isinstance(dep, str): + continue + spec = dep.strip("'").strip('"') + if "::" not in spec or "=" not in spec: + continue + channel, name_version = spec.split("::", 1) + tool_name, current_version = name_version.split("=", 1) + conda_packages.append((channel, tool_name, current_version)) + + if not conda_packages: + self.failed.append(("No pinned conda dependencies to bump", module.component_name)) return False # Don't update if blocked in blacklist @@ -173,107 +205,128 @@ def bump_module_version(self, module: NFCoreComponent) -> bool: if not config_version: self.ignored.append(("Omitting module due to config.", module.component_name)) return False - - # check for correct version and newer versions - bioconda_tool_name = bioconda_packages[0].split("=")[0].replace("bioconda::", "").strip("'").strip('"') - bp = bioconda_packages[0] - bp = bp.strip("'").strip('"') - bioconda_version = bp.split("=")[1] - - if not config_version: - try: - response = nf_core.utils.anaconda_package(bp) - except (LookupError, ValueError): + if len(conda_packages) > 1: + # A single pinned version in the config can't be mapped onto a multi-package module self.failed.append( - ( - f"Conda version not specified correctly: {Path(module.main_nf).relative_to(self.directory)}", - module.component_name, - ) + ("Cannot pin a version via config for a multi-package module", module.component_name) ) return False - # Check that required version is available at all - if bioconda_version not in response.get("versions"): - self.failed.append((f"Conda package had unknown version: `{module.main_nf}`", module.component_name)) - return False - - # Check version is latest available - last_ver = response.get("latest_version") - else: - last_ver = config_version - - if last_ver is not None and last_ver != bioconda_version: - log.debug(f"Updating version for {module.component_name}") - # Get docker and singularity container links - try: - docker_img, singularity_img = nf_core.utils.get_biocontainer_tag(bioconda_tool_name, last_ver) - except LookupError as e: - self.failed.append((f"Could not download container tags: {e}", module.component_name)) - return False - - patterns = [ - (rf"biocontainers/{bioconda_tool_name}:[^'\"\s]+", docker_img), - ( - rf"https://depot.galaxyproject.org/singularity/{bioconda_tool_name}:[^'\"\s]+", - singularity_img, - ), - ] - - with open(module.main_nf) as fh: - content = fh.read() + # Resolve the target version for every package: the latest available on Anaconda, or the + # version pinned in the config blacklist. + targets: list[tuple[str, str, str, str]] = [] # (channel, tool_name, current_version, target_version) + for channel, tool_name, current_version in conda_packages: + if config_version: + target_version = config_version + else: + dep_spec = f"{channel}::{tool_name}={current_version}" + try: + response = nf_core.utils.anaconda_package(dep_spec) + except (LookupError, ValueError): + self.failed.append( + ( + f"Conda version not specified correctly: {Path(module.environment_yml).relative_to(self.directory) if module.environment_yml else module.component_name}", + module.component_name, + ) + ) + return False - # Go over file content of main.nf and find replacements - for pattern in patterns: - found_match = False - newcontent = [] - for line in content.splitlines(): - # Match the pattern - matches_pattern = re.findall(rf"^.*{pattern[0]}.*$", line) - if matches_pattern: - found_match = True - - # Replace the match - newline = re.sub(pattern[0], pattern[1], line) - newcontent.append(newline) - # No match, keep line as it is - else: - newcontent.append(line) - - if found_match: - content = "\n".join(newcontent) + "\n" - else: + # Check that the pinned version is available at all + if current_version not in response.get("versions"): self.failed.append( - (f"Did not find pattern {pattern[0]} in module {module.component_name}", module.component_name) + (f"Conda package had unknown version: `{module.environment_yml}`", module.component_name) ) return False - # Write new content to the file - with open(module.main_nf, "w") as fh: - fh.write(content) + target_version = response.get("latest_version") - # change version in environment.yml + if target_version is None: + continue + targets.append((channel, tool_name, current_version, target_version)) + + if not targets: + self.up_to_date.append((f"Module version up to date: {module.component_name}", module.component_name)) + return False + + # Check 1: bump any dependency behind the latest. Wave rebuilds the containers from + # environment.yml afterwards (see bump_versions()), updating main.nf, meta.yml and conda locks. + outdated = [t for t in targets if t[3] != t[2]] + if outdated: if not module.environment_yml: log.error(f"Could not read `environment.yml` of {module.component_name} module.") + self.failed.append(("No environment.yml found to update", module.component_name)) return False + # Rewrite the versions in the raw file text so YAML comments (e.g. renovate hints) and + # formatting survive - a yaml load/dump round-trip would silently drop them. with open(module.environment_yml) as fh: - env_yml = yaml.safe_load(fh) - env_yml["dependencies"][0] = re.sub( - bioconda_packages[0], f"bioconda::{bioconda_tool_name}={last_ver}", env_yml["dependencies"][0] - ) + content = fh.read() + for channel, tool_name, current_version, target_version in outdated: + log.debug(f"Updating {tool_name} version for {module.component_name}") + content = re.sub( + rf"({re.escape(channel)}::{re.escape(tool_name)})=[^\s'\"]+", + rf"\g<1>={target_version}", + content, + ) + self.updated.append( + (f"Module updated: {tool_name} {current_version} --> {target_version}", module.component_name) + ) with open(module.environment_yml, "w") as fh: - yaml.dump(env_yml, fh, default_flow_style=False, Dumper=custom_yaml_dumper()) + fh.write(content) + return True - self.updated.append( - ( - f"Module updated: {bioconda_version} --> {last_ver}", - module.component_name, + # Check 2: environment.yml is already current, but make sure the containers were actually + # built for every package. An earlier run may have bumped environment.yml and then been + # interrupted before the Wave build wrote the new containers / conda locks. + for _channel, tool_name, _current_version, target_version in targets: + if not self._containers_built_for_version(module, tool_name, target_version): + self.updated.append( + (f"Containers out of date - rebuilding for version {target_version}", module.component_name) ) + return True + + self.up_to_date.append((f"Module version up to date: {module.component_name}", module.component_name)) + return False + + def _containers_built_for_version(self, module: NFCoreComponent, tool_name: str, version: str) -> bool: + """Return True if the module's Wave conda-lock files already pin ``tool_name`` at ``version``. + + Image names are content hashes, so the version-bearing conda locks are checked instead, + assuming the bioconda dependency name matches the conda package name. No ``.conda-lock`` dir + means the module was never Wave-built (left alone); an empty dir, or a meta.yml without a + ``containers:`` section, means an interrupted build (rebuilt). + """ + from nf_core.components.components_utils import read_meta_yml + + conda_lock_dir = module.component_dir / ".conda-lock" + if not conda_lock_dir.is_dir(): + log.debug( + f"No conda-lock directory for {module.component_name}; treating as up to date " + "(Wave migration is out of scope for bump-versions)" ) return True + lock_files = list(conda_lock_dir.glob("*.txt")) + if not lock_files: + # Empty dir: a finished build always leaves locks, so this one was interrupted. + log.debug(f"Empty conda-lock directory for {module.component_name}; containers need rebuilding") + return False + # Lock entries look like `.../samtools-1.21-h50ea8bc_0.conda`; trailing `-` keeps 1.21 from + # matching 1.210. Stream each file so the search short-circuits. + needle = f"/{tool_name}-{version}-" + for lock_file in lock_files: + with lock_file.open() as fh: + if not any(needle in line for line in fh): + return False - else: - self.up_to_date.append((f"Module version up to date: {module.component_name}", module.component_name)) - return True + # The containers section is written last, so its absence means an interrupted build. Only + # require it to be present, not complete (arm64 builds may legitimately be missing). + if module.meta_yml is None or not Path(module.meta_yml).exists(): + log.debug(f"No meta.yml for {module.component_name}; containers need rebuilding") + return False + if not read_meta_yml(Path(module.meta_yml)).get("containers"): + log.debug(f"meta.yml for {module.component_name} has no containers section; containers need rebuilding") + return False + + return True def get_bioconda_version(self, module: NFCoreComponent) -> list[str]: """ diff --git a/nf_core/modules/containers.py b/nf_core/modules/containers.py index bf7c1fee6d..5a1649ad3e 100644 --- a/nf_core/modules/containers.py +++ b/nf_core/modules/containers.py @@ -34,6 +34,98 @@ WAVE_URL = "https://wave.seqera.io" +def build_containers_with_progress( + manager: "ModuleContainers", + components: list[NFCoreComponent], + directory: Path, + force: bool = False, + hide_progress: bool = False, + verbose: bool = False, +) -> list[str]: + """Build Seqera/Wave containers for ``components`` with a live progress display. + + Shows an overall module bar plus, per module, the per-build spinners + (docker/singularity × platform) that :meth:`ModuleContainers.create` renders when a + progress bar is passed. + + Args: + manager: A :class:`ModuleContainers` covering the modules to build. + components: The components to build containers for. + directory: Base directory of the pipeline / modules repo. + force: Overwrite existing container directives even if already correct. + hide_progress: Disable the live progress display. + verbose: Verbose logging for per-module managers in batch mode. + + Returns: + The names of the modules whose builds failed. + """ + from rich.console import Group + from rich.live import Live + from rich.progress import BarColumn, MofNCompleteColumn, Progress, SpinnerColumn, TextColumn + + from nf_core.pipelines.containers_utils import try_generate_container_configs + from nf_core.pipelines.lint_utils import console + + failed_modules: list[str] = [] + + overall_progress = Progress( + "[bold blue]{task.description}", + BarColumn(), + MofNCompleteColumn(), + console=console, + disable=hide_progress, + ) + module_progress = Progress( + SpinnerColumn(finished_text="[green]✓[/green]"), + "[bold blue]{task.description}", + TextColumn("{task.fields[status]}"), + console=console, + disable=hide_progress, + ) + + # Batch builds defer config generation to one full scan after the loop. + batch = len(components) > 1 + + with Live(Group(overall_progress, module_progress), console=console, transient=True): + # The overall bar is only useful when processing more than one module + overall_task_id = overall_progress.add_task("modules", total=len(components), visible=len(components) > 1) + for component in components: + module_name = component.component_name + module_task_id = module_progress.add_task( + f"[cyan]{module_name}[/cyan]", + total=None, + status="building containers...", + ) + try: + if manager.all_modules: + # Per-module instance, reusing the already-scanned component list + module_manager = ModuleContainers( + module=module_name, + directory=directory, + verbose=verbose, + components=manager.available_modules, + ) + else: + module_manager = manager + _, success = module_manager.create(progress_bar=module_progress, task_id=module_task_id, force=force) + if success: + if module_manager.repo_type == "pipeline" and not batch: + try_generate_container_configs(directory, module_manager.module_directory) + else: + failed_modules.append(module_name) + except (ValueError, RuntimeError, OSError) as e: + log.error(f"✗ Failed to build containers for {module_name}: {e}") + failed_modules.append(module_name) + finally: + overall_progress.advance(overall_task_id) + module_progress.remove_task(module_task_id) + + if batch and manager.repo_type == "pipeline" and len(failed_modules) < len(components): + try_generate_container_configs(directory) + + return failed_modules + + class ModuleContainers: """ Helpers for building, linting and listing module containers. diff --git a/tests/modules/test_bump_versions.py b/tests/modules/test_bump_versions.py index 63e6eb9c2c..c5eb5b325c 100644 --- a/tests/modules/test_bump_versions.py +++ b/tests/modules/test_bump_versions.py @@ -1,6 +1,8 @@ import re import tempfile from pathlib import Path +from types import SimpleNamespace +from unittest import mock import pytest import ruamel.yaml @@ -14,7 +16,8 @@ class TestModulesBumpVersions(TestModules): - def test_modules_bump_versions_single_module(self): + @mock.patch.object(nf_core.modules.bump_versions.ModuleVersionBumper, "_build_wave_containers") + def test_modules_bump_versions_single_module(self, mock_build): """Test updating a single module""" # Change the bpipe/test version to an older version env_yml_path = Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "environment.yml") @@ -24,11 +27,13 @@ def test_modules_bump_versions_single_module(self): with open(env_yml_path, "w") as fh: fh.write(new_content) version_bumper = nf_core.modules.bump_versions.ModuleVersionBumper(pipeline_dir=self.nfcore_modules) + # mock_build keeps the Wave container rebuild from shelling out to the `wave` CLI modules = version_bumper.bump_versions(module="bpipe/test") assert len(version_bumper.failed) == 0 assert [m.component_name for m in modules] == ["bpipe/test"] - def test_modules_bump_versions_all_modules(self): + @mock.patch.object(nf_core.modules.bump_versions.ModuleVersionBumper, "_build_wave_containers") + def test_modules_bump_versions_all_modules(self, mock_build): """Test updating all modules""" version_bumper = nf_core.modules.bump_versions.ModuleVersionBumper(pipeline_dir=self.nfcore_modules) modules = version_bumper.bump_versions(all_modules=True) @@ -87,6 +92,191 @@ def test_modules_bump_versions_submodules(self): out_modules = version_bumper.bump_versions(module="fgbio", dry_run=True) assert sorted([m.component_name for m in out_modules]) == sorted(in_modules) + def test_containers_built_for_version(self): + """Check-2: detect whether a module's conda locks already pin the current version.""" + # Bypass __init__ - the method only uses the passed module's component_dir + bumper = nf_core.modules.bump_versions.ModuleVersionBumper.__new__( + nf_core.modules.bump_versions.ModuleVersionBumper + ) + mod_dir = Path(tempfile.mkdtemp()) + meta_yml = mod_dir / "meta.yml" + module = SimpleNamespace(component_dir=mod_dir, component_name="samtools/sort", meta_yml=meta_yml) + + # No .conda-lock dir at all -> not Wave-managed, treated as up to date (no rebuild) + assert bumper._containers_built_for_version(module, "samtools", "1.21") is True + + lock_dir = mod_dir / ".conda-lock" + lock_dir.mkdir() + # Empty .conda-lock dir -> a Wave build was started but interrupted before writing locks + assert bumper._containers_built_for_version(module, "samtools", "1.21") is False + + (lock_dir / "linux_amd64-bd-abc_1.txt").write_text( + "- conda: https://conda.anaconda.org/bioconda/noarch/samtools-1.21-h50ea8bc_0.conda\n" + ) + # Lock current but meta.yml missing its containers section (build interrupted before the + # final meta.yml write) -> rebuild + meta_yml.write_text("name: samtools_sort\n") + assert bumper._containers_built_for_version(module, "samtools", "1.21") is False + + # Lock current AND meta.yml carries a containers section -> already built, no rebuild + meta_yml.write_text("name: samtools_sort\ncontainers:\n docker:\n linux/amd64:\n name: img\n") + assert bumper._containers_built_for_version(module, "samtools", "1.21") is True + # Trailing dash anchors the match so 1.21 does not also match 1.210 + assert bumper._containers_built_for_version(module, "samtools", "1.2") is False + # Lock pins an older version (e.g. interrupted build left stale locks) -> rebuild + assert bumper._containers_built_for_version(module, "samtools", "1.22") is False + + @mock.patch.object(nf_core.modules.bump_versions.ModuleVersionBumper, "_build_wave_containers") + @mock.patch("nf_core.utils.anaconda_package") + def test_modules_bump_versions_stale_containers_rebuild(self, mock_anaconda, mock_build): + """Check-2 integration: version is current but the conda locks are stale -> rebuild queued. + + Mirrors an earlier run that bumped environment.yml but was interrupted before the Wave + build finished, leaving conda locks pinned to the previous version. + """ + # bpipe/test pins bioconda::bpipe=0.9.13; report that as the latest so check-1 is a no-op + mock_anaconda.return_value = {"versions": ["0.9.13"], "latest_version": "0.9.13"} + + # Leave behind a stale conda lock (older version) for the module + lock_dir = Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", ".conda-lock") + lock_dir.mkdir(parents=True, exist_ok=True) + (lock_dir / "linux_amd64-bd-stale_1.txt").write_text( + "- conda: https://conda.anaconda.org/bioconda/noarch/bpipe-0.9.12-hdfd78af_0.conda\n" + ) + + version_bumper = nf_core.modules.bump_versions.ModuleVersionBumper(pipeline_dir=self.nfcore_modules) + version_bumper.bump_versions(module="bpipe/test") + + assert len(version_bumper.failed) == 0 + mock_build.assert_called_once() + assert [m.component_name for m in mock_build.call_args.args[0]] == ["bpipe/test"] + assert any("Containers out of date" in msg for msg, _ in version_bumper.updated) + + @mock.patch.object(nf_core.modules.bump_versions.ModuleVersionBumper, "_build_wave_containers") + @mock.patch("nf_core.utils.anaconda_package") + def test_modules_bump_versions_interrupted_build_rebuild(self, mock_anaconda, mock_build): + """Check-2 integration: version current, empty .conda-lock dir (interrupted build) -> rebuild.""" + mock_anaconda.return_value = {"versions": ["0.9.13"], "latest_version": "0.9.13"} + + # An interrupted build created the .conda-lock dir but wrote no locks + lock_dir = Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", ".conda-lock") + lock_dir.mkdir(parents=True, exist_ok=True) + + version_bumper = nf_core.modules.bump_versions.ModuleVersionBumper(pipeline_dir=self.nfcore_modules) + version_bumper.bump_versions(module="bpipe/test") + + assert len(version_bumper.failed) == 0 + mock_build.assert_called_once() + assert [m.component_name for m in mock_build.call_args.args[0]] == ["bpipe/test"] + + @mock.patch.object(nf_core.modules.bump_versions.ModuleVersionBumper, "_build_wave_containers") + @mock.patch("nf_core.utils.anaconda_package") + def test_modules_bump_versions_missing_meta_containers_rebuild(self, mock_anaconda, mock_build): + """Check-2 integration: locks current but meta.yml has no containers section -> rebuild. + + Mirrors a build interrupted after writing the conda locks but before meta.yml/main.nf. + """ + mock_anaconda.return_value = {"versions": ["0.9.13"], "latest_version": "0.9.13"} + + module_dir = Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test") + # Current-version lock present... + lock_dir = module_dir / ".conda-lock" + lock_dir.mkdir(parents=True, exist_ok=True) + (lock_dir / "linux_amd64-bd-current_1.txt").write_text( + "- conda: https://conda.anaconda.org/bioconda/noarch/bpipe-0.9.13-hdfd78af_0.conda\n" + ) + # ...but the containers section never got written (build interrupted before meta.yml) + meta_path = module_dir / "meta.yml" + yaml = ruamel.yaml.YAML() + with open(meta_path) as fh: + meta = yaml.load(fh) + meta.pop("containers", None) + with open(meta_path, "w") as fh: + yaml.dump(meta, fh) + + version_bumper = nf_core.modules.bump_versions.ModuleVersionBumper(pipeline_dir=self.nfcore_modules) + version_bumper.bump_versions(module="bpipe/test") + + assert len(version_bumper.failed) == 0 + mock_build.assert_called_once() + assert [m.component_name for m in mock_build.call_args.args[0]] == ["bpipe/test"] + + @mock.patch.object(nf_core.modules.bump_versions.ModuleVersionBumper, "_build_wave_containers") + @mock.patch("nf_core.utils.anaconda_package") + def test_modules_bump_versions_current_containers_no_rebuild(self, mock_anaconda, mock_build): + """Check-2 integration: version current, locks match AND meta has containers -> no rebuild.""" + mock_anaconda.return_value = {"versions": ["0.9.13"], "latest_version": "0.9.13"} + + module_dir = Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test") + # Conda lock already pins the current version + lock_dir = module_dir / ".conda-lock" + lock_dir.mkdir(parents=True, exist_ok=True) + (lock_dir / "linux_amd64-bd-current_1.txt").write_text( + "- conda: https://conda.anaconda.org/bioconda/noarch/bpipe-0.9.13-hdfd78af_0.conda\n" + ) + # ...and meta.yml carries a containers section, so the build is considered complete + meta_path = module_dir / "meta.yml" + yaml = ruamel.yaml.YAML() + with open(meta_path) as fh: + meta = yaml.load(fh) + meta["containers"] = {"docker": {"linux/amd64": {"name": "img"}}} + with open(meta_path, "w") as fh: + yaml.dump(meta, fh) + + version_bumper = nf_core.modules.bump_versions.ModuleVersionBumper(pipeline_dir=self.nfcore_modules) + version_bumper.bump_versions(module="bpipe/test") + + assert len(version_bumper.failed) == 0 + mock_build.assert_not_called() + assert any("up to date" in msg for msg, _ in version_bumper.up_to_date) + + @mock.patch("nf_core.utils.anaconda_package") + def test_modules_bump_versions_multi_package(self, mock_anaconda): + """Multi-package ('mulled') modules are now bumped, not skipped, and YAML comments survive.""" + mock_anaconda.return_value = {"versions": ["1.21", "1.23.1"], "latest_version": "1.23.1"} + + mod_dir = Path(tempfile.mkdtemp()) + env_yml = mod_dir / "environment.yml" + env_yml.write_text( + "channels:\n" + " - conda-forge\n" + " - bioconda\n" + "dependencies:\n" + " # renovate: datasource=conda depName=bioconda/htslib\n" + " - bioconda::htslib=1.21\n" + " # renovate: datasource=conda depName=bioconda/samtools\n" + " - bioconda::samtools=1.21\n" + ) + + # Bypass __init__ - bump_module_version only needs the result lists, tools_config and directory + bumper = nf_core.modules.bump_versions.ModuleVersionBumper.__new__( + nf_core.modules.bump_versions.ModuleVersionBumper + ) + bumper.failed = [] + bumper.ignored = [] + bumper.updated = [] + bumper.up_to_date = [] + bumper.directory = mod_dir + bumper.tools_config = SimpleNamespace() + module = SimpleNamespace( + component_name="samtools/cat", + component_dir=mod_dir, + environment_yml=env_yml, + meta_yml=None, + ) + + assert bumper.bump_module_version(module) is True + assert bumper.failed == [] + assert len(bumper.updated) == 2 + + new_content = env_yml.read_text() + # Both bioconda packages were bumped... + assert "bioconda::htslib=1.23.1" in new_content + assert "bioconda::samtools=1.23.1" in new_content + # ...and the renovate comments were preserved (no yaml round-trip) + assert "# renovate: datasource=conda depName=bioconda/htslib" in new_content + assert "# renovate: datasource=conda depName=bioconda/samtools" in new_content + def test_modules_bump_versions_fail(self): """Fail updating a module with wrong name""" version_bumper = nf_core.modules.bump_versions.ModuleVersionBumper(pipeline_dir=self.nfcore_modules) diff --git a/tests/modules/test_create.py b/tests/modules/test_create.py index 88789b51f2..a0a09250f3 100644 --- a/tests/modules/test_create.py +++ b/tests/modules/test_create.py @@ -7,9 +7,12 @@ import yaml import nf_core.modules.create +from nf_core.components.create import ComponentCreate +from nf_core.modules.containers import ModuleContainers +from nf_core.modules.modules_utils import ContainerEntry +from nf_core.utils import CONTAINER_PLATFORMS, CONTAINER_SYSTEMS from tests.utils import ( mock_anaconda_api_calls, - mock_biocontainers_api_calls, mock_biotools_api_calls, ) @@ -17,11 +20,16 @@ class TestModulesCreate(TestModules): + def setUp(self): + super().setUp() + patcher = mock.patch.object(ComponentCreate, "_build_seqera_containers") + patcher.start() + self.addCleanup(patcher.stop) + def test_modules_create_succeed(self): """Succeed at creating the TrimGalore! module""" with responses.RequestsMock() as rsps: mock_anaconda_api_calls(rsps, "trim-galore", "0.6.7") - mock_biocontainers_api_calls(rsps, "trim-galore", "0.6.7") module_create = nf_core.modules.create.ModuleCreate( self.pipeline_dir, "trimgalore", "@author", "process_single", True, True, conda_name="trim-galore" ) @@ -33,7 +41,6 @@ def test_modules_create_fail_exists(self): """Fail at creating the same module twice""" with responses.RequestsMock() as rsps: mock_anaconda_api_calls(rsps, "trim-galore", "0.6.7") - mock_biocontainers_api_calls(rsps, "trim-galore", "0.6.7") module_create = nf_core.modules.create.ModuleCreate( self.pipeline_dir, "trimgalore", "@author", "process_single", False, False, conda_name="trim-galore" ) @@ -47,7 +54,6 @@ def test_modules_create_nfcore_modules(self): """Create a module in nf-core/modules clone""" with responses.RequestsMock() as rsps: mock_anaconda_api_calls(rsps, "fastqc", "0.11.9") - mock_biocontainers_api_calls(rsps, "fastqc", "0.11.9") module_create = nf_core.modules.create.ModuleCreate( self.nfcore_modules, "fastqc", "@author", "process_low", False, False ) @@ -60,7 +66,6 @@ def test_modules_create_nfcore_modules_subtool(self): """Create a tool/subtool module in a nf-core/modules clone""" with responses.RequestsMock() as rsps: mock_anaconda_api_calls(rsps, "star", "2.8.10a") - mock_biocontainers_api_calls(rsps, "star", "2.8.10a") module_create = nf_core.modules.create.ModuleCreate( self.nfcore_modules, "star/index", "@author", "process_medium", False, False ) @@ -73,7 +78,6 @@ def test_modules_meta_yml_structure_biotools_meta(self): """Test the structure of the module meta.yml file when it was generated with INFORMATION from bio.tools and WITH a meta.""" with responses.RequestsMock() as rsps: mock_anaconda_api_calls(rsps, "bpipe", "0.9.13--hdfd78af_0") - mock_biocontainers_api_calls(rsps, "bpipe", "0.9.13--hdfd78af_0") mock_biotools_api_calls(rsps, "bpipe") module_create = nf_core.modules.create.ModuleCreate( self.nfcore_modules, "bpipe/test", "@author", "process_single", has_meta=True, force=True @@ -181,7 +185,6 @@ def test_modules_meta_yml_structure_biotools_nometa(self): """Test the structure of the module meta.yml file when it was generated with INFORMATION from bio.tools and WITHOUT a meta.""" with responses.RequestsMock() as rsps: mock_anaconda_api_calls(rsps, "bpipe", "0.9.13--hdfd78af_0") - mock_biocontainers_api_calls(rsps, "bpipe", "0.9.13--hdfd78af_0") mock_biotools_api_calls(rsps, "bpipe") module_create = nf_core.modules.create.ModuleCreate( self.nfcore_modules, "bpipe/test", "@author", "process_single", has_meta=False, force=True @@ -270,15 +273,13 @@ def test_modules_meta_yml_structure_biotools_nometa(self): assert meta_yml == expected_yml @mock.patch("nf_core.utils.anaconda_package") - @mock.patch("nf_core.utils.get_biocontainer_tag") @mock.patch("nf_core.components.components_utils.get_biotools_response") @mock.patch("rich.prompt.Confirm.ask") def test_modules_meta_yml_structure_template_meta( - self, mock_rich_ask, mock_biotools_response, mock_biocontainer_tag, mock_anaconda_package + self, mock_rich_ask, mock_biotools_response, mock_anaconda_package ): """Test the structure of the module meta.yml file when it was generated with TEMPLATE data and WITH a meta.""" mock_biotools_response.return_value = {} - mock_biocontainer_tag.return_value = {} mock_anaconda_package.return_value = {} mock_rich_ask.return_value = False # Don't provide Bioconda package name module_create = nf_core.modules.create.ModuleCreate( @@ -385,15 +386,13 @@ def test_modules_meta_yml_structure_template_meta( assert meta_yml == expected_yml @mock.patch("nf_core.utils.anaconda_package") - @mock.patch("nf_core.utils.get_biocontainer_tag") @mock.patch("nf_core.components.components_utils.get_biotools_response") @mock.patch("rich.prompt.Confirm.ask") def test_modules_meta_yml_structure_template_nometa( - self, mock_rich_ask, mock_biotools_response, mock_biocontainer_tag, mock_anaconda_package + self, mock_rich_ask, mock_biotools_response, mock_anaconda_package ): """Test the structure of the module meta.yml file when it was generated with TEMPLATE data and WITHOUT a meta.""" mock_biotools_response.return_value = {} - mock_biocontainer_tag.return_value = {} mock_anaconda_package.return_value = {} mock_rich_ask.return_value = False # Don't provide Bioconda package name module_create = nf_core.modules.create.ModuleCreate( @@ -484,15 +483,11 @@ def test_modules_meta_yml_structure_template_nometa( assert meta_yml == expected_yml @mock.patch("nf_core.utils.anaconda_package") - @mock.patch("nf_core.utils.get_biocontainer_tag") @mock.patch("nf_core.components.components_utils.get_biotools_response") @mock.patch("rich.prompt.Confirm.ask") - def test_modules_meta_yml_structure_empty_meta( - self, mock_rich_ask, mock_biotools_response, mock_biocontainer_tag, mock_anaconda_package - ): + def test_modules_meta_yml_structure_empty_meta(self, mock_rich_ask, mock_biotools_response, mock_anaconda_package): """Test the structure of the module meta.yml file when it was generated with an EMPTY template and WITH a meta.""" mock_biotools_response.return_value = {} - mock_biocontainer_tag.return_value = {} mock_anaconda_package.return_value = {} mock_rich_ask.return_value = False # Don't provide Bioconda package name module_create = nf_core.modules.create.ModuleCreate( @@ -577,15 +572,13 @@ def test_modules_meta_yml_structure_empty_meta( assert meta_yml == expected_yml @mock.patch("nf_core.utils.anaconda_package") - @mock.patch("nf_core.utils.get_biocontainer_tag") @mock.patch("nf_core.components.components_utils.get_biotools_response") @mock.patch("rich.prompt.Confirm.ask") def test_modules_meta_yml_structure_empty_nometa( - self, mock_rich_ask, mock_biotools_response, mock_biocontainer_tag, mock_anaconda_package + self, mock_rich_ask, mock_biotools_response, mock_anaconda_package ): """Test the structure of the module meta.yml file when it was generated with an EMPTY template and WITHOUT a meta.""" mock_biotools_response.return_value = {} - mock_biocontainer_tag.return_value = {} mock_anaconda_package.return_value = {} mock_rich_ask.return_value = False # Don't provide Bioconda package name module_create = nf_core.modules.create.ModuleCreate( @@ -648,3 +641,44 @@ def test_modules_meta_yml_structure_empty_nometa( meta_yml = yaml.safe_load(fh) assert meta_yml == expected_yml + + +class TestModulesCreateSeqeraContainers(TestModules): + """`modules create` builds Seqera/Wave containers for the new module instead of using BioContainers.""" + + @mock.patch.object(ModuleContainers, "get_conda_lock_file") + @mock.patch.object(ModuleContainers, "request_container") + def test_create_builds_seqera_containers(self, mock_request_container, mock_get_conda_lock_file): + """Creating a module wires the Wave-built containers into main.nf. + + The Wave HTTP layer (``request_container`` / ``get_conda_lock_file``) is mocked so + the real container building – exercised in ``tests/modules/test_containers.py`` – is + not re-tested here; this only asserts the build is invoked and its result lands in + ``main.nf``. + """ + docker_image = "community.wave.seqera.io/library/fastqc:0.11.9--docker" + singularity_https = "https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/ab/abc/data" + + def fake_request_container(cs, platform, environment_yml, verbose=False, on_build_id=None, cancel_event=None): + if cs == "singularity": + return ContainerEntry(name=f"sif-{platform}", build_id=f"bd-sing-{platform}", https=singularity_https) + return ContainerEntry(name=docker_image, build_id=f"bd-docker-{platform}", scan_id="sc") + + mock_request_container.side_effect = fake_request_container + mock_get_conda_lock_file.return_value = "# conda lock file content" + + with responses.RequestsMock() as rsps: + mock_anaconda_api_calls(rsps, "fastqc", "0.11.9") + module_create = nf_core.modules.create.ModuleCreate( + self.nfcore_modules, "fastqc", "@author", "process_low", False, False + ) + with requests_cache.disabled(): + module_create.create() + + main_nf = Path(self.nfcore_modules, "modules", "nf-core", "fastqc", "main.nf").read_text() + assert docker_image in main_nf + assert singularity_https in main_nf + assert "YOUR-TOOL-HERE" not in main_nf + + # Wave was asked to build every container system / platform combination + assert mock_request_container.call_count == len(CONTAINER_SYSTEMS) * len(CONTAINER_PLATFORMS) diff --git a/tests/test_modules.py b/tests/test_modules.py index 52603d7326..3cfcb48dad 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -3,12 +3,14 @@ import json import unittest from pathlib import Path +from unittest import mock import pytest import requests_cache import responses import ruamel.yaml +import nf_core.components.create import nf_core.modules.create import nf_core.modules.install import nf_core.modules.modules_repo @@ -28,7 +30,6 @@ OLD_TRIMGALORE_SHA, create_tmp_pipeline, mock_anaconda_api_calls, - mock_biocontainers_api_calls, mock_biotools_api_calls, ) @@ -47,16 +48,20 @@ def create_modules_repo_dummy(tmp_dir): nf_core_yml = NFCoreYamlConfig(nf_core_version=__version__, repository_type="modules", org_path="nf-core") with open(Path(root_dir, ".nf-core.yml"), "w") as fh: yaml.dump(nf_core_yml.model_dump(), fh) - # mock biocontainers and anaconda response and biotools response + # mock anaconda and biotools responses with responses.RequestsMock() as rsps: mock_anaconda_api_calls(rsps, "bpipe", "0.9.13--hdfd78af_0") - mock_biocontainers_api_calls(rsps, "bpipe", "0.9.13--hdfd78af_0") mock_biotools_api_calls(rsps, "bpipe") # bpipe is a valid package on bioconda that is very unlikely to ever be added to nf-core/modules module_create = nf_core.modules.create.ModuleCreate( root_dir, "bpipe/test", "@author", "process_single", True, False ) - with requests_cache.disabled(): + # `modules create` builds Seqera/Wave containers for the new module; the dummy + # repo's containers are written explicitly below, so stub the network call out here. + with ( + requests_cache.disabled(), + mock.patch.object(nf_core.components.create.ComponentCreate, "_build_seqera_containers"), + ): assert module_create.create() # Remove doi from meta.yml which makes lint fail