diff --git a/src/main/groovy/io/seqera/wave/util/ContainerHelper.groovy b/src/main/groovy/io/seqera/wave/util/ContainerHelper.groovy index 8c2185a9c..20b350208 100644 --- a/src/main/groovy/io/seqera/wave/util/ContainerHelper.groovy +++ b/src/main/groovy/io/seqera/wave/util/ContainerHelper.groovy @@ -36,6 +36,8 @@ import org.yaml.snakeyaml.Yaml import static io.seqera.wave.api.BuildTemplate.CONDA_MICROMAMBA_V1 import static io.seqera.wave.api.BuildTemplate.CONDA_MICROMAMBA_V2 import static io.seqera.wave.api.BuildTemplate.CONDA_PIXI_V1 +import static io.seqera.wave.api.BuildTemplate.CONDA_PIXI_LOCK_V1 +import static io.seqera.wave.api.BuildTemplate.CONDA_PIXI_TOML_V1 import static io.seqera.wave.api.BuildTemplate.CRAN_INSTALLR_V1 import static io.seqera.wave.service.builder.BuildFormat.SINGULARITY import static DockerHelper.condaEnvironmentToCondaYaml @@ -70,6 +72,12 @@ class ContainerHelper { if( req.buildTemplate == CONDA_PIXI_V1 ) { return PixiHelper.containerFile(spec, req.containerImage, singularity) } + if( req.buildTemplate == CONDA_PIXI_LOCK_V1 ) { + return PixiLockHelper.containerFile(spec, req.containerImage, singularity) + } + if( req.buildTemplate == CONDA_PIXI_TOML_V1 ) { + return PixiTomlHelper.containerFile(spec, req.containerImage, singularity) + } if( spec.type == PackagesSpec.Type.CRAN && (!req.buildTemplate || req.buildTemplate == CRAN_INSTALLR_V1) ) { return CranHelper.containerFile(spec, singularity) } @@ -84,6 +92,13 @@ class ContainerHelper { if( req.packages.type != PackagesSpec.Type.CONDA ) return null + if( req.buildTemplate == CONDA_PIXI_LOCK_V1 || req.buildTemplate == CONDA_PIXI_TOML_V1 ) { + if( req.packages.environment ) { + return decodeBase64OrFail(req.packages.environment, 'packages.environment') + } + return null + } + if( req.packages.environment ) { // parse the attribute as a conda file path *and* append the base packages if any // note 'channel' is null, because they are expected to be provided in the conda file diff --git a/src/main/groovy/io/seqera/wave/util/PixiLockHelper.groovy b/src/main/groovy/io/seqera/wave/util/PixiLockHelper.groovy new file mode 100644 index 000000000..6bef79d1b --- /dev/null +++ b/src/main/groovy/io/seqera/wave/util/PixiLockHelper.groovy @@ -0,0 +1,59 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.util + +import groovy.transform.CompileStatic +import io.seqera.wave.api.PackagesSpec +import io.seqera.wave.config.PixiOpts +import io.seqera.wave.exception.BadRequestException + +import static TemplateUtils.pixiLockFileToDockerFile +import static TemplateUtils.pixiLockFileToSingularityFile +import static TemplateUtils.pixiLockUrlToDockerFile +import static TemplateUtils.pixiLockUrlToSingularityFile + +/** + * Helper class for Pixi lock file-based container builds (CONDA_PIXI_LOCK_V1 template). + * + * @author Julianus Pfeuffer <8102638+jpfeuffer@users.noreply.github.com> + */ +@CompileStatic +class PixiLockHelper { + + static String containerFile(PackagesSpec spec, String containerImage, boolean singularity) { + if( spec.type != PackagesSpec.Type.CONDA ) { + throw new BadRequestException("Package type '${spec.type}' not supported by 'conda/pixi-lock:v1' build template") + } + + final lockFileUrl = CondaHelper.tryGetLockFile(spec.entries) + final opts = spec.pixiOpts ?: new PixiOpts() + if( containerImage ) + opts.baseImage = containerImage + + if( lockFileUrl ) { + return singularity + ? pixiLockUrlToSingularityFile(lockFileUrl, opts) + : pixiLockUrlToDockerFile(lockFileUrl, opts) + } + + return singularity + ? pixiLockFileToSingularityFile(opts) + : pixiLockFileToDockerFile(opts) + } +} diff --git a/src/main/groovy/io/seqera/wave/util/PixiTomlHelper.groovy b/src/main/groovy/io/seqera/wave/util/PixiTomlHelper.groovy new file mode 100644 index 000000000..34ead6393 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/util/PixiTomlHelper.groovy @@ -0,0 +1,61 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.util + +import groovy.transform.CompileStatic +import io.seqera.wave.api.PackagesSpec +import io.seqera.wave.config.PixiOpts +import io.seqera.wave.exception.BadRequestException + +import static TemplateUtils.pixiTomlFileToDockerFile +import static TemplateUtils.pixiTomlFileToSingularityFile +import static TemplateUtils.pixiTomlUrlToDockerFile +import static TemplateUtils.pixiTomlUrlToSingularityFile + +/** + * Helper class for Pixi manifest (pixi.toml) file-based container builds (CONDA_PIXI_TOML_V1 template). + * Unlike the lock-file path, this runs {@code pixi install} with online solving, making it + * equivalent to providing a conda.yml but using the Pixi toolchain. + * + * @author Julianus Pfeuffer <8102638+jpfeuffer@users.noreply.github.com> + */ +@CompileStatic +class PixiTomlHelper { + + static String containerFile(PackagesSpec spec, String containerImage, boolean singularity) { + if( spec.type != PackagesSpec.Type.CONDA ) { + throw new BadRequestException("Package type '${spec.type}' not supported by 'conda/pixi-toml:v1' build template") + } + + final tomlFileUrl = CondaHelper.tryGetLockFile(spec.entries) + final opts = spec.pixiOpts ?: new PixiOpts() + if( containerImage ) + opts.baseImage = containerImage + + if( tomlFileUrl ) { + return singularity + ? pixiTomlUrlToSingularityFile(tomlFileUrl, opts) + : pixiTomlUrlToDockerFile(tomlFileUrl, opts) + } + + return singularity + ? pixiTomlFileToSingularityFile(opts) + : pixiTomlFileToDockerFile(opts) + } +} diff --git a/src/main/java/io/seqera/wave/util/TemplateUtils.java b/src/main/java/io/seqera/wave/util/TemplateUtils.java index 8b90a6336..41c69d382 100644 --- a/src/main/java/io/seqera/wave/util/TemplateUtils.java +++ b/src/main/java/io/seqera/wave/util/TemplateUtils.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.io.InputStream; import java.net.URL; +import java.util.Base64; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -32,7 +33,7 @@ /** * Helper class to create Dockerfile for Conda package manager * - * @author Paolo Di Tommaso + * @author Julianus Pfeuffer <8102638+jpfeuffer@users.noreply.github.com> */ public class TemplateUtils { @@ -113,6 +114,22 @@ static public String condaFileToSingularityFileUsingPixi(PixiOpts opts) { return condaFileTemplate1("/templates/conda-pixi-v1/singularityfile-conda-file.txt", opts); } + static public String pixiLockUrlToDockerFile(String lockUrl, PixiOpts opts) { + return pixiLockUrlTemplate("/templates/conda-pixi-lock-v1/dockerfile-pixi-lock-url.txt", lockUrl, opts); + } + + static public String pixiLockUrlToSingularityFile(String lockUrl, PixiOpts opts) { + return pixiLockUrlTemplate("/templates/conda-pixi-lock-v1/singularityfile-pixi-lock-url.txt", lockUrl, opts); + } + + static public String pixiLockFileToDockerFile(PixiOpts opts) { + return pixiLockFileTemplate("/templates/conda-pixi-lock-v1/dockerfile-pixi-lock-file.txt", opts); + } + + static public String pixiLockFileToSingularityFile(PixiOpts opts) { + return pixiLockFileTemplate("/templates/conda-pixi-lock-v1/singularityfile-pixi-lock-file.txt", opts); + } + static public String condaPackagesToDockerFileUsingV2(String packages, List condaChannels, CondaOpts opts) { return condaPackagesTemplate1( "/templates/conda-micromamba-v2/dockerfile-conda-packages.txt", @@ -164,6 +181,96 @@ static protected String condaFileTemplate1(String template, PixiOpts opts) { return addCommands(result, opts.commands, singularity); } + static public String pixiTomlFileToDockerFile(PixiOpts opts) { + return pixiTomlFileTemplate("/templates/conda-pixi-toml-v1/dockerfile-pixi-toml-file.txt", opts); + } + + static public String pixiTomlUrlToDockerFile(String tomlUrl, PixiOpts opts) { + return pixiTomlUrlTemplate("/templates/conda-pixi-toml-v1/dockerfile-pixi-toml-url.txt", tomlUrl, opts); + } + + static public String pixiTomlFileToSingularityFile(PixiOpts opts) { + return pixiTomlFileTemplate("/templates/conda-pixi-toml-v1/singularityfile-pixi-toml-file.txt", opts); + } + + static public String pixiTomlUrlToSingularityFile(String tomlUrl, PixiOpts opts) { + return pixiTomlUrlTemplate("/templates/conda-pixi-toml-v1/singularityfile-pixi-toml-url.txt", tomlUrl, opts); + } + + static protected String pixiTomlFileTemplate(String template, PixiOpts opts) { + final boolean singularity = template.contains("/singularityfile"); + final Map binding = new HashMap<>(); + binding.put("base_image", opts.baseImage); + binding.put("pixi_image", opts.pixiImage); + + final String result = renderTemplate0(template, binding, List.of("wave_context_dir")); + return addCommands(result, opts.commands, singularity); + } + + static protected String pixiTomlUrlTemplate(String template, String tomlUrl, PixiOpts opts) { + final boolean singularity = template.contains("/singularityfile"); + final Map binding = new HashMap<>(); + binding.put("base_image", opts.baseImage); + binding.put("pixi_image", opts.pixiImage); + binding.put("toml_url", tomlUrl); + + final String result = renderTemplate0(template, binding); + return addCommands(result, opts.commands, singularity); + } + + static protected String pixiLockUrlTemplate(String template, String lockUrl, PixiOpts opts) { + final boolean singularity = template.contains("/singularityfile"); + final Map binding = new HashMap<>(); + binding.put("base_image", opts.baseImage); + binding.put("pixi_image", opts.pixiImage); + binding.put("lock_url", lockUrl); + binding.put("manifest_generate", pixiManifestGenerate(opts.manifest, singularity)); + binding.put("base_packages", pixiGlobalInstallBasePackage0(opts.basePackages, singularity)); + + final String result = renderTemplate0(template, binding); + return addCommands(result, opts.commands, singularity); + } + + static protected String pixiLockFileTemplate(String template, PixiOpts opts) { + final boolean singularity = template.contains("/singularityfile"); + final Map binding = new HashMap<>(); + binding.put("base_image", opts.baseImage); + binding.put("pixi_image", opts.pixiImage); + binding.put("manifest_generate", pixiManifestGenerate(opts.manifest, singularity)); + binding.put("base_packages", pixiGlobalInstallBasePackage0(opts.basePackages, singularity)); + + final String result = renderTemplate0(template, binding, List.of("wave_context_dir")); + return addCommands(result, opts.commands, singularity); + } + + /** + * Generate the shell commands to create pixi.toml from either a provided manifest + * or by extracting metadata from the lock file. + */ + private static String pixiManifestGenerate(String manifest, boolean singularity) { + if (manifest != null && !manifest.isEmpty()) { + // Manifest provided: decode from base64 at build time + final String encoded = Base64.getEncoder().encodeToString(manifest.getBytes()); + if (singularity) { + return "printf '%s' '" + encoded + "' | base64 -d > pixi.toml"; + } else { + return "printf '%s' '" + encoded + "' | base64 -d > pixi.toml \\"; + } + } + // No manifest: generate from lock file by extracting channels, platforms, and deps + final String awkDeps = "awk '/- conda:/{n=split($NF,u,\"/\");f=u[n];sub(/\\.(conda|tar\\.bz2)$/,\"\",f);m=split(f,s,\"-\");v=0;for(i=2;i<=m;i++)if(s[i]~/^[0-9]/&&index(s[i],\".\")>0){v=i;break};if(!v)for(i=2;i<=m;i++)if(s[i]~/^[0-9]/){v=i;break};nm=\"\";for(i=1;i<(v?v:m+1);i++){if(nm)nm=nm\"-\";nm=nm s[i]};if(nm)print nm\" = \\\\\"*\\\\\"\"}' pixi.lock | sort -u > /tmp/deps.toml"; + final String awkPlatforms = "PLATFORMS=$(awk '/- conda:/{n=split($NF,u,\"/\");p=u[n-1];if(p!=\"noarch\")print p}' pixi.lock | sort -u | awk '{printf \"\\\\\"%s\\\\\", \", $0}' | sed 's/, $//')"; + final String awkChannels = "CHANNELS=$(awk '/^ channels:/{ch=1;next} ch&&/- url:/{gsub(/^ *- url: */,\"\");gsub(/https:\\/\\/conda\\.anaconda\\.org\\//,\"\");gsub(/\\/$/,\"\");printf \"\\\\\"%s\\\\\", \",$0;next} ch&&!/^ -/{exit}' pixi.lock | sed 's/, $//')"; + final String printfToml = "printf \"[workspace]\\nname = \\\"wave-env\\\"\\nchannels = [%s]\\nplatforms = [%s]\\n\\n[dependencies]\\n\" \"$CHANNELS\" \"$PLATFORMS\" > pixi.toml"; + final String catDeps = "cat /tmp/deps.toml >> pixi.toml"; + + if (singularity) { + return awkPlatforms + "\n " + awkChannels + "\n " + awkDeps + "\n " + printfToml + "\n " + catDeps; + } else { + return awkPlatforms + " \\\n && " + awkChannels + " \\\n && " + awkDeps + " \\\n && " + printfToml + " \\\n && " + catDeps + " \\"; + } + } + static private String renderTemplate0(String templatePath, Map binding) { return renderTemplate0(templatePath, binding, List.of()); } @@ -201,6 +308,22 @@ private static String pixiAddBasePackage0(String basePackages, boolean singulari : "&& " + result + " \\"; } + /** + * Install base packages in an isolated pixi global prefix so that the project + * environment installed via {@code pixi install --frozen} (and its lock file) is + * left untouched. This is used by the pixi lock build templates to add runtime + * tools such as {@code procps-ng} (providing the {@code ps} command) without + * re-solving the locked environment. + */ + private static String pixiGlobalInstallBasePackage0(String basePackages, boolean singularity) { + String result = !StringUtils.isEmpty(basePackages) + ? String.format("pixi global install %s", basePackages) + : null; + return result==null || singularity + ? result + : "&& " + result + " \\"; + } + static private String addCommands(String result, List commands, boolean singularity) { if( commands==null || commands.isEmpty() ) return result; diff --git a/src/main/resources/templates/conda-pixi-lock-v1/dockerfile-pixi-lock-file.txt b/src/main/resources/templates/conda-pixi-lock-v1/dockerfile-pixi-lock-file.txt new file mode 100644 index 000000000..2efae6bf9 --- /dev/null +++ b/src/main/resources/templates/conda-pixi-lock-v1/dockerfile-pixi-lock-file.txt @@ -0,0 +1,26 @@ +FROM {{pixi_image}} AS build + +COPY conda.yml /opt/wave/pixi.lock +WORKDIR /opt/wave +RUN {{manifest_generate}} + && pixi install --frozen \ + && mkdir -p /root/.pixi/bin \ + {{base_packages}} + && pixi shell-hook > /shell-hook.sh \ + && echo 'exec "$@"' >> /shell-hook.sh \ + && echo ">> CONDA_LOCK_START" \ + && cat /opt/wave/pixi.lock \ + && echo "<< CONDA_LOCK_END" + +FROM {{base_image}} AS final + +COPY --from=build /opt/wave/.pixi/envs/default /opt/wave/.pixi/envs/default +COPY --from=build /root/.pixi /root/.pixi +COPY --from=build /shell-hook.sh /shell-hook.sh + +USER root +ENV USER=root +ENV PATH=/root/.pixi/bin:$PATH + +ENTRYPOINT ["/bin/bash", "/shell-hook.sh"] +CMD ["/bin/bash"] diff --git a/src/main/resources/templates/conda-pixi-lock-v1/dockerfile-pixi-lock-url.txt b/src/main/resources/templates/conda-pixi-lock-v1/dockerfile-pixi-lock-url.txt new file mode 100644 index 000000000..4e3b3f09e --- /dev/null +++ b/src/main/resources/templates/conda-pixi-lock-v1/dockerfile-pixi-lock-url.txt @@ -0,0 +1,26 @@ +FROM {{pixi_image}} AS build + +WORKDIR /opt/wave +RUN curl -fsSL {{lock_url}} -o pixi.lock \ + && {{manifest_generate}} + && pixi install --frozen \ + && mkdir -p /root/.pixi/bin \ + {{base_packages}} + && pixi shell-hook > /shell-hook.sh \ + && echo 'exec "$@"' >> /shell-hook.sh \ + && echo ">> CONDA_LOCK_START" \ + && cat /opt/wave/pixi.lock \ + && echo "<< CONDA_LOCK_END" + +FROM {{base_image}} AS final + +COPY --from=build /opt/wave/.pixi/envs/default /opt/wave/.pixi/envs/default +COPY --from=build /root/.pixi /root/.pixi +COPY --from=build /shell-hook.sh /shell-hook.sh + +USER root +ENV USER=root +ENV PATH=/root/.pixi/bin:$PATH + +ENTRYPOINT ["/bin/bash", "/shell-hook.sh"] +CMD ["/bin/bash"] diff --git a/src/main/resources/templates/conda-pixi-lock-v1/singularityfile-pixi-lock-file.txt b/src/main/resources/templates/conda-pixi-lock-v1/singularityfile-pixi-lock-file.txt new file mode 100644 index 000000000..8583530f6 --- /dev/null +++ b/src/main/resources/templates/conda-pixi-lock-v1/singularityfile-pixi-lock-file.txt @@ -0,0 +1,16 @@ +BootStrap: docker +From: {{pixi_image}} +%files + {{wave_context_dir}}/conda.yml /opt/wave/pixi.lock +%post + cd /opt/wave + {{manifest_generate}} + pixi install --frozen + {{base_packages}} + pixi shell-hook > /shell-hook.sh + echo ">> CONDA_LOCK_START" + cat /opt/wave/pixi.lock + echo "<< CONDA_LOCK_END" +%environment + export PATH=/root/.pixi/bin:$PATH + . /shell-hook.sh diff --git a/src/main/resources/templates/conda-pixi-lock-v1/singularityfile-pixi-lock-url.txt b/src/main/resources/templates/conda-pixi-lock-v1/singularityfile-pixi-lock-url.txt new file mode 100644 index 000000000..bfc001b38 --- /dev/null +++ b/src/main/resources/templates/conda-pixi-lock-v1/singularityfile-pixi-lock-url.txt @@ -0,0 +1,15 @@ +BootStrap: docker +From: {{pixi_image}} +%post + cd /opt/wave + curl -fsSL {{lock_url}} -o pixi.lock + {{manifest_generate}} + pixi install --frozen + {{base_packages}} + pixi shell-hook > /shell-hook.sh + echo ">> CONDA_LOCK_START" + cat /opt/wave/pixi.lock + echo "<< CONDA_LOCK_END" +%environment + export PATH=/root/.pixi/bin:$PATH + . /shell-hook.sh diff --git a/src/main/resources/templates/conda-pixi-toml-v1/dockerfile-pixi-toml-file.txt b/src/main/resources/templates/conda-pixi-toml-v1/dockerfile-pixi-toml-file.txt new file mode 100644 index 000000000..bab6430a3 --- /dev/null +++ b/src/main/resources/templates/conda-pixi-toml-v1/dockerfile-pixi-toml-file.txt @@ -0,0 +1,21 @@ +FROM {{pixi_image}} AS build + +COPY conda.yml /opt/wave/pixi.toml +WORKDIR /opt/wave +RUN pixi install \ + && pixi shell-hook > /shell-hook.sh \ + && echo 'exec "$@"' >> /shell-hook.sh \ + && echo ">> PIXI_LOCK_START" \ + && cat /opt/wave/pixi.lock \ + && echo "<< PIXI_LOCK_END" + +FROM {{base_image}} AS final + +COPY --from=build /opt/wave/.pixi/envs/default /opt/wave/.pixi/envs/default +COPY --from=build /shell-hook.sh /shell-hook.sh + +USER root +ENV USER=root + +ENTRYPOINT ["/bin/bash", "/shell-hook.sh"] +CMD ["/bin/bash"] diff --git a/src/main/resources/templates/conda-pixi-toml-v1/dockerfile-pixi-toml-url.txt b/src/main/resources/templates/conda-pixi-toml-v1/dockerfile-pixi-toml-url.txt new file mode 100644 index 000000000..8f5d5feeb --- /dev/null +++ b/src/main/resources/templates/conda-pixi-toml-v1/dockerfile-pixi-toml-url.txt @@ -0,0 +1,21 @@ +FROM {{pixi_image}} AS build + +WORKDIR /opt/wave +RUN curl -fsSL {{toml_url}} -o pixi.toml \ + && pixi install \ + && pixi shell-hook > /shell-hook.sh \ + && echo 'exec "$@"' >> /shell-hook.sh \ + && echo ">> PIXI_LOCK_START" \ + && cat /opt/wave/pixi.lock \ + && echo "<< PIXI_LOCK_END" + +FROM {{base_image}} AS final + +COPY --from=build /opt/wave/.pixi/envs/default /opt/wave/.pixi/envs/default +COPY --from=build /shell-hook.sh /shell-hook.sh + +USER root +ENV USER=root + +ENTRYPOINT ["/bin/bash", "/shell-hook.sh"] +CMD ["/bin/bash"] diff --git a/src/main/resources/templates/conda-pixi-toml-v1/singularityfile-pixi-toml-file.txt b/src/main/resources/templates/conda-pixi-toml-v1/singularityfile-pixi-toml-file.txt new file mode 100644 index 000000000..c7d7fded1 --- /dev/null +++ b/src/main/resources/templates/conda-pixi-toml-v1/singularityfile-pixi-toml-file.txt @@ -0,0 +1,13 @@ +BootStrap: docker +From: {{pixi_image}} +%files + {{wave_context_dir}}/conda.yml /opt/wave/pixi.toml +%post + cd /opt/wave + pixi install + pixi shell-hook > /shell-hook.sh + echo ">> PIXI_LOCK_START" + cat /opt/wave/pixi.lock + echo "<< PIXI_LOCK_END" +%environment + . /shell-hook.sh diff --git a/src/main/resources/templates/conda-pixi-toml-v1/singularityfile-pixi-toml-url.txt b/src/main/resources/templates/conda-pixi-toml-v1/singularityfile-pixi-toml-url.txt new file mode 100644 index 000000000..73bd0553a --- /dev/null +++ b/src/main/resources/templates/conda-pixi-toml-v1/singularityfile-pixi-toml-url.txt @@ -0,0 +1,13 @@ +BootStrap: docker +From: {{pixi_image}} +%post + mkdir -p /opt/wave + curl -fsSL {{toml_url}} -o /opt/wave/pixi.toml + cd /opt/wave + pixi install + pixi shell-hook > /shell-hook.sh + echo ">> PIXI_LOCK_START" + cat /opt/wave/pixi.lock + echo "<< PIXI_LOCK_END" +%environment + . /shell-hook.sh diff --git a/src/test/groovy/io/seqera/wave/controller/ContainerControllerTest.groovy b/src/test/groovy/io/seqera/wave/controller/ContainerControllerTest.groovy index 4c2fababf..1fc56f9e7 100644 --- a/src/test/groovy/io/seqera/wave/controller/ContainerControllerTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/ContainerControllerTest.groovy @@ -33,6 +33,7 @@ import io.micronaut.http.client.annotation.Client import io.micronaut.http.server.util.HttpClientAddressResolver import io.micronaut.test.annotation.MockBean import io.micronaut.test.extensions.spock.annotation.MicronautTest +import io.seqera.wave.api.BuildTemplate import io.seqera.wave.api.ContainerConfig import io.seqera.wave.api.ContainerLayer import io.seqera.wave.api.ContainerStatusResponse @@ -589,6 +590,118 @@ class ContainerControllerTest extends Specification { } } + def 'should prepare pixi lock build request from inline packages environment'() { + given: + def dockerAuth = Mock(ContainerInspectServiceImpl) + def builder = Mock(ContainerBuildService) + def proxyRegistry = Mock(RegistryProxyService) + def addressResolver = Mock(HttpClientAddressResolver) + def tokenService = Mock(ContainerRequestService) + def persistence = Mock(PersistenceService) + def controller = new ContainerController( + freezeService: new FreezeServiceImpl(inspectService: dockerAuth), + buildService: builder, + inspectService: dockerAuth, + registryProxyService: proxyRegistry, + buildConfig: buildConfig, + inclusionService: Mock(ContainerInclusionService), + addressResolver: addressResolver, + containerService: tokenService, + persistenceService: persistence, + validationService: validationService, + serverUrl: 'http://wave.com') + def lockContent = '''\ + version: 6 + environments: + default: + channels: + - url: https://conda.anaconda.org/conda-forge/ + '''.stripIndent() + def req = new SubmitContainerTokenRequest( + packages: new PackagesSpec( + type: PackagesSpec.Type.CONDA, + environment: encode(lockContent) + ), + buildTemplate: BuildTemplate.CONDA_PIXI_LOCK_V1 + ) + def user = new User(email: 'pixi-inline@bar.com', userName: 'pixi-inline') + def id = PlatformId.of(user, req) + + when: + def response = controller.handleRequest(null, req, id, true) + + then: + 1 * builder.buildImage(_ as BuildRequest) >> { BuildRequest build -> + assert build.buildTemplate == BuildTemplate.CONDA_PIXI_LOCK_V1 + assert build.condaFile == lockContent + assert build.containerFile.contains('COPY conda.yml /opt/wave/pixi.lock') + assert build.containerFile.contains('pixi install --frozen') + new BuildTrack('build-inline-pixi', build.targetImage, false, true) + } + 1 * tokenService.computeToken(_) >> new TokenData('wavetoken-inline', Instant.now().plus(1, ChronoUnit.HOURS)) + and: + response.status.code == 200 + verifyAll(response.body.get() as SubmitContainerTokenResponse) { + buildId == 'build-inline-pixi' + containerToken == 'wavetoken-inline' + cached == false + succeeded == true + } + } + + def 'should prepare pixi lock build request from packages URL'() { + given: + def dockerAuth = Mock(ContainerInspectServiceImpl) + def builder = Mock(ContainerBuildService) + def proxyRegistry = Mock(RegistryProxyService) + def addressResolver = Mock(HttpClientAddressResolver) + def tokenService = Mock(ContainerRequestService) + def persistence = Mock(PersistenceService) + def controller = new ContainerController( + freezeService: new FreezeServiceImpl(inspectService: dockerAuth), + buildService: builder, + inspectService: dockerAuth, + registryProxyService: proxyRegistry, + buildConfig: buildConfig, + inclusionService: Mock(ContainerInclusionService), + addressResolver: addressResolver, + containerService: tokenService, + persistenceService: persistence, + validationService: validationService, + serverUrl: 'http://wave.com') + def lockUrl = 'https://example.com/pixi.lock' + def req = new SubmitContainerTokenRequest( + packages: new PackagesSpec( + type: PackagesSpec.Type.CONDA, + entries: [lockUrl] + ), + buildTemplate: BuildTemplate.CONDA_PIXI_LOCK_V1 + ) + def user = new User(email: 'pixi-url@bar.com', userName: 'pixi-url') + def id = PlatformId.of(user, req) + + when: + def response = controller.handleRequest(null, req, id, true) + + then: + 1 * builder.buildImage(_ as BuildRequest) >> { BuildRequest build -> + assert build.buildTemplate == BuildTemplate.CONDA_PIXI_LOCK_V1 + assert build.condaFile == null + assert build.containerFile.contains("curl -fsSL ${lockUrl} -o pixi.lock") + assert build.containerFile.contains('pixi install --frozen') + new BuildTrack('build-url-pixi', build.targetImage, false, true) + } + 1 * tokenService.computeToken(_) >> new TokenData('wavetoken-url', Instant.now().plus(1, ChronoUnit.HOURS)) + and: + response.status.code == 200 + verifyAll(response.body.get() as SubmitContainerTokenResponse) { + buildId == 'build-url-pixi' + containerToken == 'wavetoken-url' + cached == false + succeeded == true + } + } + def 'should throw BadRequestException when more than one artifact (container image, container file or packages) is provided in the request' () { given: def controller = new ContainerController(validationService: validationService, inclusionService: Mock(ContainerInclusionService), allowAnonymous: false) diff --git a/src/test/groovy/io/seqera/wave/util/ContainerHelperTest.groovy b/src/test/groovy/io/seqera/wave/util/ContainerHelperTest.groovy index a00bc93d6..3ce526c95 100644 --- a/src/test/groovy/io/seqera/wave/util/ContainerHelperTest.groovy +++ b/src/test/groovy/io/seqera/wave/util/ContainerHelperTest.groovy @@ -160,6 +160,36 @@ class ContainerHelperTest extends Specification { '''.stripIndent() } + def 'should dispatch to PixiLockHelper for pixi-lock template with URL'() { + given: + def req = new SubmitContainerTokenRequest( + packages: new PackagesSpec(type: PackagesSpec.Type.CONDA, entries: ['https://example.com/pixi.lock']), + buildTemplate: 'conda/pixi-lock:v1' + ) + + when: + def result = ContainerHelper.containerFileFromRequest(req) + + then: + result.contains('curl -fsSL https://example.com/pixi.lock -o pixi.lock') + result.contains('pixi install --frozen') + } + + def 'should dispatch to PixiLockHelper for pixi-lock template with environment'() { + given: + def req = new SubmitContainerTokenRequest( + packages: new PackagesSpec(type: PackagesSpec.Type.CONDA, environment: 'dmVyc2lvbjogNgo='), + buildTemplate: 'conda/pixi-lock:v1' + ) + + when: + def result = ContainerHelper.containerFileFromRequest(req) + + then: + result.contains('COPY conda.yml /opt/wave/pixi.lock') + result.contains('pixi install --frozen') + } + def 'should validate conda file helper' () { given: def CONDA = 'this and that' @@ -233,6 +263,36 @@ class ContainerHelperTest extends Specification { '''.stripIndent() } + def 'should return pixi lock content from condaFileFromRequest for pixi-lock template'() { + given: + def lockContent = 'version: 6\nenvironments:\n default:\n' + def encoded = Base64.encoder.encodeToString(lockContent.bytes) + def req = new SubmitContainerTokenRequest( + packages: new PackagesSpec(type: PackagesSpec.Type.CONDA, environment: encoded), + buildTemplate: 'conda/pixi-lock:v1' + ) + + when: + def result = ContainerHelper.condaFileFromRequest(req) + + then: + result == lockContent + } + + def 'should return null from condaFileFromRequest for pixi-lock template with URL'() { + given: + def req = new SubmitContainerTokenRequest( + packages: new PackagesSpec(type: PackagesSpec.Type.CONDA, entries: ['https://example.com/pixi.lock']), + buildTemplate: 'conda/pixi-lock:v1' + ) + + when: + def result = ContainerHelper.condaFileFromRequest(req) + + then: + result == null + } + def 'should create response v1' () { given: def data = ContainerRequest.of( diff --git a/src/test/groovy/io/seqera/wave/util/PixiLockHelperTest.groovy b/src/test/groovy/io/seqera/wave/util/PixiLockHelperTest.groovy new file mode 100644 index 000000000..664c542e1 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/util/PixiLockHelperTest.groovy @@ -0,0 +1,135 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.util + +import spock.lang.Specification + +import io.seqera.wave.api.PackagesSpec +import io.seqera.wave.config.PixiOpts +import io.seqera.wave.exception.BadRequestException + +class PixiLockHelperTest extends Specification { + + def 'should create docker file from remote lock URL'() { + given: + def packages = new PackagesSpec(type: PackagesSpec.Type.CONDA, entries: ['https://foo.com/pixi.lock']) + + when: + def result = PixiLockHelper.containerFile(packages, null, false) + + then: + result.contains('FROM public.cr.seqera.io/wave/pixi:0.61.0-noble AS build') + result.contains('curl -fsSL https://foo.com/pixi.lock -o pixi.lock') + result.contains('pixi install --frozen') + result.contains('pixi shell-hook > /shell-hook.sh') + result.contains('FROM ubuntu:24.04 AS final') + result.contains('COPY --from=build /opt/wave/.pixi/envs/default /opt/wave/.pixi/envs/default') + result.contains('COPY --from=build /root/.pixi /root/.pixi') + result.contains('pixi global install conda-forge::procps-ng') + result.contains('ENTRYPOINT ["/bin/bash", "/shell-hook.sh"]') + !result.contains('pixi add') + result.contains('>> CONDA_LOCK_START') + result.contains('<< CONDA_LOCK_END') + } + + def 'should create singularity file from remote lock URL'() { + given: + def packages = new PackagesSpec(type: PackagesSpec.Type.CONDA, entries: ['https://foo.com/pixi.lock']) + + when: + def result = PixiLockHelper.containerFile(packages, null, true) + + then: + result.contains('BootStrap: docker') + result.contains('From: public.cr.seqera.io/wave/pixi:0.61.0-noble') + result.contains('curl -fsSL https://foo.com/pixi.lock -o pixi.lock') + result.contains('pixi install --frozen') + result.contains('pixi shell-hook > /shell-hook.sh') + result.contains('%post') + result.contains('%environment') + result.contains('. /shell-hook.sh') + } + + def 'should throw for non-CONDA type'() { + given: + def packages = new PackagesSpec(type: PackagesSpec.Type.CRAN, entries: ['dplyr']) + + when: + PixiLockHelper.containerFile(packages, null, false) + + then: + def ex = thrown(BadRequestException) + ex.message.contains("not supported by 'conda/pixi-lock:v1' build template") + } + + def 'should use custom pixi opts'() { + given: + def opts = new PixiOpts([ + pixiImage: 'ghcr.io/prefix-dev/pixi:0.47.0-jammy-cuda-12.8.1', + baseImage: 'base/image', + basePackages: null + ]) + def packages = new PackagesSpec(type: PackagesSpec.Type.CONDA, environment: 'ZW52', pixiOpts: opts) + + when: + def result = PixiLockHelper.containerFile(packages, null, false) + + then: + result.contains('FROM ghcr.io/prefix-dev/pixi:0.47.0-jammy-cuda-12.8.1 AS build') + result.contains('COPY conda.yml /opt/wave/pixi.lock') + result.contains('pixi install --frozen') + result.contains('FROM base/image AS final') + !result.contains('pixi add') + } + + def 'should install base packages via pixi global for lock build'() { + given: + def opts = new PixiOpts([ + pixiImage: 'ghcr.io/prefix-dev/pixi:0.47.0-jammy-cuda-12.8.1', + baseImage: 'base/image', + basePackages: 'foo::one bar::two' + ]) + def packages = new PackagesSpec(type: PackagesSpec.Type.CONDA, environment: 'ZW52', pixiOpts: opts) + + when: + def result = PixiLockHelper.containerFile(packages, null, false) + + then: + // base packages are installed in an isolated global prefix, never touching the frozen env + result.contains('pixi install --frozen') + result.contains('pixi global install foo::one bar::two') + result.contains('COPY --from=build /root/.pixi /root/.pixi') + result.contains('ENV PATH=/root/.pixi/bin:$PATH') + !result.contains('pixi add') + } + + def 'should use custom container image as base image'() { + given: + def opts = new PixiOpts([baseImage: 'base/image', basePackages: null]) + def packages = new PackagesSpec(type: PackagesSpec.Type.CONDA, environment: 'ZW52', pixiOpts: opts) + + when: + def result = PixiLockHelper.containerFile(packages, 'override/base:2.0', false) + + then: + result.contains('FROM public.cr.seqera.io/wave/pixi:0.61.0-noble AS build') + result.contains('FROM override/base:2.0 AS final') + !result.contains('FROM base/image AS final') + } +} diff --git a/src/test/groovy/io/seqera/wave/util/TemplateUtilsTest.groovy b/src/test/groovy/io/seqera/wave/util/TemplateUtilsTest.groovy index f423a5f90..c113f34bf 100644 --- a/src/test/groovy/io/seqera/wave/util/TemplateUtilsTest.groovy +++ b/src/test/groovy/io/seqera/wave/util/TemplateUtilsTest.groovy @@ -460,6 +460,63 @@ class TemplateUtilsTest extends Specification { '''.stripIndent() } + def 'should render pixi lock URL docker template'() { + given: + def opts = new PixiOpts([pixiImage: 'pixi:1.0', baseImage: 'ubuntu:22.04', basePackages: null]) + + when: + def result = TemplateUtils.pixiLockUrlToDockerFile('https://example.com/pixi.lock', opts) + + then: + result.contains('FROM pixi:1.0 AS build') + result.contains('curl -fsSL https://example.com/pixi.lock -o pixi.lock') + result.contains('pixi install --frozen') + result.contains('FROM ubuntu:22.04 AS final') + !result.contains('pixi add') + } + + def 'should render pixi lock file docker template'() { + given: + def opts = new PixiOpts([pixiImage: 'pixi:1.0', baseImage: 'ubuntu:22.04', basePackages: null]) + + when: + def result = TemplateUtils.pixiLockFileToDockerFile(opts) + + then: + result.contains('FROM pixi:1.0 AS build') + result.contains('COPY conda.yml /opt/wave/pixi.lock') + result.contains('pixi install --frozen') + result.contains('FROM ubuntu:22.04 AS final') + } + + def 'should render pixi lock URL singularity template'() { + given: + def opts = new PixiOpts([pixiImage: 'pixi:1.0', baseImage: 'ubuntu:22.04', basePackages: null]) + + when: + def result = TemplateUtils.pixiLockUrlToSingularityFile('https://example.com/pixi.lock', opts) + + then: + result.contains('BootStrap: docker') + result.contains('From: pixi:1.0') + result.contains('curl -fsSL https://example.com/pixi.lock -o pixi.lock') + result.contains('pixi install --frozen') + } + + def 'should render pixi lock file singularity template'() { + given: + def opts = new PixiOpts([pixiImage: 'pixi:1.0', baseImage: 'ubuntu:22.04', basePackages: null]) + + when: + def result = TemplateUtils.pixiLockFileToSingularityFile(opts) + + then: + result.contains('BootStrap: docker') + result.contains('From: pixi:1.0') + result.contains('pixi install --frozen') + result.contains('/opt/wave/pixi.lock') + } + /* ********************************************************************************* * Micromamba v2 template tests * diff --git a/wave-api/src/main/java/io/seqera/wave/api/BuildTemplate.java b/wave-api/src/main/java/io/seqera/wave/api/BuildTemplate.java index d8ad70230..9500d1ec2 100644 --- a/wave-api/src/main/java/io/seqera/wave/api/BuildTemplate.java +++ b/wave-api/src/main/java/io/seqera/wave/api/BuildTemplate.java @@ -29,6 +29,16 @@ public final class BuildTemplate { */ public static final String CONDA_PIXI_V1 = "conda/pixi:v1"; + /** + * Build template for Pixi lock file-based multi-stage builds + */ + public static final String CONDA_PIXI_LOCK_V1 = "conda/pixi-lock:v1"; + + /** + * Build template for Pixi manifest (pixi.toml) file-based multi-stage builds (online solving) + */ + public static final String CONDA_PIXI_TOML_V1 = "conda/pixi-toml:v1"; + /** * Build template for Micromamba v1-based multi-stage builds */ diff --git a/wave-api/src/main/java/io/seqera/wave/config/PixiOpts.java b/wave-api/src/main/java/io/seqera/wave/config/PixiOpts.java index a633e48df..e284ddd39 100644 --- a/wave-api/src/main/java/io/seqera/wave/config/PixiOpts.java +++ b/wave-api/src/main/java/io/seqera/wave/config/PixiOpts.java @@ -68,6 +68,12 @@ public class PixiOpts { */ public String baseImage; + /** + * Optional pixi manifest (pixi.toml) content. When provided alongside a pixi lock file, + * this manifest is used directly instead of generating one from the lock file. + */ + public String manifest; + /** * Creates a new instance with default values. */ @@ -86,6 +92,7 @@ public PixiOpts(Map opts) { this.baseImage = opts.containsKey("baseImage") ? opts.get("baseImage").toString(): DEFAULT_BASE_IMAGE; this.commands = opts.containsKey("commands") ? (List)opts.get("commands") : null; this.basePackages = opts.containsKey("basePackages") ? (String)opts.get("basePackages") : DEFAULT_PACKAGES; + this.manifest = opts.containsKey("manifest") ? opts.get("manifest").toString() : null; } public PixiOpts withPixiImage(String value) { @@ -108,13 +115,19 @@ public PixiOpts withBaseImage(String value) { return this; } + public PixiOpts withManifest(String value) { + this.manifest = value; + return this; + } + @Override public String toString() { - return String.format("PixiOpts(pixiImage=%s; basePackages=%s, commands=%s, baseImage=%s)", + return String.format("PixiOpts(pixiImage=%s; basePackages=%s, commands=%s, baseImage=%s, manifest=%s)", pixiImage, basePackages, commands != null ? String.join(",", commands) : "null", - baseImage + baseImage, + manifest != null ? "[provided]" : "null" ); } @@ -127,12 +140,13 @@ public boolean equals(Object object) { && Objects.equals(commands, pixiOpts.commands) && Objects.equals(basePackages, pixiOpts.basePackages) && Objects.equals(baseImage, pixiOpts.baseImage) + && Objects.equals(manifest, pixiOpts.manifest) ; } @Override public int hashCode() { - return Objects.hash(pixiImage, commands, basePackages, baseImage); + return Objects.hash(pixiImage, commands, basePackages, baseImage, manifest); } }