diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 47ed7c5..fb33cfd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -43,80 +43,3 @@ jobs: push: ${{ github.event_name == 'push' && github.repository == 'DIRACGrid/container-images' && github.ref_name == 'main' }} tags: ghcr.io/diracgrid/diracx/${{ matrix.image-name }}:latest platforms: linux/amd64,linux/arm64 - build_chain: - runs-on: ubuntu-latest - timeout-minutes: 30 - strategy: - fail-fast: false - steps: - - name: Checkout - uses: actions/checkout@v6 - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Set up oras - uses: oras-project/setup-oras@v1 - - name: Login to GitHub container registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build base - id: build_base - uses: docker/build-push-action@v6 - with: - context: base - outputs: type=oci,dest=output-base.tar - tags: ghcr.io/diracgrid/diracx/base:latest - platforms: linux/amd64,linux/arm64 - - name: Extract base - run: | - mkdir output-base - tar -C output-base -xf output-base.tar - - - name: Build services-base - uses: docker/build-push-action@v6 - with: - context: services-base - build-contexts: | - ghcr.io/diracgrid/diracx/base=oci-layout://output-base@${{steps.build_base.outputs.digest}} - outputs: type=oci,dest=output-services-base.tar - tags: ghcr.io/diracgrid/diracx/services-base:latest - platforms: linux/amd64,linux/arm64 - - name: Extract services-base - run: | - mkdir output-services-base - tar -C output-services-base -xf output-services-base.tar - - - name: Build client-base - uses: docker/build-push-action@v6 - with: - context: client-base - build-contexts: | - ghcr.io/diracgrid/diracx/base=oci-layout://output-base@${{steps.build_base.outputs.digest}} - outputs: type=oci,dest=output-client-base.tar - tags: ghcr.io/diracgrid/diracx/client-base:latest - platforms: linux/amd64,linux/arm64 - - name: Extract client-base - run: | - mkdir output-client-base - tar -C output-client-base -xf output-client-base.tar - - - name: Push images - if: ${{ github.event_name != 'pull_request' && github.repository == 'DIRACGrid/container-images' && github.ref_name == 'main' }} - run: | - set -x - - image_version=$(date +'%Y.%m.%d') - # Check how many tags already exist for the current date and increment the version - count=$(oras repo tags "ghcr.io/diracgrid/diracx/base" | { grep -c "${image_version}" || true; } ) - image_version=${image_version}.${count} - echo "Pushing version ${image_version}" - - for image_name in base services-base client-base; do - image_repo=ghcr.io/diracgrid/diracx/${image_name} - oras cp --from-oci-layout "$PWD/output-${image_name}:latest" "${image_repo}:latest,${image_version}" - done diff --git a/README.md b/README.md index be7efa4..a5a87b4 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,12 @@ # Container images for DiracX -This repository contains the recipes and CI for building the base images used by DiracX. +> **Note:** As of diracx v0.0.12, diracx ships its own pixi-based +> runtime image (see `containers/Dockerfile` in the diracx repo). +> The `base`, `services-base`, and `client-base` images previously +> built here are no longer used and are not built by CI. +> Only `secret-generation` is maintained in this repository. -See [documentation](https://github.com/DIRACGrid/diracx/blob/main/docs/VERSIONING.md#container-images) +This repository contains the recipes and CI for building the container images used by DiracX. ## Available images @@ -10,14 +14,10 @@ See [documentation](https://github.com/DIRACGrid/diracx/blob/main/docs/VERSIONIN This image is used by the [helm chart](https://github.com/DIRACGrid/diracx-charts) to run batch jobs within the cluster that automatically generate kubernetes secrets. - - ## How to build The most up to date documentation on how to build is the [CI job](.github/workflows/main.yml) ```bash - -docker build -t ghcr.io/diracgrid/diracx/base:latest base -docker build -t ghcr.io/diracgrid/diracx/servces-base:latest services-base/ -``` \ No newline at end of file +docker build -t ghcr.io/diracgrid/diracx/secret-generation:latest secret-generation/ +``` diff --git a/base/Dockerfile b/base/Dockerfile deleted file mode 100644 index 3d7d92c..0000000 --- a/base/Dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -FROM mambaorg/micromamba:latest - -# Copying in ENTRYPOINT script and environment specification -COPY --chown=$MAMBA_USER:$MAMBA_USER environment.yml diraccommon_dependencies.yml /tmp/ -COPY --chown=$MAMBA_USER:$MAMBA_USER entrypoint.sh / -RUN chmod 755 /entrypoint.sh - -RUN micromamba install --freeze-installed --yes --file /tmp/environment.yml --name=base && \ - micromamba install --freeze-installed --yes --file /tmp/diraccommon_dependencies.yml --name=base && \ - micromamba clean --all --yes --force-pkgs-dirs && \ - rm -rf /tmp/environment.yml /tmp/diraccommon_dependencies.yml - -ARG MAMBA_DOCKERFILE_ACTIVATE=1 - -# In many clusters the container is ran as a random uid for security reasons. -# If we mark the conda directory as group 0 and give it group write permissions -# then we're still able to manage the environment from inside the container. -USER 0 -RUN chown -R $MAMBA_USER:0 /opt/conda && chmod -R g=u /opt/conda -USER $MAMBA_USER - -# We use _entrypoint.sh to be able to use the micromamba env in -# the entrypoint -# https://micromamba-docker.readthedocs.io/en/latest/quick_start.html#activating-a-conda-environment-for-entrypoint-commands -# Use tini to avoid zombie processes and properly handle signal forwarding -ENTRYPOINT [ "/usr/local/bin/_entrypoint.sh", "tini", "--", "/entrypoint.sh" ] - diff --git a/base/diraccommon_dependencies.yml b/base/diraccommon_dependencies.yml deleted file mode 100644 index 65b6c4e..0000000 --- a/base/diraccommon_dependencies.yml +++ /dev/null @@ -1,14 +0,0 @@ -# This yaml file contains the list of DIRAC dependencies that must be satisfied -# even if we won't need it. For example, gfal2 needs to be installed in order to -# be able to install DIRAC, even in the services. -# These dependencies are separated from the main environment.yml in the hope that -# we can one day remove entirely this file - -name: diracx -channels: - - conda-forge - - nodefaults -dependencies: - - typing-extensions >=4.0.0 - - diraccfg - - pydantic >=2.0.0 diff --git a/base/entrypoint.sh b/base/entrypoint.sh deleted file mode 100644 index 0f2f016..0000000 --- a/base/entrypoint.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/bash -set -e - -eval "$(micromamba shell hook --shell=posix)" -micromamba activate base - -function install_sources() { - extension_name=$1 - source_prefix=$2 - image_packages=$3 - - IFS=',' - to_install=() - for dir in ${!source_prefix}; do - for package_name in ${!image_packages}; do - if [[ "${package_name}" == "." ]]; then - wheel_name="${extension_name}" - else - wheel_name="${extension_name}_${package_name}" - fi - wheels=( $(find "${dir}" -name "${wheel_name}-*.whl") ) - if [[ ${#wheels[@]} -gt 1 ]]; then - echo "ERROR: Multiple wheels found for ${package_name} in ${dir}" - exit 1 - elif [[ ${#wheels[@]} -eq 1 ]]; then - to_install+=("${wheels[0]}") - else - if [[ "${package_name}" == "." ]]; then - src_dir=("${dir}") - else - src_dir=("${dir}-${package_name}") - fi - if [[ -f "${src_dir}/pyproject.toml" ]]; then - to_install+=("${src_dir}") - fi - fi - done - done - if [[ ${#to_install[@]} -gt 0 ]]; then - pip install --no-deps "${to_install[@]}" - fi -} - - -# If we have extensions, we install them all the same way -if [[ -n "${DIRACX_EXTENSIONS:-}" ]]; then - - # Loop over the extension in reverse order - IFS=', ' read -r -a extension_array <<< "$DIRACX_EXTENSIONS" - for (( idx=${#extension_array[@]}-1 ; idx>=0 ; idx-- )) ; do - - extension_name="${extension_array[idx]}" - source_prefix="${extension_name^^}_CUSTOM_SOURCE_PREFIXES" - image_packages="${extension_name^^}_IMAGE_PACKAGES" - - if [[ -n "${!source_prefix:-}" ]]; then - install_sources "${extension_name}" "${source_prefix}" "${image_packages}" - fi - done -# No extensions, just diracx -elif [[ -n "${DIRACX_CUSTOM_SOURCE_PREFIXES:-}" ]]; then - install_sources "diracx" "DIRACX_CUSTOM_SOURCE_PREFIXES" "DIRACX_IMAGE_PACKAGES" -fi - - -exec "$@" \ No newline at end of file diff --git a/base/environment.yml b/base/environment.yml deleted file mode 100644 index 48cda3b..0000000 --- a/base/environment.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: diracx -channels: - - diracgrid - - conda-forge - - nodefaults -dependencies: - - authlib - - aiobotocore - - botocore - - cachetools - # Needed because coverage runs inside the image - # when doing the integration tests - - coverage - - email-validator - - git - - gitpython - - httpx - - joserfc - - pip - - pydantic >=2 - - pydantic-settings - - python =3.11 - - pyyaml - - pyjwt - - requests - - urllib3 <2 - - sh - - tini - - zstandard \ No newline at end of file diff --git a/client-base/Dockerfile b/client-base/Dockerfile deleted file mode 100644 index f96a5d2..0000000 --- a/client-base/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM ghcr.io/diracgrid/diracx/base - -# Copying in ENTRYPOINT script and environment specification -COPY --chown=$MAMBA_USER:$MAMBA_USER environment.yml /tmp/ - -ENV DIRACX_IMAGE_PACKAGES=core,client,api,cli,. - -RUN micromamba install --freeze-installed --yes --file /tmp/environment.yml --name=base && \ - micromamba clean --all --yes --force-pkgs-dirs && \ - rm -rf /tmp/environment.yml - -# In many clusters the container is ran as a random uid for security reasons. -# If we mark the conda directory as group 0 and give it group write permissions -# then we're still able to manage the environment from inside the container. -USER 0 -RUN chown -R $MAMBA_USER:0 /opt/conda && chmod -R g=u /opt/conda -USER $MAMBA_USER diff --git a/client-base/environment.yml b/client-base/environment.yml deleted file mode 100644 index a5b09cf..0000000 --- a/client-base/environment.yml +++ /dev/null @@ -1,15 +0,0 @@ -name: diracx -channels: - - diracgrid - - conda-forge - - nodefaults -dependencies: - - aiohttp - - azure-core - - cachetools - - isodate - - python-dotenv - - python-multipart - - rich - - typer - - uuid-utils diff --git a/services-base/Dockerfile b/services-base/Dockerfile deleted file mode 100644 index 1a64623..0000000 --- a/services-base/Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -FROM ghcr.io/diracgrid/diracx/base -EXPOSE 8000 - -# Copying in ENTRYPOINT script and environment specification -COPY --chown=$MAMBA_USER:$MAMBA_USER environment.yml /tmp/ - -ENV DIRACX_IMAGE_PACKAGES=core,db,logic,routers - -RUN micromamba install --freeze-installed --yes --file /tmp/environment.yml --name=base && \ - micromamba clean --all --yes --force-pkgs-dirs && \ - rm -rf /tmp/environment.yml - -# In many clusters the container is ran as a random uid for security reasons. -# If we mark the conda directory as group 0 and give it group write permissions -# then we're still able to manage the environment from inside the container. -USER 0 -RUN chown -R $MAMBA_USER:0 /opt/conda && chmod -R g=u /opt/conda -USER $MAMBA_USER diff --git a/services-base/environment.yml b/services-base/environment.yml deleted file mode 100644 index 74a1323..0000000 --- a/services-base/environment.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: diracx -channels: - - diracgrid - - conda-forge - - nodefaults -dependencies: - - aiomysql - - aiosqlite - - authlib - - email-validator - - fastapi - - isodate - - joserfc - - opensearch-py - - pyjwt - - coverage - - python-dotenv - - python-jose - - python-multipart - - sqlalchemy - - uvicorn - - aiobotocore >=2.12 - - botocore - - ca-policy-lcg - - opentelemetry-api - - opentelemetry-exporter-otlp - - opentelemetry-instrumentation-fastapi - - opentelemetry-instrumentation-logging - - opentelemetry-sdk - - uuid-utils - - python-dateutil