ci: bump base image/runner to v26.4 and drop v26.3-only workarounds

.github/workflows/ci.yaml

@@ -25,10 +25,10 @@ permissions:
 env:
   PRIMUS_TURBO_COMMIT: edc8d2ccb0be4888e80ee7c6e765fd3956026a32 # feat: add quantized tensor support for blockwise grouped gemm (#422)
   PRIMUS_TURBO_AITER_COMMIT: 0f3c58e6edb6754940bcf9fd5f09ccb6f389f52e # AITER v0.1.14.post1 (tag commit) — required by Primus-Turbo main aiter_utils.py
-  ROCSHMEM_COMMIT: 17ff985c026f9f97f85068647e863ab541dd5645 # Update version to 3.2.0 for 7.2.0 rocm release (#351) (#355)
+  #ROCSHMEM_COMMIT: 17ff985c026f9f97f85068647e863ab541dd5645 # Update version to 3.2.0 for 7.2.0 rocm release (#351) (#355)
   UCCL_COMMIT: 5afb4117893c58cc0c8557d9286336141a301053 # [EP]: fix fp8 error of internode_ll on amd gfx950 arch. (#710)
   TRITON_COMMIT: 88b227e23f0445f3f695bad05bbf1a363b4f50e0
-  BASE_IMAGE: docker.io/rocm/primus:v26.3
+  BASE_IMAGE: docker.io/rocm/primus:v26.4
   MAXTEXT_BASE_IMAGE: docker.io/rocm/jax-training:maxtext-v26.2
 
 jobs:
@@ -123,16 +123,20 @@ jobs:
           echo "> Build Docker Image with tag: ${{ env.IMAGE_TAG }}"
           echo "> Build dependencies"
           start_time=$(date +%s)
+          # Optional build-args, off by default (kept as an array so a commented-out
+          # element here can't break the docker build \-continuation below).
+          EXTRA_BUILD_ARGS=()
+          # EXTRA_BUILD_ARGS+=(--build-arg ROCSHMEM_COMMIT=${ROCSHMEM_COMMIT})
           docker build -f $CWS/.github/workflows/docker/Dockerfile \
                --network=host \
                -t tasimage/primus:${{env.IMAGE_TAG}} \
                --build-arg BASE_IMAGE=${BASE_IMAGE} \
                --build-arg PRIMUS_TURBO_COMMIT=${PRIMUS_TURBO_COMMIT} \
                --build-arg PRIMUS_TURBO_AITER_COMMIT=${PRIMUS_TURBO_AITER_COMMIT} \
-               --build-arg ROCSHMEM_COMMIT=${ROCSHMEM_COMMIT} \
                --build-arg PRIMUS_TURBO_FRAMEWORK=PYTORCH \
                --build-arg UCCL_COMMIT=${UCCL_COMMIT} \
                --build-arg TRITON_COMMIT=${TRITON_COMMIT} \
+               "${EXTRA_BUILD_ARGS[@]}" \
                $CWS/.github/workflows/docker
           end_time=$(date +%s)
           elapsed=$((end_time - start_time))
@@ -143,7 +147,7 @@ jobs:
           docker push docker.io/tasimage/primus:${{env.IMAGE_TAG}}
           # docker login -u rocmshared -p ${{ secrets.ROCM_DOCKER_HUB_TOKEN }}
 
-          # Primus v26.3 already includes AINIC under /workspace. Re-enable this
+          # Primus v26.4 already includes AINIC under /workspace. Re-enable this
           # Dockerfile.ainic build only when we need to refresh the tasimage -ainic image.
           echo "> Build Docker Image with tag: ${{ env.IMAGE_TAG }}-ainic"
           start_time=$(date +%s)
@@ -304,16 +308,18 @@ jobs:
           : > "$RUNNER_TEMP/runtime.tsv"  # reset the CI runtime log for this job
           IMG="docker.io/tasimage/primus:${{ env.IMAGE_TAG }}"
           echo "Target image: ${IMG}"
-          # Reuse the container only if it is already running on the same image;
-          # otherwise recreate it from the freshly pushed image.
-          cur_img="$(docker inspect -f '{{.Config.Image}}' "${UT_CONTAINER}" 2>/dev/null || true)"
+          # Pull first, then compare image id (not tag name): IMAGE_TAG is
+          # constant ("latest") on main, so a tag-only check can't detect a
+          # same-tag repush and would keep reusing a stale container.
+          docker pull "${IMG}"
+          target_id="$(docker image inspect -f '{{.Id}}' "${IMG}" 2>/dev/null || true)"
+          cur_id="$(docker inspect -f '{{.Image}}' "${UT_CONTAINER}" 2>/dev/null || true)"
           running="$(docker inspect -f '{{.State.Running}}' "${UT_CONTAINER}" 2>/dev/null || true)"
-          if [ "${running}" = "true" ] && [ "${cur_img}" = "${IMG}" ]; then
-            echo "Reusing container ${UT_CONTAINER} (already on ${IMG})."
+          if [ "${running}" = "true" ] && [ -n "${target_id}" ] && [ "${cur_id}" = "${target_id}" ]; then
+            echo "Reusing container ${UT_CONTAINER} (already on ${IMG}, id=${target_id})."
           else
-            echo "Recreating container ${UT_CONTAINER} (running=${running}, image=${cur_img})."
+            echo "Recreating container ${UT_CONTAINER} (running=${running}, cur_id=${cur_id}, target_id=${target_id})."
             docker rm -f "${UT_CONTAINER}" >/dev/null 2>&1 || true
-            docker pull "${IMG}"
             docker run -d --name "${UT_CONTAINER}" \
               --network host --ipc host --privileged \
               --group-add video \
@@ -333,36 +339,8 @@ jobs:
           set -e
           pip install -r requirements.txt
           IN
-      # v26.3-only: drop this whole step on v26.4. Unit tests import torchtitan
-      # from the submodule source via conftest sys.path injection (v0.2.2 has
-      # PEP-420 namespace packages that need the source root on sys.path). This
-      # just removes any stale torchtitan in the image that would shadow it.
-      - name: Purge stale TorchTitan install (v26.3-only, drop on v26.4)
-        run: |
-          docker exec -i -e GITHUB_WORKSPACE="${CWS}" -w "${CWS}" "${UT_CONTAINER}" bash -s <<'IN'
-          set -e
-          pip uninstall -y torchtitan || true
-          python - <<'PY'
-          import glob, os, shutil, site
-          dirs = set(site.getsitepackages() + [site.getusersitepackages()])
-          patterns = (
-              "__editable__*torchtitan*",
-              "__editable__.torchtitan*.pth",
-              "torchtitan*.pth",
-              "torchtitan*.egg-link",
-              "torchtitan*.dist-info",
-              "torchtitan*.egg-info",
-          )
-          for d in dirs:
-              for pat in patterns:
-                  for f in glob.glob(os.path.join(d, pat)):
-                      try:
-                          shutil.rmtree(f) if os.path.isdir(f) else os.remove(f)
-                          print("removed", f)
-                      except OSError as e:
-                          print("skip", f, e)
-          PY
-          IN
+      # No origami in this image (see docker/Dockerfile); primus_turbo falls
+      # back to heuristic kernel selection for MoE grouped-gemm.
       - name: Set UT_LOG_PATH
         run: |
           ts="$(date +%Y%m%d-%H%M%S)"

.github/workflows/docker/Dockerfile

@@ -1,10 +1,10 @@
-ARG BASE_IMAGE=docker.io/rocm/primus:v26.3
+ARG BASE_IMAGE=docker.io/rocm/primus:v26.4
 FROM ${BASE_IMAGE}
 
 ARG PRIMUS_TURBO_COMMIT
 ARG PRIMUS_TURBO_AITER_COMMIT
 ARG PRIMUS_TURBO_FRAMEWORK
-ARG ROCSHMEM_COMMIT
+# ARG ROCSHMEM_COMMIT # only needed if the rocSHMEM rebuild below is restored
 ARG UCCL_COMMIT
 ARG TRITON_COMMIT
 # Non-interactive APT
@@ -21,34 +21,25 @@ RUN rm -rf /var/lib/apt/lists/*
 # ---------------------------------------------------------------------------
 # Enviroment variables
 # ---------------------------------------------------------------------------
-ENV ROCSHMEM_HOME=/opt/rocshmem
-ENV UCX_HOME=/opt/ucx
-# ENV MPI_HOME=/opt/ompi
-# Use the system OpenMPI prefix from the v26.3 base image.
-ENV MPI_HOME=/usr/lib/x86_64-linux-gnu/openmpi
-ENV ROCM_HOME=/opt/rocm
 ENV PRIMUS_TURBO_FRAMEWORK=${PRIMUS_TURBO_FRAMEWORK}
 
-# ENV PATH="/opt/ompi/bin:/opt/ompi/sbin:${PATH}"
-# ENV LD_LIBRARY_PATH="/opt/ompi/lib:${LD_LIBRARY_PATH}"
 ENV GPU_ARCHS="gfx942;gfx950"
 ENV PYTORCH_ROCM_ARCH="gfx942;gfx950"
 ENV HCC_AMDGPU_TARGET="gfx942,gfx950"
-# ---------------------------------------------------------------------------
-# Install rocSHMEM
-# ---------------------------------------------------------------------------
-RUN mkdir -p /opt && cd /opt && \
-    git clone https://github.com/ROCm/rocSHMEM.git && \
-    cd rocSHMEM && \
-    git checkout ${ROCSHMEM_COMMIT} && \
-    mkdir build && \
-    cd build && \
-    MPI_ROOT=${MPI_HOME} UCX_ROOT=${UCX_HOME} INSTALL_PREFIX=${ROCSHMEM_HOME} ../scripts/build_configs/gda \
-                                                                                            -DGDA_IONIC=ON \
-                                                                                            -DGDA_MLX5=ON  \
-                                                                                            -DGDA_BNXT=ON  \
-                                                                                            -DUSE_IPC=ON
-RUN rm -rf /opt/rocSHMEM
+
+# rocSHMEM: used by Primus ODC via Primus-Turbo's odc_rocshmem_* extensions.
+# This base image already ships a prebuilt install (commit 17ff985c, version
+# 3.2.0) at $ROCSHMEM_HOME=/opt/rocshmem, so building it again is redundant.
+#
+# If a future base image drops it, restore by uncommenting below (and the
+# `ARG ROCSHMEM_COMMIT` above). Note: cmake reads the ROCm version from
+# -DROCM_PATH=, not $ROCM_PATH; omitting it falls back to the nonexistent
+# /opt/rocm and fails.
+# RUN mkdir -p /opt && cd /opt && git clone https://github.com/ROCm/rocSHMEM.git && \
+#     cd rocSHMEM && git checkout ${ROCSHMEM_COMMIT} && mkdir build && cd build && \
+#     MPI_ROOT=${MPI_HOME} UCX_ROOT=${UCX_HOME} INSTALL_PREFIX=${ROCSHMEM_HOME} \
+#     ../scripts/build_configs/gda -DROCM_PATH=${ROCM_PATH} \
+#       -DGDA_IONIC=ON -DGDA_MLX5=ON -DGDA_BNXT=ON -DUSE_IPC=ON
 
 # ---------------------------------------------------------------------------
 # Install Primus-Turbo
@@ -100,23 +91,9 @@ RUN if [ "$PRIMUS_TURBO_FRAMEWORK" != "JAX" ]; then \
     rm -rf /opt/uccl; \
     fi
 
-# ---------------------------------------------------------------------------
-# Install fixed origami (rocm-libraries@223648a) over the base image's bundled
-# 0.1.0. The bundled origami's rank_configs() raises
-# `ValueError: vector::reserve` during MoE grouped-gemm kernel selection and
-# crashes training (turbo's _safe_rank_configs only catches RuntimeError).
-# Skipped for JAX. TODO: drop once a base image ships origami with the fix.
-RUN if [ "$PRIMUS_TURBO_FRAMEWORK" != "JAX" ]; then \
-    rm -rf /tmp/rocm-libraries && \
-    git clone --filter=blob:none --no-checkout https://github.com/ROCm/rocm-libraries.git /tmp/rocm-libraries && \
-    cd /tmp/rocm-libraries && \
-    git sparse-checkout init --cone && \
-    git sparse-checkout set shared/origami && \
-    git checkout 223648a26928ebed7f3dd0ccdc044c09f1dccf9b && \
-    (pip uninstall -y origami || true) && \
-    pip install --no-cache-dir ./shared/origami/python && \
-    rm -rf /tmp/rocm-libraries; \
-    fi
+# No origami in this image; primus_turbo falls back to heuristic kernel
+# selection for MoE grouped-gemm. To add it back:
+#   pip install "git+https://github.com/ROCm/rocm-libraries.git@223648a26928ebed7f3dd0ccdc044c09f1dccf9b#subdirectory=shared/origami/python"
 
 # Set the default working directory
 WORKDIR /opt

README.md

@@ -91,7 +91,7 @@ primus-cli deps sync --dir ~/.cache/Primus/third_party
 
     ```bash
     # For Megatron-LM and TorchTitan backends
-    docker pull rocm/primus:v26.3
+    docker pull rocm/primus:v26.4
     # For MaxText backend
     docker pull rocm/jax-training:maxtext-v26.4-jax0.9.1-te2.12.0
     ```
@@ -102,7 +102,7 @@ primus-cli deps sync --dir ~/.cache/Primus/third_party
     git clone --recurse-submodules https://github.com/AMD-AGI/Primus.git
     cd Primus
     # checkout the branch for the specific release
-    git checkout release/v26.3
+    git checkout release/v26.4
     git submodule update --init --recursive
     ```
 
@@ -113,7 +113,7 @@ primus-cli deps sync --dir ~/.cache/Primus/third_party
     # NOTE: If your config downloads weights/tokenizer from Hugging Face Hub,
     #       you typically need to pass HF_TOKEN into the container.
     # Run in the Primus repository root directory
-    ./primus-cli container --image rocm/primus:v26.3 \
+    ./primus-cli container --image rocm/primus:v26.4 \
       --env HF_TOKEN="hf_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" \
       -- train pretrain --config examples/megatron/configs/MI300X/llama2_7B-BF16-pretrain.yaml
     ```
@@ -131,7 +131,7 @@ For more detailed usage instructions, see the [CLI User Guide](./docs/02-user-gu
     python -m venv primus-env
     source primus-env/bin/activate
     # Install Primus
-    pip install "primus==26.3.1" --no-deps --extra-index-url https://amd-agi.github.io/Primus/simple/
+    pip install "primus==26.4.0" --no-deps --extra-index-url https://amd-agi.github.io/Primus/simple/
 
     ```
 
@@ -142,7 +142,7 @@ For more detailed usage instructions, see the [CLI User Guide](./docs/02-user-gu
 2. **Run training in container using pip-installed Primus**
 
     ```bash
-    primus-cli container --image rocm/primus:v26.3 \
+    primus-cli container --image rocm/primus:v26.4 \
     --env HF_TOKEN="hf_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" \
     --volume /path/to/your/data:/data  -- --log_file /data/run.log \
     -- train pretrain --config /data/your/config.yaml

benchmark/kernel/rccl/run_slurm.sh

@@ -10,7 +10,7 @@
 # Usage:
 #   DOCKER_IMAGE=<image> sbatch run_slurm.sh
 #   DOCKER_IMAGE=<image> NNODES=2 PARTITION=my-gpu sbatch run_slurm.sh
-#   DOCKER_IMAGE=rocm/primus:v26.3 NNODES=2 sbatch -N2 -w smci355-ccs-aus-n04-[25,29] -p Compute-DCPT ./run_slurm.sh
+#   DOCKER_IMAGE=rocm/primus:v26.4 NNODES=2 sbatch -N2 -w smci355-ccs-aus-n04-[25,29] -p Compute-DCPT ./run_slurm.sh
 #
 # Environment variables (all optional except DOCKER_IMAGE):
 #   DOCKER_IMAGE        Docker image to use (required)

benchmark/kernel/rccl/submit_pairs.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-DOCKER_IMAGE="rocm/primus:v26.3"
+DOCKER_IMAGE="rocm/primus:v26.4"
 #DOCKER_IMAGE="docker.gpuperf:5000/aai_2026_training/rocm/primus_megatron:v25.11_gpt_oss_sink"
 #DOCKER_IMAGE="docker.gpuperf:5000/gpuperf/primus:v26.1_sinkfa"
 NNODES=2

docs/01-getting-started/quickstart.md

@@ -65,7 +65,7 @@ Install Primus in a virtual environment:
 ```bash
 python -m venv primus-env
 source primus-env/bin/activate
-pip install "primus==26.3.1" --no-deps --extra-index-url https://amd-agi.github.io/Primus/simple/
+pip install "primus==26.4.0" --no-deps --extra-index-url https://amd-agi.github.io/Primus/simple/
 ```
 
 > **Note:** This installs only the Primus CLI into your virtual environment (under `site-packages`), without other dependencies. Third-party submodules are downloaded on the first run of the container, and the complete training software stack is provided in the AMD-published Docker images. You can launch `primus-cli` from any directory.

examples/README.md

@@ -49,7 +49,7 @@ We recommend using the official [rocm/megatron-lm Docker image](https://hub.dock
 
 ```bash
 # Pull the latest Docker image
-docker pull docker.io/rocm/primus:v26.3
+docker pull docker.io/rocm/primus:v26.4
 
 ```
 
@@ -126,7 +126,7 @@ Multi-node training is launched via **SLURM**.
 Specify the number of nodes and the model config:
 
 ```bash
-export DOCKER_IMAGE="docker.io/rocm/primus:v26.3"
+export DOCKER_IMAGE="docker.io/rocm/primus:v26.4"
 export NNODES=8
 
 # Example for megatron llama3.1_8B
@@ -292,7 +292,7 @@ When using the `create` command to start a new training workload, the following
 | `--gpu`        | Number of GPUs                                       | 8                                        |
 | `--exp`        | Path to experiment (training config) file (required) | —                                        |
 | `--data_path`  | Path to training data                                | —                                        |
-| `--image`      | Docker image to use                                  | `docker.io/rocm/primus:v26.3` |
+| `--image`      | Docker image to use                                  | `docker.io/rocm/primus:v26.4` |
 | `--hf_token`   | HuggingFace token                                    | Read from env var `HF_TOKEN`             |
 | `--workspace`  | Workspace name                                       | `primus-safe-pretrain`                   |
 | `--nodelist`   | Comma-separated list of node hostnames to run on     | —                                        |

examples/run_k8s_pretrain.sh

@@ -15,7 +15,7 @@ GPU="8"
 EXP_PATH=""
 DATA_PATH=""
 BACKEND="megatron"
-IMAGE="docker.io/rocm/primus:v26.3"
+IMAGE="docker.io/rocm/primus:v26.4"
 HF_TOKEN="${HF_TOKEN:-}"
 WORKSPACE="primus-safe-pretrain"
 NODELIST=""
@@ -38,7 +38,7 @@ Options for create:
     --backend <name>            Training backend, e.g. megatron | torchtitan(default: megatron)
     --exp <exp_path>            Path to EXP config (optional)
     --data_path <data_path>     Data path (optional)
-    --image <docker_image>      Docker image to use (default: docker.io/rocm/primus:v26.3)
+    --image <docker_image>      Docker image to use (default: docker.io/rocm/primus:v26.4)
     --hf_token <token>          HuggingFace token (default: from env HF_TOKEN)
     --workspace <workspace>     Workspace name (default: safe-cluster-dev)
     --nodelist <node1,node2>    Comma-separated list of node names to run on (optional)

examples/run_local_pretrain.sh

@@ -16,7 +16,7 @@ Usage: bash run_local_pretrain.sh
 This script launches a Primus pretraining task inside a Docker/Podman container.
 
 Environment Variables:
-    DOCKER_IMAGE   Docker image to use [Default: docker.io/rocm/primus:v26.3]
+    DOCKER_IMAGE   Docker image to use [Default: docker.io/rocm/primus:v26.4]
     MASTER_ADDR    Master node IP or hostname [Default: localhost]
     MASTER_PORT    Master node port [Default: 1234]
     NNODES         Total number of nodes [Default: 1]
@@ -44,7 +44,7 @@ export EXP=${EXP:-"examples/megatron/exp_pretrain.yaml"}
 if [ "${BACKEND:-}" = "MaxText" ]; then
     DOCKER_IMAGE=${DOCKER_IMAGE:-"docker.io/rocm/jax-training:maxtext-v26.2"}
 else
-    DOCKER_IMAGE=${DOCKER_IMAGE:-"docker.io/rocm/primus:v26.3"}
+    DOCKER_IMAGE=${DOCKER_IMAGE:-"docker.io/rocm/primus:v26.4"}
 fi
 
 # Project root

examples/run_local_pretrain_cli.sh

@@ -15,7 +15,7 @@ EXP=${EXP:-"examples/megatron/exp_pretrain.yaml"}
 if [ "${BACKEND:-}" = "MaxText" ]; then
     DOCKER_IMAGE=${DOCKER_IMAGE:-"docker.io/rocm/jax-training:maxtext-v25.9"}
 else
-    DOCKER_IMAGE=${DOCKER_IMAGE:-"docker.io/rocm/primus:v26.3"}
+    DOCKER_IMAGE=${DOCKER_IMAGE:-"docker.io/rocm/primus:v26.4"}
 fi
 
 # ------------------ Cluster Env Defaults ------------------

examples/run_slurm_pretrain_cli.sh

@@ -35,7 +35,7 @@ mkdir -p "$LOG_DIR"
 # NOTE: The --env entries below are passed into the container and will be visible
 # to the Primus training process (and system hooks) inside the container.
 bash "$PRIMUS_PATH/runner/primus-cli" slurm "${SLURM_ARGS[@]}" \
--- --image "${DOCKER_IMAGE:-rocm/primus:v26.3}" \
+-- --image "${DOCKER_IMAGE:-rocm/primus:v26.4}" \
 -- \
   --env "USING_AINIC=${USING_AINIC:-0}" \
   --env "PATCH_TE_FLASH_ATTN=${PATCH_TE_FLASH_ATTN:-0}" \