From 0409a58b9a1e27c32505912bc7514be8af6c4bee Mon Sep 17 00:00:00 2001 From: Alex Krochak Date: Mon, 11 Aug 2025 15:08:50 +0200 Subject: [PATCH 1/6] Initial set-up for the runner --- .github/workflows/juwels-runner.yml | 58 +++++++++++++++++++++++++++++ pytest-runner/kill_runner.sh | 22 +++++++++++ pytest-runner/monitor_runner.sh | 32 ++++++++++++++++ pytest-runner/start_runner.sh | 29 +++++++++++++++ pytest.sh | 23 ++++++++++++ 5 files changed, 164 insertions(+) create mode 100644 .github/workflows/juwels-runner.yml create mode 100755 pytest-runner/kill_runner.sh create mode 100755 pytest-runner/monitor_runner.sh create mode 100755 pytest-runner/start_runner.sh create mode 100755 pytest.sh diff --git a/.github/workflows/juwels-runner.yml b/.github/workflows/juwels-runner.yml new file mode 100644 index 000000000..e6b612c49 --- /dev/null +++ b/.github/workflows/juwels-runner.yml @@ -0,0 +1,58 @@ +name: Test Juwels Runner + +on: + push: + branches: + - 'main' + +jobs: + run-command: + runs-on: [self-hosted, linux] # adjust labels if you used custom ones + env: + GITHUB_WORKSPACE: /p/project1/intertwin/krochak1/itwinai + SLURM_SCRIPT: /p/project1/intertwin/krochak1/itwinai/test.sh + SLURM_LOGS: /p/project1/intertwin/krochak1/itwinai/pytest-runner/actions-runner-logs + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Submit and monitor SLURM job + run: | + + cd $GITHUB_WORKSPACE || exit + + echo "Submitting job..." + JOBID=$(sbatch --parsable "$SLURM_SCRIPT") + echo "Submitted as $JOBID" + echo "JOBID=$JOBID" >> $GITHUB_ENV + + echo "Waiting for job to complete..." + while squeue -j "$JOBID" 2>/dev/null | grep -q "$JOBID"; do + sleep 10 + done + + echo "Job $JOBID appears to have finished." + + echo "Querying SLURM exit code..." + EXIT_CODE=$(sacct -j "${JOBID}.batch" --format=ExitCode -n | awk '{print $1}' | cut -d':' -f1) + + if [[ -z "$EXIT_CODE" ]]; then + echo "ERROR: Could not retrieve exit code from sacct." + exit 1 + fi + + if [[ "$EXIT_CODE" -ne 0 ]]; then + echo "SLURM job failed with exit code $EXIT_CODE" + exit 1 + else + echo "SLURM job completed successfully with exit code 0" + fi + + - name: Read SLURM logs + run: | + echo "=== SLURM STDOUT ===" + cat "${SLURM_LOGS}/output.${JOBID}.out" || echo "No output file found." + + echo "=== SLURM STDERR ===" + cat "${SLURM_LOGS}/error.${JOBID}.out" || echo "No error file found." \ No newline at end of file diff --git a/pytest-runner/kill_runner.sh b/pytest-runner/kill_runner.sh new file mode 100755 index 000000000..8b65851e5 --- /dev/null +++ b/pytest-runner/kill_runner.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +stop_pid() { + local name=$1 + local file=$2 + + if [ -f "$file" ]; then + local pid + pid=$(cat "$file") + if ps -p "$pid" > /dev/null 2>&1; then + kill "$pid" && echo "Stopped $name (PID: $pid)" + else + echo "$name not running." + fi + rm -f "$file" + else + echo "No $name PID file found." + fi +} + +stop_pid "Runner.Listener" "runner-listener.pid" +stop_pid "helper script" "runner-helper.pid" \ No newline at end of file diff --git a/pytest-runner/monitor_runner.sh b/pytest-runner/monitor_runner.sh new file mode 100755 index 000000000..55223555d --- /dev/null +++ b/pytest-runner/monitor_runner.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +check_pid() { + local name=$1 + local file=$2 + + if [ -f "$file" ]; then + local pid + pid=$(cat "$file") + if ps -p "$pid" > /dev/null 2>&1; then + echo "$name is running (PID: $pid)" + return 0 + else + echo "$name not running." + return 1 + fi + else + echo "No $name PID file found." + return 1 + fi +} + +# Check for the helper script existence +check_pid "Helper script" "runner-helper.pid" +helper_status=$? + +# Check for the listener script existence +check_pid "Runner.Listener" "runner-listener.pid" +listener_status=$? + +# Return exit status +exit $(( helper_status != 0 || listener_status != 0 )) \ No newline at end of file diff --git a/pytest-runner/start_runner.sh b/pytest-runner/start_runner.sh new file mode 100755 index 000000000..d0337e822 --- /dev/null +++ b/pytest-runner/start_runner.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +LOGFILE="log" +RUNNER_DIR="/p/project1/intertwin/krochak1/actions-runner" +HELPER="$RUNNER_DIR/run.sh" + +# launch the runner, get PID and write it file +rm -f "$LOGFILE" +nohup "$HELPER" > "$LOGFILE" 2>&1 & +HELPER_PID=$! +echo $HELPER_PID > runner-helper.pid +disown + +# Wait briefly to allow Runner.Listener to start +sleep 2 + +# Get the PID of Runner.Listener via pgrep +LISTENER_PID=$(pgrep -f "$RUNNER_DIR/bin/Runner.Listener run") +if [ -n "$LISTENER_PID" ]; then + echo "$LISTENER_PID" > runner-listener.pid +else + echo "ERROR: Runner.Listener process not found." + rm -f runner-helper.pid + exit 1 +fi + +echo "Runner started." +echo "Helper PID: $HELPER_PID" +echo "Listener PID: $LISTENER_PID" \ No newline at end of file diff --git a/pytest.sh b/pytest.sh new file mode 100755 index 000000000..0d14cde29 --- /dev/null +++ b/pytest.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +## general configuration of the job +#SBATCH --job-name=itwinai-radio-astronomy +#SBATCH --account=slfse +#SBATCH --partition=devel +#SBATCH --output=/p/project1/intertwin/krochak1/pulsar-plugin/pytest-runner/actions-runner-logs/output.%j.out +#SBATCH --error=/p/project1/intertwin/krochak1/pulsar-plugin/pytest-runner/actions-runner-logs/error.%j.out +#SBATCH --time=00:10:00 +#SBATCH --nodes=1 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=48 + +ml --force purge +ml Stages/2024 GCC OpenMPI CUDA/12 cuDNN MPI-settings/CUDA +ml Python CMake HDF5 PnetCDF libaio mpi4py +ml X11/20230603 OpenGL/2023a + +#shellcheck disable=SC1091 +source /p/project1/intertwin/krochak1/pulsar-plugin/.venv-juwels-cluster/bin/activate + +cd /p/project1/intertwin/krochak1/pulsar-plugin || exit +pytest tests \ No newline at end of file From 7c5e78a6659335369b8f9ecb1c73debccabd6232 Mon Sep 17 00:00:00 2001 From: Alex Krochak Date: Mon, 18 Aug 2025 16:16:29 +0200 Subject: [PATCH 2/6] update configuration, move runner scripts to main intertwin directory --- .github/workflows/pytest.yml | 84 +++++++++++++++++++++------------ pytest-runner/kill_runner.sh | 22 --------- pytest-runner/monitor_runner.sh | 32 ------------- pytest-runner/start_runner.sh | 29 ------------ pytest.sh | 23 --------- 5 files changed, 55 insertions(+), 135 deletions(-) delete mode 100755 pytest-runner/kill_runner.sh delete mode 100755 pytest-runner/monitor_runner.sh delete mode 100755 pytest-runner/start_runner.sh delete mode 100755 pytest.sh diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 3bd1f31f3..e3d45d109 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -3,8 +3,61 @@ name: Testing with pytest on: pull_request: branches: [main, dev] - + push: + branches: [juwels-runner] jobs: + test-on-juwels: + name: Testing with Pytest on self-hosted runner on JUWELS Booster + if: github.ref == 'refs/heads/juwels-runner' + runs-on: [self-hosted, linux] + env: + GITHUB_WORKSPACE: /p/project1/intertwin/krochak1/itwinai + SLURM_SCRIPT: /p/project1/intertwin/juwels-selfhosted-runner/RUNNER/batch-pytest.sh + SLURM_LOGS: /p/project1/intertwin/juwels-selfhosted-runner/RUNNER/slurm-logs + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Submit and monitor SLURM job + run: | + cd $GITHUB_WORKSPACE || exit + + echo "Submitting job..." + JOBID=$(sbatch --parsable "$SLURM_SCRIPT") + echo "Submitted as $JOBID" + echo "JOBID=$JOBID" >> $GITHUB_ENV + + echo "Waiting for job to complete..." + while squeue -j "$JOBID" 2>/dev/null | grep -q "$JOBID"; do + sleep 10 + done + + echo "Job $JOBID appears to have finished." + + echo "Querying SLURM exit code..." + EXIT_CODE=$(sacct -j "${JOBID}.batch" --format=ExitCode -n | awk '{print $1}' | cut -d':' -f1) + + if [[ -z "$EXIT_CODE" ]]; then + echo "ERROR: Could not retrieve exit code from sacct." + exit 1 + fi + + if [[ "$EXIT_CODE" -ne 0 ]]; then + echo "SLURM job failed with exit code $EXIT_CODE" + exit 1 + else + echo "SLURM job completed successfully with exit code 0" + fi + + - name: Read SLURM logs + run: | + echo "=== SLURM STDOUT ===" + cat "${SLURM_LOGS}/output.${JOBID}.out" || echo "No output file found." + + echo "=== SLURM STDERR ===" + cat "${SLURM_LOGS}/error.${JOBID}.out" || echo "No error file found." + test-torch: name: Testing with pytest runs-on: ubuntu-latest @@ -25,7 +78,7 @@ jobs: - uses: actions/checkout@v4 - name: Move Docker directory - shell: bash + shell: bash run: | sudo mv /var/lib/docker /docker/ && sudo ln -s /docker/docker /var/lib/docker && @@ -47,30 +100,3 @@ jobs: cloud-token: ${{ secrets.DAGGER_CLOUD_TOKEN }} version: latest - # test-torch: - # name: Testing with pytest - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v4 - - # - name: Set up Python 3.11.9 - # uses: actions/setup-python@v5 - # with: - # python-version: 3.11.9 - - # - name: Make PyTorch virtualenv - # shell: bash -l {0} - # run: make torch-env-cpu - - # # Comment this back in to also build tensorflow env - # # - name: Make Tensorflow virtualenv - # # shell: bash -l {0} - # # run: make tensorflow-env-cpu - - # # NOTE, to change the name of the env in which tests are run, set custom TORCH_ENV - # # and TF_ENV env variables. Default environment names are ".venv-pytorch" and - # # ".venv-tf" - - # - name: Run pytest for workflows - # shell: bash -l {0} - # run: .venv-pytorch/bin/pytest -v -n logical ./tests/ -m "not hpc and not tensorflow" diff --git a/pytest-runner/kill_runner.sh b/pytest-runner/kill_runner.sh deleted file mode 100755 index 8b65851e5..000000000 --- a/pytest-runner/kill_runner.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -stop_pid() { - local name=$1 - local file=$2 - - if [ -f "$file" ]; then - local pid - pid=$(cat "$file") - if ps -p "$pid" > /dev/null 2>&1; then - kill "$pid" && echo "Stopped $name (PID: $pid)" - else - echo "$name not running." - fi - rm -f "$file" - else - echo "No $name PID file found." - fi -} - -stop_pid "Runner.Listener" "runner-listener.pid" -stop_pid "helper script" "runner-helper.pid" \ No newline at end of file diff --git a/pytest-runner/monitor_runner.sh b/pytest-runner/monitor_runner.sh deleted file mode 100755 index 55223555d..000000000 --- a/pytest-runner/monitor_runner.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -check_pid() { - local name=$1 - local file=$2 - - if [ -f "$file" ]; then - local pid - pid=$(cat "$file") - if ps -p "$pid" > /dev/null 2>&1; then - echo "$name is running (PID: $pid)" - return 0 - else - echo "$name not running." - return 1 - fi - else - echo "No $name PID file found." - return 1 - fi -} - -# Check for the helper script existence -check_pid "Helper script" "runner-helper.pid" -helper_status=$? - -# Check for the listener script existence -check_pid "Runner.Listener" "runner-listener.pid" -listener_status=$? - -# Return exit status -exit $(( helper_status != 0 || listener_status != 0 )) \ No newline at end of file diff --git a/pytest-runner/start_runner.sh b/pytest-runner/start_runner.sh deleted file mode 100755 index d0337e822..000000000 --- a/pytest-runner/start_runner.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -LOGFILE="log" -RUNNER_DIR="/p/project1/intertwin/krochak1/actions-runner" -HELPER="$RUNNER_DIR/run.sh" - -# launch the runner, get PID and write it file -rm -f "$LOGFILE" -nohup "$HELPER" > "$LOGFILE" 2>&1 & -HELPER_PID=$! -echo $HELPER_PID > runner-helper.pid -disown - -# Wait briefly to allow Runner.Listener to start -sleep 2 - -# Get the PID of Runner.Listener via pgrep -LISTENER_PID=$(pgrep -f "$RUNNER_DIR/bin/Runner.Listener run") -if [ -n "$LISTENER_PID" ]; then - echo "$LISTENER_PID" > runner-listener.pid -else - echo "ERROR: Runner.Listener process not found." - rm -f runner-helper.pid - exit 1 -fi - -echo "Runner started." -echo "Helper PID: $HELPER_PID" -echo "Listener PID: $LISTENER_PID" \ No newline at end of file diff --git a/pytest.sh b/pytest.sh deleted file mode 100755 index 0d14cde29..000000000 --- a/pytest.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -## general configuration of the job -#SBATCH --job-name=itwinai-radio-astronomy -#SBATCH --account=slfse -#SBATCH --partition=devel -#SBATCH --output=/p/project1/intertwin/krochak1/pulsar-plugin/pytest-runner/actions-runner-logs/output.%j.out -#SBATCH --error=/p/project1/intertwin/krochak1/pulsar-plugin/pytest-runner/actions-runner-logs/error.%j.out -#SBATCH --time=00:10:00 -#SBATCH --nodes=1 -#SBATCH --ntasks-per-node=1 -#SBATCH --cpus-per-task=48 - -ml --force purge -ml Stages/2024 GCC OpenMPI CUDA/12 cuDNN MPI-settings/CUDA -ml Python CMake HDF5 PnetCDF libaio mpi4py -ml X11/20230603 OpenGL/2023a - -#shellcheck disable=SC1091 -source /p/project1/intertwin/krochak1/pulsar-plugin/.venv-juwels-cluster/bin/activate - -cd /p/project1/intertwin/krochak1/pulsar-plugin || exit -pytest tests \ No newline at end of file From 53de1ce2e9147b02510b7c973a5a79b3845b6b8d Mon Sep 17 00:00:00 2001 From: Alex Krochak Date: Tue, 2 Sep 2025 14:34:31 +0200 Subject: [PATCH 3/6] updated runner configuration --- .github/workflows/juwels-runner.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/juwels-runner.yml b/.github/workflows/juwels-runner.yml index e6b612c49..1fd8109bc 100644 --- a/.github/workflows/juwels-runner.yml +++ b/.github/workflows/juwels-runner.yml @@ -1,17 +1,17 @@ -name: Test Juwels Runner +name: Pytest execution on JUWELS Booster via Apptainer on: push: branches: - - 'main' + - 'juwels-runner' jobs: run-command: runs-on: [self-hosted, linux] # adjust labels if you used custom ones env: GITHUB_WORKSPACE: /p/project1/intertwin/krochak1/itwinai - SLURM_SCRIPT: /p/project1/intertwin/krochak1/itwinai/test.sh - SLURM_LOGS: /p/project1/intertwin/krochak1/itwinai/pytest-runner/actions-runner-logs + SLURM_SCRIPT: /p/project1/intertwin/juwels-selfhosted-runner/RUNNER/container.sh + SLURM_LOGS: /p/project1/intertwin/juwels-selfhosted-runner/RUNNER/slurm-logs steps: - name: Checkout code From 41ada188a5de13b3e182bb8176c5ec8284cf891f Mon Sep 17 00:00:00 2001 From: Alex Krochak Date: Tue, 2 Sep 2025 14:56:31 +0200 Subject: [PATCH 4/6] linter --- .github/workflows/juwels-runner.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/juwels-runner.yml b/.github/workflows/juwels-runner.yml index 1fd8109bc..aa77d67fa 100644 --- a/.github/workflows/juwels-runner.yml +++ b/.github/workflows/juwels-runner.yml @@ -17,15 +17,14 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - - name: Submit and monitor SLURM job + - name: Submit SLURM job and wait run: | - - cd $GITHUB_WORKSPACE || exit + cd "$GITHUB_WORKSPACE" || exit echo "Submitting job..." - JOBID=$(sbatch --parsable "$SLURM_SCRIPT") + JOBID="$(sbatch --parsable "$SLURM_SCRIPT")" echo "Submitted as $JOBID" - echo "JOBID=$JOBID" >> $GITHUB_ENV + echo "JOBID=$JOBID" >> "$GITHUB_ENV" echo "Waiting for job to complete..." while squeue -j "$JOBID" 2>/dev/null | grep -q "$JOBID"; do @@ -35,7 +34,7 @@ jobs: echo "Job $JOBID appears to have finished." echo "Querying SLURM exit code..." - EXIT_CODE=$(sacct -j "${JOBID}.batch" --format=ExitCode -n | awk '{print $1}' | cut -d':' -f1) + EXIT_CODE="$(sacct -j "${JOBID}.batch" --format=ExitCode -n | awk '{print $1}' | cut -d':' -f1)" if [[ -z "$EXIT_CODE" ]]; then echo "ERROR: Could not retrieve exit code from sacct." From a2362007dfc5e1201d9360c9e8dbbf89d3863597 Mon Sep 17 00:00:00 2001 From: Alex Krochak Date: Tue, 2 Sep 2025 15:13:27 +0200 Subject: [PATCH 5/6] retry runner --- .github/workflows/juwels-runner.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/juwels-runner.yml b/.github/workflows/juwels-runner.yml index aa77d67fa..4ae99761e 100644 --- a/.github/workflows/juwels-runner.yml +++ b/.github/workflows/juwels-runner.yml @@ -17,7 +17,7 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - - name: Submit SLURM job and wait + - name: Submit and monitor SLURM job run: | cd "$GITHUB_WORKSPACE" || exit From 91db548a59dd0c1731f7f54214e7c9d7534247a7 Mon Sep 17 00:00:00 2001 From: Alex Krochak Date: Tue, 2 Sep 2025 15:16:05 +0200 Subject: [PATCH 6/6] restore original pytest workflow --- .github/workflows/pytest.yml | 112 +++++++++++------------------------ 1 file changed, 35 insertions(+), 77 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index e3d45d109..4ddccaba4 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -1,88 +1,47 @@ +# -------------------------------------------------------------------------------------- +# Part of the interTwin Project: https://www.intertwin.eu/ +# +# Created by: Matteo Bunino +# +# Credit: +# - Matteo Bunino - CERN +# -------------------------------------------------------------------------------------- + name: Testing with pytest on: pull_request: - branches: [main, dev] - push: - branches: [juwels-runner] -jobs: - test-on-juwels: - name: Testing with Pytest on self-hosted runner on JUWELS Booster - if: github.ref == 'refs/heads/juwels-runner' - runs-on: [self-hosted, linux] - env: - GITHUB_WORKSPACE: /p/project1/intertwin/krochak1/itwinai - SLURM_SCRIPT: /p/project1/intertwin/juwels-selfhosted-runner/RUNNER/batch-pytest.sh - SLURM_LOGS: /p/project1/intertwin/juwels-selfhosted-runner/RUNNER/slurm-logs - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Submit and monitor SLURM job - run: | - cd $GITHUB_WORKSPACE || exit - - echo "Submitting job..." - JOBID=$(sbatch --parsable "$SLURM_SCRIPT") - echo "Submitted as $JOBID" - echo "JOBID=$JOBID" >> $GITHUB_ENV - - echo "Waiting for job to complete..." - while squeue -j "$JOBID" 2>/dev/null | grep -q "$JOBID"; do - sleep 10 - done - - echo "Job $JOBID appears to have finished." - - echo "Querying SLURM exit code..." - EXIT_CODE=$(sacct -j "${JOBID}.batch" --format=ExitCode -n | awk '{print $1}' | cut -d':' -f1) - - if [[ -z "$EXIT_CODE" ]]; then - echo "ERROR: Could not retrieve exit code from sacct." - exit 1 - fi - - if [[ "$EXIT_CODE" -ne 0 ]]; then - echo "SLURM job failed with exit code $EXIT_CODE" - exit 1 - else - echo "SLURM job completed successfully with exit code 0" - fi - - - name: Read SLURM logs - run: | - echo "=== SLURM STDOUT ===" - cat "${SLURM_LOGS}/output.${JOBID}.out" || echo "No output file found." - - echo "=== SLURM STDERR ===" - cat "${SLURM_LOGS}/error.${JOBID}.out" || echo "No error file found." + branches: [main] +jobs: test-torch: name: Testing with pytest runs-on: ubuntu-latest steps: - - name: Maximize Disk Space - uses: easimon/maximize-build-space@v10 - with: - # Reserve space on root for docker/dagger cache - build-mount-path: /docker - root-reserve-mb: 2048 - overprovision-lvm: false - swap-size-mb: 4096 - remove-dotnet: true - remove-android: true - remove-haskell: true - remove-codeql: true - - - uses: actions/checkout@v4 - - name: Move Docker directory - shell: bash - run: | - sudo mv /var/lib/docker /docker/ && - sudo ln -s /docker/docker /var/lib/docker && - sudo systemctl restart docker + # Uncomment this only if you run out of disk space! + # - name: Maximize Disk Space + # uses: easimon/maximize-build-space@v10 + # with: + # # Reserve space on root for docker/dagger cache + # build-mount-path: /docker + # root-reserve-mb: 2048 + # overprovision-lvm: false + # swap-size-mb: 4096 + # remove-dotnet: true + # remove-android: true + # remove-haskell: true + # remove-codeql: true + + - uses: actions/checkout@v5 + + # ALSO uncomment this only if you run out of disk space! + # - name: Move Docker directory + # shell: bash + # run: | + # sudo mv /var/lib/docker /docker/ && + # sudo ln -s /docker/docker /var/lib/docker && + # sudo systemctl restart docker # Run tests with pytest in a container - name: Run Integration Test (development pipeline) @@ -98,5 +57,4 @@ jobs: --cmd "pytest,-v,--disable-warnings,-n,logical,/app/tests/,--dist,loadfile,-m,not hpc and not tensorflow" logs cloud-token: ${{ secrets.DAGGER_CLOUD_TOKEN }} - version: latest - + version: latest \ No newline at end of file