Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
198 changes: 166 additions & 32 deletions .github/workflows/pr-build.yaml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: PR build

Check warning on line 1 in .github/workflows/pr-build.yaml

View check run for this annotation

power-z-gha-runner / wheel_build_py310

Provisioning Failed

Unable to create the job container.
Raw output
The system failed to provision the LXD container environment for Job 85376955688: create container error: timeout: can't create container ghaProdLon04P10B2-74919273-1011062379-85376955688: Failed creating instance record: Failed initialising instance: Failed loading storage pool: Storage pool not found
Retrying job 85376955688 (Attempt 1) in 1m0s

on:
pull_request:
Expand Down Expand Up @@ -28,33 +28,81 @@

jobs:

check_changes:
runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }}
outputs:
should_build: ${{ steps.filter.outputs.should_build }}
changed_files: ${{ steps.filter.outputs.changed_files }}
steps:
- name: Checkout code
uses: actions/checkout@v6
with:
fetch-depth: 0
repository: ${{ github.event.pull_request.head.repo.full_name }}
ref: ${{ github.event.pull_request.head.ref }}

- name: Fix workspace permissions
run: sudo chown -R $USER:$USER .

- name: Check for relevant file changes
id: filter
run: |
if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
echo "should_build=true" >> $GITHUB_OUTPUT
echo "Workflow dispatch - proceeding with build"
exit 0
fi

# Fetch base branch with full history
git fetch origin ${{ github.base_ref }}
CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD)

echo "Changed files:"
echo "$CHANGED_FILES"

# Store changed files for reuse in later jobs
echo "changed_files<<EOF" >> $GITHUB_OUTPUT
echo "$CHANGED_FILES" >> $GITHUB_OUTPUT
echo "EOF" >> $GITHUB_OUTPUT

# Check for build_info.json, .sh scripts (excluding specific directories), or Dockerfile
RELEVANT_CHANGES=$(echo "$CHANGED_FILES" | grep -E '(build_info\.json|\.sh$|Dockerfile)' | grep -vE '^(gha-script|process_bom|script|templates|travis-currency-ymls|travis-ymls)/' || true)

if [ -n "$RELEVANT_CHANGES" ]; then
echo "should_build=true" >> $GITHUB_OUTPUT
echo "✅ Found relevant changes:"
echo "$RELEVANT_CHANGES"
else
echo "should_build=false" >> $GITHUB_OUTPUT
echo "⏭️ Skipping PR build CI check - no changes related to build_info.json, build scripts (.sh), or Dockerfile"
fi

build_info:
needs: check_changes
if: needs.check_changes.outputs.should_build == 'true'
runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }}
outputs:
wheel_build_enabled: ${{ steps.set_flags.outputs.wheel_build_enabled }}
has_sh_changes: ${{ steps.set_flags.outputs.has_sh_changes }}
has_dockerfile_changes: ${{ steps.set_flags.outputs.has_dockerfile_changes }}
docker_build_enabled: ${{ steps.set_flags.outputs.docker_build_enabled }}
build_package_enabled: ${{ steps.set_flags.outputs.build_package_enabled }}

steps:
- name: Checkout code (Pull Request)
if: github.event_name == 'pull_request'
uses: actions/checkout@v6
with:
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha }}
fetch-depth: 0
repository: ${{ github.event.pull_request.head.repo.full_name }}
ref: ${{ github.event.pull_request.head.ref }}

- name: Checkout code (Workflow Dispatch)
if: github.event_name == 'workflow_dispatch'
uses: actions/checkout@v6
with:
ref: refs/pull/${{ inputs.pr_number }}/head

- name: Install required packages
run: |
sudo apt update -y
sudo apt-get install -y file jq

- name: Install Python dependencies
run: |
pip3 install --force-reinstall -v "requests==2.31.0"
pip3 install --upgrade docker

- name: Debug - Check user and workspace ownership
run: |
echo "Current user: $USER"
Expand All @@ -69,6 +117,15 @@
- name: Fix workspace permissions
run: sudo chown -R $USER:$USER .

- name: Install required packages
run: |
sudo apt update -y
sudo apt-get install -y file jq

- name: Install Python dependencies
run: |
pip3 install --force-reinstall -v "requests==2.31.0"
pip3 install --upgrade docker

- name: Set PR number
run: echo "PR_NUMBER=${{ github.event.pull_request.number || inputs.pr_number }}" >> $GITHUB_ENV
Expand All @@ -90,10 +147,19 @@
echo "Script completed successfully for PR #${PR_NUMBER}"
fi

- name: Fetch base branch
run: git fetch origin ${{ github.base_ref }} --depth=1

- name: Locate and parse build_info.json
run: |
# Reuse changed files from check_changes job
CHANGED_FILES="${{ needs.check_changes.outputs.changed_files }}"

CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD)
# If workflow_dispatch, check_changes did not compute changed_files — fetch them now
if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
git fetch origin ${{ github.base_ref }} --depth=1
CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD)
fi

BUILD_INFO_FILE=$(echo "$CHANGED_FILES" | grep 'build_info.json' | head -n 1)

Expand Down Expand Up @@ -122,13 +188,74 @@
echo "$CHANGED_FILES" >> $GITHUB_ENV
echo "EOF" >> $GITHUB_ENV


- name: Read build_info.json
run: |
sudo chown -R $USER:$USER .
chmod +x ./gha-script/read_buildinfo.sh
bash ./gha-script/read_buildinfo.sh

- name: Set job control flags
id: set_flags
run: |
PACKAGE_DIR=$(jq -r '.package_dir // ""' $BUILD_INFO_FILE)
# Strip trailing slash if present — prevents double-slash in regex (e.g. "m/multidict/" → "m/multidict")
PACKAGE_DIR="${PACKAGE_DIR%/}"
WHEEL_BUILD=$(jq -r '.wheel_build // "false"' $BUILD_INFO_FILE)

# Robust docker_build extraction
if jq -e '.docker_build == true or .docker_build == "true"' "$BUILD_INFO_FILE" > /dev/null; then
DOCKER_BUILD="true"
else
DOCKER_BUILD="false"
fi

# Check if .sh scripts changed in this package
SH_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true)

# Check if Dockerfile changed
DOCKERFILE_CHANGED=$(echo "$CHANGED_FILES" | grep -i 'Dockerfile' || true)

# Set outputs for wheel builds
if [ "$WHEEL_BUILD" == "true" ] && [ -n "$SH_SCRIPT_CHANGED" ]; then
echo "wheel_build_enabled=true" >> $GITHUB_OUTPUT
echo "✅ Wheel builds will run (WHEEL_BUILD=true and .sh scripts changed)"
else
echo "wheel_build_enabled=false" >> $GITHUB_OUTPUT
echo "⏭️ Wheel builds will be skipped (WHEEL_BUILD=$WHEEL_BUILD, .sh changes: ${SH_SCRIPT_CHANGED:-none})"
fi

# Set output for sh changes
if [ -n "$SH_SCRIPT_CHANGED" ]; then
echo "has_sh_changes=true" >> $GITHUB_OUTPUT
else
echo "has_sh_changes=false" >> $GITHUB_OUTPUT
fi

# Set outputs for docker build
if [ "$DOCKER_BUILD" == "true" ] && [ -n "$DOCKERFILE_CHANGED" ]; then
echo "docker_build_enabled=true" >> $GITHUB_OUTPUT
echo "has_dockerfile_changes=true" >> $GITHUB_OUTPUT
echo "✅ Docker build will run (BUILD_DOCKER=true and Dockerfile changed)"
else
echo "docker_build_enabled=false" >> $GITHUB_OUTPUT
if [ -n "$DOCKERFILE_CHANGED" ]; then
echo "has_dockerfile_changes=true" >> $GITHUB_OUTPUT
else
echo "has_dockerfile_changes=false" >> $GITHUB_OUTPUT
fi
echo "⏭️ Docker build will be skipped (BUILD_DOCKER=$DOCKER_BUILD, Dockerfile changes: ${DOCKERFILE_CHANGED:-none})"
fi

# Set output for build job (runs when build_info.json or .sh scripts change)
BUILD_INFO_CHANGED=$(echo "$CHANGED_FILES" | grep 'build_info\.json' || true)
if [ -n "$BUILD_INFO_CHANGED" ] || [ -n "$SH_SCRIPT_CHANGED" ]; then
echo "build_package_enabled=true" >> $GITHUB_OUTPUT
echo "✅ Build package job will run (build_info.json or .sh scripts changed)"
else
echo "build_package_enabled=false" >> $GITHUB_OUTPUT
echo "⏭️ Build package job will be skipped (no build_info.json or .sh script changes)"
fi

- name: Create scanner-env.sh
run: |
mkdir -p package-cache
Expand Down Expand Up @@ -165,12 +292,14 @@

build:
needs: build_info
if: needs.build_info.outputs.build_package_enabled == 'true'
runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }}

steps:
- uses: actions/checkout@v6
with:
ref: ${{ github.event.pull_request.head.sha }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
ref: ${{ github.event.pull_request.head.ref }}

- name: Download package-cache
uses: actions/download-artifact@v7
Expand All @@ -191,16 +320,16 @@
cat package-cache/scanner-env.sh

sudo chown -R $USER:$USER .
chmod +x ./gha-script/build_package.sh
bash ./gha-script/build_package.sh
echo "------------------- Executing changed scripts -----------------------------"
chmod +x ./gha-script/execute_changed_scripts.py
python3 ./gha-script/execute_changed_scripts.py

# ===================== WHEEL JOBS =====================



wheel_build_py310:
needs: build_info
if: ${{ success() }}
if: needs.build_info.outputs.wheel_build_enabled == 'true'
runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }}
continue-on-error: false
env:
Expand All @@ -213,7 +342,8 @@
uses: actions/checkout@v6
with:
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
ref: ${{ github.event.pull_request.head.ref }}

- name: Install system dependencies
run: |
Expand Down Expand Up @@ -264,7 +394,7 @@

wheel_build_py311:
needs: build_info
if: ${{ success() }}
if: needs.build_info.outputs.wheel_build_enabled == 'true'
runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }}
continue-on-error: false
env:
Expand All @@ -277,7 +407,8 @@
uses: actions/checkout@v6
with:
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
ref: ${{ github.event.pull_request.head.ref }}

- name: Install system dependencies
run: |
Expand Down Expand Up @@ -326,10 +457,9 @@
sudo uname -a



wheel_build_py312:
needs: build_info
if: ${{ success() }}
if: needs.build_info.outputs.wheel_build_enabled == 'true'
runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }}
continue-on-error: false
env:
Expand All @@ -342,7 +472,8 @@
uses: actions/checkout@v6
with:
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
ref: ${{ github.event.pull_request.head.ref }}

- name: Install system dependencies
run: |
Expand Down Expand Up @@ -390,9 +521,10 @@
sudo lsb_release -a 2>/dev/null || echo "lsb_release not available"
sudo uname -a


wheel_build_py313:
needs: build_info
if: ${{ success() }}
if: needs.build_info.outputs.wheel_build_enabled == 'true'
runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }}
continue-on-error: true
env:
Expand All @@ -405,7 +537,8 @@
uses: actions/checkout@v6
with:
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
ref: ${{ github.event.pull_request.head.ref }}

- name: Install system dependencies
run: |
Expand Down Expand Up @@ -456,7 +589,7 @@

wheel_build_py314:
needs: build_info
if: ${{ success() }}
if: needs.build_info.outputs.wheel_build_enabled == 'true'
runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }}
continue-on-error: true
env:
Expand All @@ -469,7 +602,8 @@
uses: actions/checkout@v6
with:
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
ref: ${{ github.event.pull_request.head.ref }}

- name: Install system dependencies
run: |
Expand Down Expand Up @@ -518,17 +652,17 @@
sudo uname -a



build_docker:
needs: build_info
if: ${{ success() }}
if: needs.build_info.outputs.docker_build_enabled == 'true'
runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }}

steps:
- name: Checkout code
uses: actions/checkout@v6
with:
ref: ${{ github.event.pull_request.head.sha }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
ref: ${{ github.event.pull_request.head.ref }}

- name: Download package-cache
uses: actions/download-artifact@v7
Expand Down
2 changes: 1 addition & 1 deletion gha-script/download-scripts/download_file.sh
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ if [[ $(echo "$token_request" | jq -r '.errorCode') == "null" ]]; then
token=$(echo "$token_request" | jq -r '.access_token')
echo "Downloading $1 from COS..."
curl -X GET -H "Authorization: bearer $token" \
-o $1 "https://s3.us.cloud-object-storage.appdomain.cloud/ose-power-toolci-bucket-production/$PACKAGE_NAME/$VERSION/$1"
-o $1 "https://s3.us.cloud-object-storage.appdomain.cloud/ose-power-toolci-bucket-stag/$PACKAGE_NAME/$VERSION/$1"
echo "Download completed: $1"
else
echo "Error: Token request failed. Response: $token_request"
Expand Down
2 changes: 1 addition & 1 deletion gha-script/download-scripts/download_wheels.sh
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ token_request=$(curl -s -X POST https://iam.cloud.ibm.com/identity/token \

if [[ $(echo "$token_request" | jq -r '.errorCode') == "null" ]]; then
token=$(echo "$token_request" | jq -r '.access_token')
bucket_url="https://s3.us.cloud-object-storage.appdomain.cloud/ose-power-artifacts-production"
bucket_url="https://s3.us.cloud-object-storage.appdomain.cloud/ose-power-artifacts-stag"

echo "Fetching wheel list from COS..."
echo "Checking prefix: $PACKAGE_NAME/$VERSION/"
Expand Down
Loading