Skip to content
198 changes: 166 additions & 32 deletions .github/workflows/pr-build.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -28,33 +28,81 @@ run-name: >

jobs:

check_changes:
runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }}
outputs:
should_build: ${{ steps.filter.outputs.should_build }}
changed_files: ${{ steps.filter.outputs.changed_files }}
steps:
- name: Checkout code
uses: actions/checkout@v6
with:
fetch-depth: 0
repository: ${{ github.event.pull_request.head.repo.full_name }}
ref: ${{ github.event.pull_request.head.ref }}

- name: Fix workspace permissions
run: sudo chown -R $USER:$USER .

- name: Check for relevant file changes
id: filter
run: |
if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
echo "should_build=true" >> $GITHUB_OUTPUT
echo "Workflow dispatch - proceeding with build"
exit 0
fi

# Fetch base branch with full history
git fetch origin ${{ github.base_ref }}
CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD)

echo "Changed files:"
echo "$CHANGED_FILES"

# Store changed files for reuse in later jobs
echo "changed_files<<EOF" >> $GITHUB_OUTPUT
echo "$CHANGED_FILES" >> $GITHUB_OUTPUT
echo "EOF" >> $GITHUB_OUTPUT

# Check for build_info.json, .sh scripts (excluding specific directories), or Dockerfile
RELEVANT_CHANGES=$(echo "$CHANGED_FILES" | grep -E '(build_info\.json|\.sh$|Dockerfile)' | grep -vE '^(gha-script|process_bom|script|templates|travis-currency-ymls|travis-ymls)/' || true)

if [ -n "$RELEVANT_CHANGES" ]; then
echo "should_build=true" >> $GITHUB_OUTPUT
echo "✅ Found relevant changes:"
echo "$RELEVANT_CHANGES"
else
echo "should_build=false" >> $GITHUB_OUTPUT
echo "⏭️ Skipping PR build CI check - no changes related to build_info.json, build scripts (.sh), or Dockerfile"
fi

build_info:
needs: check_changes
if: needs.check_changes.outputs.should_build == 'true'
runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }}
outputs:
wheel_build_enabled: ${{ steps.set_flags.outputs.wheel_build_enabled }}
has_sh_changes: ${{ steps.set_flags.outputs.has_sh_changes }}
has_dockerfile_changes: ${{ steps.set_flags.outputs.has_dockerfile_changes }}
docker_build_enabled: ${{ steps.set_flags.outputs.docker_build_enabled }}
build_package_enabled: ${{ steps.set_flags.outputs.build_package_enabled }}

steps:
- name: Checkout code (Pull Request)
if: github.event_name == 'pull_request'
uses: actions/checkout@v6
with:
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha }}
fetch-depth: 0
repository: ${{ github.event.pull_request.head.repo.full_name }}
ref: ${{ github.event.pull_request.head.ref }}

- name: Checkout code (Workflow Dispatch)
if: github.event_name == 'workflow_dispatch'
uses: actions/checkout@v6
with:
ref: refs/pull/${{ inputs.pr_number }}/head

- name: Install required packages
run: |
sudo apt update -y
sudo apt-get install -y file jq

- name: Install Python dependencies
run: |
pip3 install --force-reinstall -v "requests==2.31.0"
pip3 install --upgrade docker

- name: Debug - Check user and workspace ownership
run: |
echo "Current user: $USER"
Expand All @@ -69,6 +117,15 @@ jobs:
- name: Fix workspace permissions
run: sudo chown -R $USER:$USER .

- name: Install required packages
run: |
sudo apt update -y
sudo apt-get install -y file jq

- name: Install Python dependencies
run: |
pip3 install --force-reinstall -v "requests==2.31.0"
pip3 install --upgrade docker

- name: Set PR number
run: echo "PR_NUMBER=${{ github.event.pull_request.number || inputs.pr_number }}" >> $GITHUB_ENV
Expand All @@ -90,10 +147,19 @@ jobs:
echo "Script completed successfully for PR #${PR_NUMBER}"
fi

- name: Fetch base branch
run: git fetch origin ${{ github.base_ref }} --depth=1

- name: Locate and parse build_info.json
run: |
# Reuse changed files from check_changes job
CHANGED_FILES="${{ needs.check_changes.outputs.changed_files }}"

CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD)
# If workflow_dispatch, check_changes did not compute changed_files — fetch them now
if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
git fetch origin ${{ github.base_ref }} --depth=1
CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD)
fi

BUILD_INFO_FILE=$(echo "$CHANGED_FILES" | grep 'build_info.json' | head -n 1)

Expand Down Expand Up @@ -122,13 +188,74 @@ jobs:
echo "$CHANGED_FILES" >> $GITHUB_ENV
echo "EOF" >> $GITHUB_ENV


- name: Read build_info.json
run: |
sudo chown -R $USER:$USER .
chmod +x ./gha-script/read_buildinfo.sh
bash ./gha-script/read_buildinfo.sh

- name: Set job control flags
id: set_flags
run: |
PACKAGE_DIR=$(jq -r '.package_dir // ""' $BUILD_INFO_FILE)
# Strip trailing slash if present — prevents double-slash in regex (e.g. "m/multidict/" → "m/multidict")
PACKAGE_DIR="${PACKAGE_DIR%/}"
WHEEL_BUILD=$(jq -r '.wheel_build // "false"' $BUILD_INFO_FILE)

# Robust docker_build extraction
if jq -e '.docker_build == true or .docker_build == "true"' "$BUILD_INFO_FILE" > /dev/null; then
DOCKER_BUILD="true"
else
DOCKER_BUILD="false"
fi

# Check if .sh scripts changed in this package
SH_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true)

# Check if Dockerfile changed
DOCKERFILE_CHANGED=$(echo "$CHANGED_FILES" | grep -i 'Dockerfile' || true)

# Set outputs for wheel builds
if [ "$WHEEL_BUILD" == "true" ] && [ -n "$SH_SCRIPT_CHANGED" ]; then
echo "wheel_build_enabled=true" >> $GITHUB_OUTPUT
echo "✅ Wheel builds will run (WHEEL_BUILD=true and .sh scripts changed)"
else
echo "wheel_build_enabled=false" >> $GITHUB_OUTPUT
echo "⏭️ Wheel builds will be skipped (WHEEL_BUILD=$WHEEL_BUILD, .sh changes: ${SH_SCRIPT_CHANGED:-none})"
fi

# Set output for sh changes
if [ -n "$SH_SCRIPT_CHANGED" ]; then
echo "has_sh_changes=true" >> $GITHUB_OUTPUT
else
echo "has_sh_changes=false" >> $GITHUB_OUTPUT
fi

# Set outputs for docker build
if [ "$DOCKER_BUILD" == "true" ] && [ -n "$DOCKERFILE_CHANGED" ]; then
echo "docker_build_enabled=true" >> $GITHUB_OUTPUT
echo "has_dockerfile_changes=true" >> $GITHUB_OUTPUT
echo "✅ Docker build will run (BUILD_DOCKER=true and Dockerfile changed)"
else
echo "docker_build_enabled=false" >> $GITHUB_OUTPUT
if [ -n "$DOCKERFILE_CHANGED" ]; then
echo "has_dockerfile_changes=true" >> $GITHUB_OUTPUT
else
echo "has_dockerfile_changes=false" >> $GITHUB_OUTPUT
fi
echo "⏭️ Docker build will be skipped (BUILD_DOCKER=$DOCKER_BUILD, Dockerfile changes: ${DOCKERFILE_CHANGED:-none})"
fi

# Set output for build job (runs when build_info.json or .sh scripts change)
BUILD_INFO_CHANGED=$(echo "$CHANGED_FILES" | grep 'build_info\.json' || true)
if [ -n "$BUILD_INFO_CHANGED" ] || [ -n "$SH_SCRIPT_CHANGED" ]; then
echo "build_package_enabled=true" >> $GITHUB_OUTPUT
echo "✅ Build package job will run (build_info.json or .sh scripts changed)"
else
echo "build_package_enabled=false" >> $GITHUB_OUTPUT
echo "⏭️ Build package job will be skipped (no build_info.json or .sh script changes)"
fi

- name: Create scanner-env.sh
run: |
mkdir -p package-cache
Expand Down Expand Up @@ -165,12 +292,14 @@ jobs:

build:
needs: build_info
if: needs.build_info.outputs.build_package_enabled == 'true'
runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }}

steps:
- uses: actions/checkout@v6
with:
ref: ${{ github.event.pull_request.head.sha }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
ref: ${{ github.event.pull_request.head.ref }}

- name: Download package-cache
uses: actions/download-artifact@v7
Expand All @@ -191,16 +320,16 @@ jobs:
cat package-cache/scanner-env.sh

sudo chown -R $USER:$USER .
chmod +x ./gha-script/build_package.sh
bash ./gha-script/build_package.sh
echo "------------------- Executing changed scripts -----------------------------"
chmod +x ./gha-script/execute_changed_scripts.py
python3 ./gha-script/execute_changed_scripts.py

# ===================== WHEEL JOBS =====================



wheel_build_py310:
needs: build_info
if: ${{ success() }}
if: needs.build_info.outputs.wheel_build_enabled == 'true'
runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }}
continue-on-error: false
env:
Expand All @@ -213,7 +342,8 @@ jobs:
uses: actions/checkout@v6
with:
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
ref: ${{ github.event.pull_request.head.ref }}

- name: Install system dependencies
run: |
Expand Down Expand Up @@ -264,7 +394,7 @@ jobs:

wheel_build_py311:
needs: build_info
if: ${{ success() }}
if: needs.build_info.outputs.wheel_build_enabled == 'true'
runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }}
continue-on-error: false
env:
Expand All @@ -277,7 +407,8 @@ jobs:
uses: actions/checkout@v6
with:
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
ref: ${{ github.event.pull_request.head.ref }}

- name: Install system dependencies
run: |
Expand Down Expand Up @@ -326,10 +457,9 @@ jobs:
sudo uname -a



wheel_build_py312:
needs: build_info
if: ${{ success() }}
if: needs.build_info.outputs.wheel_build_enabled == 'true'
runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }}
continue-on-error: false
env:
Expand All @@ -342,7 +472,8 @@ jobs:
uses: actions/checkout@v6
with:
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
ref: ${{ github.event.pull_request.head.ref }}

- name: Install system dependencies
run: |
Expand Down Expand Up @@ -390,9 +521,10 @@ jobs:
sudo lsb_release -a 2>/dev/null || echo "lsb_release not available"
sudo uname -a


wheel_build_py313:
needs: build_info
if: ${{ success() }}
if: needs.build_info.outputs.wheel_build_enabled == 'true'
runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }}
continue-on-error: true
env:
Expand All @@ -405,7 +537,8 @@ jobs:
uses: actions/checkout@v6
with:
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
ref: ${{ github.event.pull_request.head.ref }}

- name: Install system dependencies
run: |
Expand Down Expand Up @@ -456,7 +589,7 @@ jobs:

wheel_build_py314:
needs: build_info
if: ${{ success() }}
if: needs.build_info.outputs.wheel_build_enabled == 'true'
runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }}
continue-on-error: true
env:
Expand All @@ -469,7 +602,8 @@ jobs:
uses: actions/checkout@v6
with:
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
ref: ${{ github.event.pull_request.head.ref }}

- name: Install system dependencies
run: |
Expand Down Expand Up @@ -518,17 +652,17 @@ jobs:
sudo uname -a



build_docker:
needs: build_info
if: ${{ success() }}
if: needs.build_info.outputs.docker_build_enabled == 'true'
runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }}

steps:
- name: Checkout code
uses: actions/checkout@v6
with:
ref: ${{ github.event.pull_request.head.sha }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
ref: ${{ github.event.pull_request.head.ref }}

- name: Download package-cache
uses: actions/download-artifact@v7
Expand Down
2 changes: 1 addition & 1 deletion gha-script/download-scripts/download_file.sh
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ if [[ $(echo "$token_request" | jq -r '.errorCode') == "null" ]]; then
token=$(echo "$token_request" | jq -r '.access_token')
echo "Downloading $1 from COS..."
curl -X GET -H "Authorization: bearer $token" \
-o $1 "https://s3.us.cloud-object-storage.appdomain.cloud/ose-power-toolci-bucket-production/$PACKAGE_NAME/$VERSION/$1"
-o $1 "https://s3.us.cloud-object-storage.appdomain.cloud/ose-power-toolci-bucket-stag/$PACKAGE_NAME/$VERSION/$1"
echo "Download completed: $1"
else
echo "Error: Token request failed. Response: $token_request"
Expand Down
2 changes: 1 addition & 1 deletion gha-script/download-scripts/download_wheels.sh
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ token_request=$(curl -s -X POST https://iam.cloud.ibm.com/identity/token \

if [[ $(echo "$token_request" | jq -r '.errorCode') == "null" ]]; then
token=$(echo "$token_request" | jq -r '.access_token')
bucket_url="https://s3.us.cloud-object-storage.appdomain.cloud/ose-power-artifacts-production"
bucket_url="https://s3.us.cloud-object-storage.appdomain.cloud/ose-power-artifacts-stag"

echo "Fetching wheel list from COS..."
echo "Checking prefix: $PACKAGE_NAME/$VERSION/"
Expand Down
Loading
Loading