From aa86057af5dc565d5d46eeb6878d39bf6ee4a16d Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Mon, 6 Jul 2026 02:21:21 +0530 Subject: [PATCH 01/13] Update pr-build.yaml --- .github/workflows/pr-build.yaml | 170 +++++++++++++++++++++++++++----- 1 file changed, 145 insertions(+), 25 deletions(-) diff --git a/.github/workflows/pr-build.yaml b/.github/workflows/pr-build.yaml index 45f3a366f1..4e47c2c818 100644 --- a/.github/workflows/pr-build.yaml +++ b/.github/workflows/pr-build.yaml @@ -28,16 +28,72 @@ run-name: > jobs: + check_changes: + runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} + outputs: + should_build: ${{ steps.filter.outputs.should_build }} + changed_files: ${{ steps.filter.outputs.changed_files }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + + - name: Fix workspace permissions + run: sudo chown -R $USER:$USER . + + - name: Check for relevant file changes + id: filter + run: | + if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then + echo "should_build=true" >> $GITHUB_OUTPUT + echo "Workflow dispatch - proceeding with build" + exit 0 + fi + + # Fetch base branch with full history + git fetch origin ${{ github.base_ref }} + CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD) + + echo "Changed files:" + echo "$CHANGED_FILES" + + # Store changed files for reuse in later jobs + echo "changed_files<> $GITHUB_OUTPUT + echo "$CHANGED_FILES" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + + # Check for build_info.json, .sh scripts (excluding specific directories), or Dockerfile + RELEVANT_CHANGES=$(echo "$CHANGED_FILES" | grep -E '(build_info\.json|\.sh$|Dockerfile)' | grep -vE '^(gha-script|process_bom|script|templates|travis-currency-ymls|travis-ymls)/' || true) + + if [ -n "$RELEVANT_CHANGES" ]; then + echo "should_build=true" >> $GITHUB_OUTPUT + echo "✅ Found relevant changes:" + echo "$RELEVANT_CHANGES" + else + echo "should_build=false" >> $GITHUB_OUTPUT + echo "⏭️ Skipping PR build CI check - no changes related to build_info.json, build scripts (.sh), or Dockerfile" + fi + build_info: + needs: check_changes + if: needs.check_changes.outputs.should_build == 'true' runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} + outputs: + wheel_build_enabled: ${{ steps.set_flags.outputs.wheel_build_enabled }} + has_sh_changes: ${{ steps.set_flags.outputs.has_sh_changes }} + has_dockerfile_changes: ${{ steps.set_flags.outputs.has_dockerfile_changes }} + docker_build_enabled: ${{ steps.set_flags.outputs.docker_build_enabled }} + build_package_enabled: ${{ steps.set_flags.outputs.build_package_enabled }} steps: - name: Checkout code (Pull Request) if: github.event_name == 'pull_request' uses: actions/checkout@v6 with: - fetch-depth: 0 - ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} - name: Checkout code (Workflow Dispatch) if: github.event_name == 'workflow_dispatch' @@ -45,16 +101,6 @@ jobs: with: ref: refs/pull/${{ inputs.pr_number }}/head - - name: Install required packages - run: | - sudo apt update -y - sudo apt-get install -y file jq - - - name: Install Python dependencies - run: | - pip3 install --force-reinstall -v "requests==2.31.0" - pip3 install --upgrade docker - - name: Debug - Check user and workspace ownership run: | echo "Current user: $USER" @@ -69,6 +115,15 @@ jobs: - name: Fix workspace permissions run: sudo chown -R $USER:$USER . + - name: Install required packages + run: | + sudo apt update -y + sudo apt-get install -y file jq + + - name: Install Python dependencies + run: | + pip3 install --force-reinstall -v "requests==2.31.0" + pip3 install --upgrade docker - name: Set PR number run: echo "PR_NUMBER=${{ github.event.pull_request.number || inputs.pr_number }}" >> $GITHUB_ENV @@ -92,8 +147,14 @@ jobs: - name: Locate and parse build_info.json run: | + # Reuse changed files from check_changes job + CHANGED_FILES="${{ needs.check_changes.outputs.changed_files }}" - CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD) + # If workflow_dispatch, check_changes did not compute changed_files — fetch them now + if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then + git fetch origin ${{ github.base_ref }} --depth=1 + CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD) + fi BUILD_INFO_FILE=$(echo "$CHANGED_FILES" | grep 'build_info.json' | head -n 1) @@ -122,13 +183,72 @@ jobs: echo "$CHANGED_FILES" >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV - - name: Read build_info.json run: | sudo chown -R $USER:$USER . chmod +x ./gha-script/read_buildinfo.sh bash ./gha-script/read_buildinfo.sh + - name: Set job control flags + id: set_flags + run: | + PACKAGE_DIR=$(jq -r '.package_dir // ""' $BUILD_INFO_FILE) + WHEEL_BUILD=$(jq -r '.wheel_build // "false"' $BUILD_INFO_FILE) + + # Robust docker_build extraction + if jq -e '.docker_build == true or .docker_build == "true"' "$BUILD_INFO_FILE" > /dev/null; then + DOCKER_BUILD="true" + else + DOCKER_BUILD="false" + fi + + # Check if .sh scripts changed in this package + SH_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) + + # Check if Dockerfile changed + DOCKERFILE_CHANGED=$(echo "$CHANGED_FILES" | grep -i 'Dockerfile' || true) + + # Set outputs for wheel builds + if [ "$WHEEL_BUILD" == "true" ] && [ -n "$SH_SCRIPT_CHANGED" ]; then + echo "wheel_build_enabled=true" >> $GITHUB_OUTPUT + echo "✅ Wheel builds will run (WHEEL_BUILD=true and .sh scripts changed)" + else + echo "wheel_build_enabled=false" >> $GITHUB_OUTPUT + echo "⏭️ Wheel builds will be skipped (WHEEL_BUILD=$WHEEL_BUILD, .sh changes: ${SH_SCRIPT_CHANGED:-none})" + fi + + # Set output for sh changes + if [ -n "$SH_SCRIPT_CHANGED" ]; then + echo "has_sh_changes=true" >> $GITHUB_OUTPUT + else + echo "has_sh_changes=false" >> $GITHUB_OUTPUT + fi + + # Set outputs for docker build + if [ "$DOCKER_BUILD" == "true" ] && [ -n "$DOCKERFILE_CHANGED" ]; then + echo "docker_build_enabled=true" >> $GITHUB_OUTPUT + echo "has_dockerfile_changes=true" >> $GITHUB_OUTPUT + echo "✅ Docker build will run (BUILD_DOCKER=true and Dockerfile changed)" + else + echo "docker_build_enabled=false" >> $GITHUB_OUTPUT + if [ -n "$DOCKERFILE_CHANGED" ]; then + echo "has_dockerfile_changes=true" >> $GITHUB_OUTPUT + else + echo "has_dockerfile_changes=false" >> $GITHUB_OUTPUT + fi + echo "⏭️ Docker build will be skipped (BUILD_DOCKER=$DOCKER_BUILD, Dockerfile changes: ${DOCKERFILE_CHANGED:-none})" + fi + + # Set output for build job (runs when build_info.json or .sh scripts change) + BUILD_INFO_CHANGED=$(echo "$CHANGED_FILES" | grep 'build_info\.json' || true) + if [ -n "$BUILD_INFO_CHANGED" ] || [ -n "$SH_SCRIPT_CHANGED" ]; then + echo "build_package_enabled=true" >> $GITHUB_OUTPUT + echo "✅ Build package job will run (build_info.json or .sh scripts changed)" + else + echo "build_package_enabled=false" >> $GITHUB_OUTPUT + echo "⏭️ Build package job will be skipped (no build_info.json or .sh script changes)" + fi + - name: Create scanner-env.sh run: | mkdir -p package-cache @@ -165,6 +285,7 @@ jobs: build: needs: build_info + if: needs.build_info.outputs.build_package_enabled == 'true' runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} steps: @@ -191,16 +312,16 @@ jobs: cat package-cache/scanner-env.sh sudo chown -R $USER:$USER . - chmod +x ./gha-script/build_package.sh - bash ./gha-script/build_package.sh + echo "------------------- Executing changed scripts -----------------------------" + chmod +x ./gha-script/execute_changed_scripts.py + python3 ./gha-script/execute_changed_scripts.py # ===================== WHEEL JOBS ===================== - wheel_build_py310: needs: build_info - if: ${{ success() }} + if: needs.build_info.outputs.wheel_build_enabled == 'true' runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} continue-on-error: false env: @@ -264,7 +385,7 @@ jobs: wheel_build_py311: needs: build_info - if: ${{ success() }} + if: needs.build_info.outputs.wheel_build_enabled == 'true' runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} continue-on-error: false env: @@ -326,10 +447,9 @@ jobs: sudo uname -a - wheel_build_py312: needs: build_info - if: ${{ success() }} + if: needs.build_info.outputs.wheel_build_enabled == 'true' runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} continue-on-error: false env: @@ -390,9 +510,10 @@ jobs: sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" sudo uname -a + wheel_build_py313: needs: build_info - if: ${{ success() }} + if: needs.build_info.outputs.wheel_build_enabled == 'true' runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} continue-on-error: true env: @@ -456,7 +577,7 @@ jobs: wheel_build_py314: needs: build_info - if: ${{ success() }} + if: needs.build_info.outputs.wheel_build_enabled == 'true' runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} continue-on-error: true env: @@ -518,10 +639,9 @@ jobs: sudo uname -a - build_docker: needs: build_info - if: ${{ success() }} + if: needs.build_info.outputs.docker_build_enabled == 'true' runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} steps: From ab1a9182b4be4d88ca666b43a3ea95dd8415ac77 Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Mon, 6 Jul 2026 02:21:56 +0530 Subject: [PATCH 02/13] Update validate_builds.py --- gha-script/validate_builds.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/gha-script/validate_builds.py b/gha-script/validate_builds.py index 54094dcce1..f2dc81acf3 100644 --- a/gha-script/validate_builds.py +++ b/gha-script/validate_builds.py @@ -315,14 +315,20 @@ def trigger_build_validation_ci(pr_number): # perform basic validation check trigger_basic_validation_checks(file_name) - #check ci-check from package header + #check ci-check from package header ci_check=package_data['ci_check'].lower() - if ci_check=="true": - # Build/test script files - trigger_script_validation_checks(file_name) - else: - print("Skipping Build script validation for {} as CI-Check flag is set to False".format(file_name)) + # COMMENTED OUT: Script execution moved to build job via execute_changed_scripts.py + # This avoids duplicate execution - validation happens here, actual build happens in build job + # if ci_check=="true": + # # Build/test script files + # trigger_script_validation_checks(file_name) + # else: + # print("Skipping Build script validation for {} as CI-Check flag is set to False".format(file_name)) + + print("✅ Basic validation passed for {} (headers, license, line endings)".format(file_name)) + print(" Script execution will happen in build job with proper version parameter") + # Keep track of validated files. validated_file_list.append(file_name) elif file_name.lower().endswith('build_info.json') and status != "removed": From 03ae057c9183abadf8a8c0a737fc96be5fdcadcf Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Mon, 6 Jul 2026 02:22:46 +0530 Subject: [PATCH 03/13] Create execute_changed_scripts.py --- gha-script/execute_changed_scripts.py | 155 ++++++++++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 gha-script/execute_changed_scripts.py diff --git a/gha-script/execute_changed_scripts.py b/gha-script/execute_changed_scripts.py new file mode 100644 index 0000000000..25baa72a4a --- /dev/null +++ b/gha-script/execute_changed_scripts.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 + +import os +import sys + + +def extract_script_metadata(script_path): + """ + Extract metadata from script header comments. + Returns dict with package_version, tested_on, ci_check, etc. + """ + metadata = { + 'package_version': None, + 'tested_on': 'UBI:9.3', # default + 'ci_check': 'true', # default + 'package_name': None, + 'use_non_root_user': 'false' # default + } + + key_mapping = { + '# Package': 'package_name', + '# Version': 'package_version', + '# Tested on': 'tested_on', + '# Ci-Check': 'ci_check', + '# Use Non-Root User': 'use_non_root_user' + } + + try: + with open(script_path, 'r') as f: + for line in f: + line = line.strip() + for key, field in key_mapping.items(): + if line.startswith(key): + value = line.split(':', 1)[-1].strip() + metadata[field] = value + break + except Exception as e: + print(f"Error reading script {script_path}: {e}") + return None + + return metadata + + +def get_changed_scripts(): + """ + Get list of changed .sh scripts from CHANGED_FILES environment variable. + """ + changed_files = os.environ.get('CHANGED_FILES', '') + if not changed_files: + print("No CHANGED_FILES environment variable found") + return [] + + scripts = [] + for line in changed_files.split('\n'): + line = line.strip() + if line.endswith('.sh') and 'dockerfile' not in line.lower(): + scripts.append(line) + + return scripts + + +def main(): + print("=" * 80) + print("EXECUTING CHANGED BUILD SCRIPTS") + print("=" * 80) + + # Get changed scripts + changed_scripts = get_changed_scripts() + + if not changed_scripts: + print("No .sh scripts found in changed files") + print("Skipping script execution") + return 0 + + print(f"\nFound {len(changed_scripts)} changed script(s):") + for script in changed_scripts: + print(f" - {script}") + print() + + # Process each script + executed_count = 0 + skipped_count = 0 + + for script_path in changed_scripts: + print("=" * 80) + print(f"Processing: {script_path}") + print("=" * 80) + + # Check if script exists + if not os.path.exists(script_path): + print(f"⚠️ Script not found: {script_path}") + continue + + # Extract metadata from script header + metadata = extract_script_metadata(script_path) + if not metadata: + print(f"❌ Failed to extract metadata from {script_path}") + continue + + version = metadata['package_version'] + ci_check = metadata['ci_check'].lower() + tested_on = metadata['tested_on'] + use_non_root = metadata['use_non_root_user'].lower() + + print(f"📋 Metadata extracted:") + print(f" Package: {metadata['package_name']}") + print(f" Version: {version}") + print(f" Tested on: {tested_on}") + print(f" CI-Check: {ci_check}") + print(f" Use Non-Root: {use_non_root}") + + # Check CI-Check flag + if ci_check != "true": + print(f"⏭️ Skipping {script_path} - CI-Check flag is set to {ci_check}") + skipped_count += 1 + continue + + if not version: + print(f"⚠️ No version found in script header, skipping {script_path}") + skipped_count += 1 + continue + + # Set environment variables for build_package.sh + os.environ['PKG_DIR_PATH'] = os.path.dirname(script_path) + '/' + os.environ['BUILD_SCRIPT'] = os.path.basename(script_path) + os.environ['VERSION'] = version + os.environ['TESTED_ON'] = tested_on + os.environ['NON_ROOT_BUILD'] = use_non_root + + print(f"\n🚀 Executing: {script_path} with version {version}") + print(f" Command: bash gha-script/build_package.sh") + + # Execute build_package.sh which will run the script + exit_code = os.system('bash gha-script/build_package.sh') + + if exit_code != 0: + print(f"\n❌ Script execution failed for {script_path} with exit code {exit_code}") + return exit_code + else: + print(f"\n✅ Script execution completed successfully for {script_path}") + executed_count += 1 + + print("\n" + "=" * 80) + print("EXECUTION SUMMARY") + print("=" * 80) + print(f"✅ Executed: {executed_count}") + print(f"⏭️ Skipped: {skipped_count}") + print(f"📊 Total: {len(changed_scripts)}") + print("=" * 80) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 91d0fa350830b53fc1ee7382587e938d4016e58c Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Mon, 6 Jul 2026 02:23:38 +0530 Subject: [PATCH 04/13] Update validate_builds.py --- gha-script/validate_builds.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gha-script/validate_builds.py b/gha-script/validate_builds.py index f2dc81acf3..031009426c 100644 --- a/gha-script/validate_builds.py +++ b/gha-script/validate_builds.py @@ -10,7 +10,7 @@ -GITHUB_BUILD_SCRIPT_BASE_REPO = "build-scripts" +GITHUB_BUILD_SCRIPT_BASE_REPO = "stutiibm" GITHUB_BUILD_SCRIPT_BASE_OWNER = "ppc64le" HOME = os.getcwd() From 8eeb3059146f36a6f8fb75cc74cd73e6373b6f42 Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Mon, 6 Jul 2026 02:23:54 +0530 Subject: [PATCH 05/13] Update download_wheels.sh --- gha-script/download-scripts/download_wheels.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gha-script/download-scripts/download_wheels.sh b/gha-script/download-scripts/download_wheels.sh index 8830015303..c7c90017ae 100644 --- a/gha-script/download-scripts/download_wheels.sh +++ b/gha-script/download-scripts/download_wheels.sh @@ -11,7 +11,7 @@ token_request=$(curl -s -X POST https://iam.cloud.ibm.com/identity/token \ if [[ $(echo "$token_request" | jq -r '.errorCode') == "null" ]]; then token=$(echo "$token_request" | jq -r '.access_token') - bucket_url="https://s3.us.cloud-object-storage.appdomain.cloud/ose-power-artifacts-production" + bucket_url="https://s3.us.cloud-object-storage.appdomain.cloud/ose-power-artifacts-stag" echo "Fetching wheel list from COS..." echo "Checking prefix: $PACKAGE_NAME/$VERSION/" From beea9240a10bd64c8595bb0626b50fdcf74c92b8 Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Mon, 6 Jul 2026 02:24:10 +0530 Subject: [PATCH 06/13] Update download_file.sh --- gha-script/download-scripts/download_file.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gha-script/download-scripts/download_file.sh b/gha-script/download-scripts/download_file.sh index 5db90bc478..7d706493ae 100644 --- a/gha-script/download-scripts/download_file.sh +++ b/gha-script/download-scripts/download_file.sh @@ -9,7 +9,7 @@ if [[ $(echo "$token_request" | jq -r '.errorCode') == "null" ]]; then token=$(echo "$token_request" | jq -r '.access_token') echo "Downloading $1 from COS..." curl -X GET -H "Authorization: bearer $token" \ - -o $1 "https://s3.us.cloud-object-storage.appdomain.cloud/ose-power-toolci-bucket-production/$PACKAGE_NAME/$VERSION/$1" + -o $1 "https://s3.us.cloud-object-storage.appdomain.cloud/ose-power-toolci-bucket-stag/$PACKAGE_NAME/$VERSION/$1" echo "Download completed: $1" else echo "Error: Token request failed. Response: $token_request" From 4001b60a8d90b41c9737d4ac516686bd635fc52e Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Mon, 6 Jul 2026 02:25:01 +0530 Subject: [PATCH 07/13] Update upload_wheel.sh --- gha-script/upload-scripts/upload_wheel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gha-script/upload-scripts/upload_wheel.sh b/gha-script/upload-scripts/upload_wheel.sh index cc1e8f1e27..9f56c10392 100644 --- a/gha-script/upload-scripts/upload_wheel.sh +++ b/gha-script/upload-scripts/upload_wheel.sh @@ -27,7 +27,7 @@ if [[ $(echo "$token_request" | jq -r '.errorCode') == "null" ]]; then token=$(echo "$token_request" | jq -r '.access_token') # curl command for uploading the file - #response=$(curl -X PUT -H "Authorization: bearer $token" -H "Content-Type: application/octet-stream" -T $1 "https://s3.us.cloud-object-storage.appdomain.cloud/ose-power-artifacts-prod/$PACKAGE_NAME/$VERSION/$1") + #response=$(curl -X PUT -H "Authorization: bearer $token" -H "Content-Type: application/octet-stream" -T $1 "https://s3.us.cloud-object-storage.appdomain.cloud/ose-power-artifacts-stag/$PACKAGE_NAME/$VERSION/$1") response=$(curl -X PUT \ -H "Authorization: bearer $token" \ -H "Content-Type: application/octet-stream" \ From f4008a76323be4ab9964d1712551dc8313102964 Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Mon, 6 Jul 2026 02:25:14 +0530 Subject: [PATCH 08/13] Update upload_file.sh --- gha-script/upload-scripts/upload_file.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gha-script/upload-scripts/upload_file.sh b/gha-script/upload-scripts/upload_file.sh index 8a676f486f..31cebb26b0 100644 --- a/gha-script/upload-scripts/upload_file.sh +++ b/gha-script/upload-scripts/upload_file.sh @@ -14,7 +14,7 @@ if [[ $(echo "$token_request" | jq -r '.errorCode') == "null" ]]; then # curl command for uploading the file #response=$(curl -X PUT -H "Authorization: bearer $token" -H "Content-Type: application/gzip" -T $1 "https://s3.us.cloud-object-storage.appdomain.cloud/ose-power-toolci-bucket-prod/$PACKAGE_NAME/$VERSION/$1") - response=$(curl -X PUT -H "Authorization: bearer $token" -H "Content-Type: application/gzip" -T $1 "https://s3.us.cloud-object-storage.appdomain.cloud/ose-power-toolci-bucket-production/$PACKAGE_NAME/$VERSION/$1") + response=$(curl -X PUT -H "Authorization: bearer $token" -H "Content-Type: application/gzip" -T $1 "https://s3.us.cloud-object-storage.appdomain.cloud/ose-power-toolci-bucket-stag/$PACKAGE_NAME/$VERSION/$1") # Check if the PUT request was successful based on the absence of an block if ! echo "$response" | grep -q ""; then From 6266c68aacc6b8015b80807ce665c59a4c007c8a Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Mon, 6 Jul 2026 02:38:59 +0530 Subject: [PATCH 09/13] Update pr-build.yaml --- .github/workflows/pr-build.yaml | 527 +------------------------------- 1 file changed, 8 insertions(+), 519 deletions(-) diff --git a/.github/workflows/pr-build.yaml b/.github/workflows/pr-build.yaml index 4e47c2c818..ecf51cd83a 100644 --- a/.github/workflows/pr-build.yaml +++ b/.github/workflows/pr-build.yaml @@ -38,7 +38,8 @@ jobs: uses: actions/checkout@v6 with: fetch-depth: 0 - ref: ${{ github.event.pull_request.head.sha }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.ref }} - name: Fix workspace permissions run: sudo chown -R $USER:$USER . @@ -52,19 +53,16 @@ jobs: exit 0 fi - # Fetch base branch with full history git fetch origin ${{ github.base_ref }} CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD) echo "Changed files:" echo "$CHANGED_FILES" - # Store changed files for reuse in later jobs echo "changed_files<> $GITHUB_OUTPUT echo "$CHANGED_FILES" >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT - # Check for build_info.json, .sh scripts (excluding specific directories), or Dockerfile RELEVANT_CHANGES=$(echo "$CHANGED_FILES" | grep -E '(build_info\.json|\.sh$|Dockerfile)' | grep -vE '^(gha-script|process_bom|script|templates|travis-currency-ymls|travis-ymls)/' || true) if [ -n "$RELEVANT_CHANGES" ]; then @@ -93,7 +91,8 @@ jobs: uses: actions/checkout@v6 with: fetch-depth: 0 - ref: ${{ github.event.pull_request.head.sha }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.ref }} - name: Checkout code (Workflow Dispatch) if: github.event_name == 'workflow_dispatch' @@ -129,8 +128,6 @@ jobs: run: echo "PR_NUMBER=${{ github.event.pull_request.number || inputs.pr_number }}" >> $GITHUB_ENV - name: Run validate_builds.py script with live logs - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | python3 -u gha-script/validate_builds.py ${PR_NUMBER:-false} 2>&1 | tee build_log my_pid_status=${PIPESTATUS[0]} @@ -145,12 +142,13 @@ jobs: echo "Script completed successfully for PR #${PR_NUMBER}" fi + - name: Fetch base branch + run: git fetch origin ${{ github.base_ref }} --depth=1 + - name: Locate and parse build_info.json run: | - # Reuse changed files from check_changes job CHANGED_FILES="${{ needs.check_changes.outputs.changed_files }}" - # If workflow_dispatch, check_changes did not compute changed_files — fetch them now if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then git fetch origin ${{ github.base_ref }} --depth=1 CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD) @@ -160,16 +158,12 @@ jobs: if [ -z "$BUILD_INFO_FILE" ]; then echo "No build_info.json modified, trying to detect from changed files..." - PACKAGE_DIR=$(echo "$CHANGED_FILES" | head -n 1 | cut -d'/' -f1-2) - BUILD_INFO_FILE="$PACKAGE_DIR/build_info.json" - if [ ! -f "$BUILD_INFO_FILE" ]; then echo "Could not locate build_info.json!" exit 1 fi - echo "Using fallback build_info: $BUILD_INFO_FILE" fi @@ -189,509 +183,4 @@ jobs: chmod +x ./gha-script/read_buildinfo.sh bash ./gha-script/read_buildinfo.sh - - name: Set job control flags - id: set_flags - run: | - PACKAGE_DIR=$(jq -r '.package_dir // ""' $BUILD_INFO_FILE) - WHEEL_BUILD=$(jq -r '.wheel_build // "false"' $BUILD_INFO_FILE) - - # Robust docker_build extraction - if jq -e '.docker_build == true or .docker_build == "true"' "$BUILD_INFO_FILE" > /dev/null; then - DOCKER_BUILD="true" - else - DOCKER_BUILD="false" - fi - - # Check if .sh scripts changed in this package - SH_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) - - # Check if Dockerfile changed - DOCKERFILE_CHANGED=$(echo "$CHANGED_FILES" | grep -i 'Dockerfile' || true) - - # Set outputs for wheel builds - if [ "$WHEEL_BUILD" == "true" ] && [ -n "$SH_SCRIPT_CHANGED" ]; then - echo "wheel_build_enabled=true" >> $GITHUB_OUTPUT - echo "✅ Wheel builds will run (WHEEL_BUILD=true and .sh scripts changed)" - else - echo "wheel_build_enabled=false" >> $GITHUB_OUTPUT - echo "⏭️ Wheel builds will be skipped (WHEEL_BUILD=$WHEEL_BUILD, .sh changes: ${SH_SCRIPT_CHANGED:-none})" - fi - - # Set output for sh changes - if [ -n "$SH_SCRIPT_CHANGED" ]; then - echo "has_sh_changes=true" >> $GITHUB_OUTPUT - else - echo "has_sh_changes=false" >> $GITHUB_OUTPUT - fi - - # Set outputs for docker build - if [ "$DOCKER_BUILD" == "true" ] && [ -n "$DOCKERFILE_CHANGED" ]; then - echo "docker_build_enabled=true" >> $GITHUB_OUTPUT - echo "has_dockerfile_changes=true" >> $GITHUB_OUTPUT - echo "✅ Docker build will run (BUILD_DOCKER=true and Dockerfile changed)" - else - echo "docker_build_enabled=false" >> $GITHUB_OUTPUT - if [ -n "$DOCKERFILE_CHANGED" ]; then - echo "has_dockerfile_changes=true" >> $GITHUB_OUTPUT - else - echo "has_dockerfile_changes=false" >> $GITHUB_OUTPUT - fi - echo "⏭️ Docker build will be skipped (BUILD_DOCKER=$DOCKER_BUILD, Dockerfile changes: ${DOCKERFILE_CHANGED:-none})" - fi - - # Set output for build job (runs when build_info.json or .sh scripts change) - BUILD_INFO_CHANGED=$(echo "$CHANGED_FILES" | grep 'build_info\.json' || true) - if [ -n "$BUILD_INFO_CHANGED" ] || [ -n "$SH_SCRIPT_CHANGED" ]; then - echo "build_package_enabled=true" >> $GITHUB_OUTPUT - echo "✅ Build package job will run (build_info.json or .sh scripts changed)" - else - echo "build_package_enabled=false" >> $GITHUB_OUTPUT - echo "⏭️ Build package job will be skipped (no build_info.json or .sh script changes)" - fi - - - name: Create scanner-env.sh - run: | - mkdir -p package-cache - - PACKAGE_DIR=$(jq -r '.package_dir // ""' $BUILD_INFO_FILE) - WHEEL_BUILD=$(jq -r '.wheel_build // "false"' $BUILD_INFO_FILE) - - # Robust docker_build extraction - if jq -e '.docker_build == true or .docker_build == "true"' "$BUILD_INFO_FILE" > /dev/null; then - DOCKER_BUILD="true" - else - DOCKER_BUILD="false" - fi - - cat < package-cache/scanner-env.sh - export PACKAGE_NAME=$PACKAGE_NAME - export VERSION=$VERSION - export PACKAGE_DIR=$PACKAGE_DIR - export WHEEL_BUILD=$WHEEL_BUILD - export BUILD_DOCKER=$DOCKER_BUILD - export CHANGED_FILES="$CHANGED_FILES" - EOF - - mv variable.sh package-cache/ - - - name: Archive package cache - run: tar -czf package-cache.tar.gz package-cache/ - - - name: Upload package cache - uses: actions/upload-artifact@v6 - with: - name: package-cache - path: package-cache.tar.gz - - build: - needs: build_info - if: needs.build_info.outputs.build_package_enabled == 'true' - runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} - - steps: - - uses: actions/checkout@v6 - with: - ref: ${{ github.event.pull_request.head.sha }} - - - name: Download package-cache - uses: actions/download-artifact@v7 - with: - name: package-cache - - - name: Extract package cache - run: tar -xzf package-cache.tar.gz - - - name: Build Package - run: | - source package-cache/variable.sh - source package-cache/scanner-env.sh - - echo "------------------- variable.sh -----------------------------" - cat package-cache/variable.sh - echo "------------------- scanner-env.sh -----------------------------" - cat package-cache/scanner-env.sh - - sudo chown -R $USER:$USER . - echo "------------------- Executing changed scripts -----------------------------" - chmod +x ./gha-script/execute_changed_scripts.py - python3 ./gha-script/execute_changed_scripts.py - -# ===================== WHEEL JOBS ===================== - - - wheel_build_py310: - needs: build_info - if: needs.build_info.outputs.wheel_build_enabled == 'true' - runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} - continue-on-error: false - env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} - GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} - PYTHON_VERSION: "3.10" - - steps: - - name: Checkout code - uses: actions/checkout@v6 - with: - fetch-depth: 0 - ref: ${{ github.event.pull_request.head.sha }} - - - name: Install system dependencies - run: | - sudo apt update -y - - - name: Download package-cache from previous step - uses: actions/download-artifact@v7 - with: - name: package-cache - - - name: Extract package cache - run: tar -xzf package-cache.tar.gz - - - name: Run build_wheels.sh - run: | - ls - echo "---------------------updated cache-----------------------" - ls package-cache - - chmod +x package-cache/variable.sh - chmod +x package-cache/scanner-env.sh - source package-cache/variable.sh - source package-cache/scanner-env.sh - - # CONTROL FLAG HERE - if [ "$WHEEL_BUILD" != "true" ]; then - echo "Skipping wheel build as WHEEL_BUILD=false" - exit 0 - fi - - # Check if any .sh build script is modified in this package - BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) - - if [[ -z "$BUILD_SCRIPT_CHANGED" ]]; then - echo "Skipping wheel build as no .sh build script changes detected" - exit 0 - fi - - sudo chown -R $USER:$USER . - chmod +x ./gha-script/build_wheels.sh - bash ./gha-script/build_wheels.sh - - echo "===========after execution ==================" - sudo apt update -y - sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" - sudo uname -a - - - wheel_build_py311: - needs: build_info - if: needs.build_info.outputs.wheel_build_enabled == 'true' - runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} - continue-on-error: false - env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} - GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} - PYTHON_VERSION: "3.11" - - steps: - - name: Checkout code - uses: actions/checkout@v6 - with: - fetch-depth: 0 - ref: ${{ github.event.pull_request.head.sha }} - - - name: Install system dependencies - run: | - sudo apt update -y - - - name: Download package-cache from previous step - uses: actions/download-artifact@v7 - with: - name: package-cache - - - name: Extract package cache - run: tar -xzf package-cache.tar.gz - - - name: Run build_wheels.sh - run: | - ls - echo "---------------------updated cache-----------------------" - ls package-cache - - chmod +x package-cache/variable.sh - chmod +x package-cache/scanner-env.sh - source package-cache/variable.sh - source package-cache/scanner-env.sh - - # CONTROL FLAG HERE - if [ "$WHEEL_BUILD" != "true" ]; then - echo "Skipping wheel build as WHEEL_BUILD=false" - exit 0 - fi - - # Check if any .sh build script is modified in this package - BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) - - if [[ -z "$BUILD_SCRIPT_CHANGED" ]]; then - echo "Skipping wheel build as no .sh build script changes detected" - exit 0 - fi - - sudo chown -R $USER:$USER . - chmod +x ./gha-script/build_wheels.sh - bash ./gha-script/build_wheels.sh - - echo "===========after execution ==================" - sudo apt update -y - sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" - sudo uname -a - - - wheel_build_py312: - needs: build_info - if: needs.build_info.outputs.wheel_build_enabled == 'true' - runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} - continue-on-error: false - env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} - GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} - PYTHON_VERSION: "3.12" - - steps: - - name: Checkout code - uses: actions/checkout@v6 - with: - fetch-depth: 0 - ref: ${{ github.event.pull_request.head.sha }} - - - name: Install system dependencies - run: | - sudo apt update -y - - - name: Download package-cache from previous step - uses: actions/download-artifact@v7 - with: - name: package-cache - - - name: Extract package cache - run: tar -xzf package-cache.tar.gz - - - name: Run build_wheels.sh - run: | - ls - echo "---------------------updated cache-----------------------" - ls package-cache - - chmod +x package-cache/variable.sh - chmod +x package-cache/scanner-env.sh - source package-cache/variable.sh - source package-cache/scanner-env.sh - - # CONTROL FLAG HERE - if [ "$WHEEL_BUILD" != "true" ]; then - echo "Skipping wheel build as WHEEL_BUILD=false" - exit 0 - fi - - # Check if any .sh build script is modified in this package - BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) - - if [[ -z "$BUILD_SCRIPT_CHANGED" ]]; then - echo "Skipping wheel build as no .sh build script changes detected" - exit 0 - fi - - sudo chown -R $USER:$USER . - chmod +x ./gha-script/build_wheels.sh - bash ./gha-script/build_wheels.sh - - echo "===========after execution ==================" - sudo apt update -y - sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" - sudo uname -a - - - wheel_build_py313: - needs: build_info - if: needs.build_info.outputs.wheel_build_enabled == 'true' - runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} - continue-on-error: true - env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} - GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} - PYTHON_VERSION: "3.13" - - steps: - - name: Checkout code - uses: actions/checkout@v6 - with: - fetch-depth: 0 - ref: ${{ github.event.pull_request.head.sha }} - - - name: Install system dependencies - run: | - sudo apt update -y - - - name: Download package-cache from previous step - uses: actions/download-artifact@v7 - with: - name: package-cache - - - name: Extract package cache - run: tar -xzf package-cache.tar.gz - - - name: Run build_wheels.sh - run: | - ls - echo "---------------------updated cache-----------------------" - ls package-cache - - chmod +x package-cache/variable.sh - chmod +x package-cache/scanner-env.sh - source package-cache/variable.sh - source package-cache/scanner-env.sh - - # CONTROL FLAG HERE - if [ "$WHEEL_BUILD" != "true" ]; then - echo "Skipping wheel build as WHEEL_BUILD=false" - exit 0 - fi - - # Check if any .sh build script is modified in this package - BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) - - if [[ -z "$BUILD_SCRIPT_CHANGED" ]]; then - echo "Skipping wheel build as no .sh build script changes detected" - exit 0 - fi - - sudo chown -R $USER:$USER . - chmod +x ./gha-script/build_wheels.sh - bash ./gha-script/build_wheels.sh - - echo "===========after execution ==================" - sudo apt update -y - sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" - sudo uname -a - - - wheel_build_py314: - needs: build_info - if: needs.build_info.outputs.wheel_build_enabled == 'true' - runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} - continue-on-error: true - env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} - GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} - PYTHON_VERSION: "3.14" - - steps: - - name: Checkout code - uses: actions/checkout@v6 - with: - fetch-depth: 0 - ref: ${{ github.event.pull_request.head.sha }} - - - name: Install system dependencies - run: | - sudo apt update -y - - - name: Download package-cache from previous step - uses: actions/download-artifact@v7 - with: - name: package-cache - - - name: Extract package cache - run: tar -xzf package-cache.tar.gz - - - name: Run build_wheels.sh - run: | - ls - echo "---------------------updated cache-----------------------" - ls package-cache - - chmod +x package-cache/variable.sh - chmod +x package-cache/scanner-env.sh - source package-cache/variable.sh - source package-cache/scanner-env.sh - - # CONTROL FLAG HERE - if [ "$WHEEL_BUILD" != "true" ]; then - echo "Skipping wheel build as WHEEL_BUILD=false" - exit 0 - fi - - # Check if any .sh build script is modified in this package - BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) - - if [[ -z "$BUILD_SCRIPT_CHANGED" ]]; then - echo "Skipping wheel build as no .sh build script changes detected" - exit 0 - fi - - sudo chown -R $USER:$USER . - chmod +x ./gha-script/build_wheels.sh - bash ./gha-script/build_wheels.sh - - echo "===========after execution ==================" - sudo apt update -y - sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" - sudo uname -a - - - build_docker: - needs: build_info - if: needs.build_info.outputs.docker_build_enabled == 'true' - runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} - - steps: - - name: Checkout code - uses: actions/checkout@v6 - with: - ref: ${{ github.event.pull_request.head.sha }} - - - name: Download package-cache - uses: actions/download-artifact@v7 - with: - name: package-cache - - - name: Extract package cache - run: tar -xzf package-cache.tar.gz - - - name: Build Docker Image - run: | - echo "===== Sourcing environment =====" - ls package-cache - - chmod +x package-cache/variable.sh - chmod +x package-cache/scanner-env.sh - source package-cache/variable.sh - source package-cache/scanner-env.sh - - BUILD_DOCKER=$(echo "$BUILD_DOCKER" | tr -d '[:space:]' | tr '[:upper:]' '[:lower:]') - DOCKERFILE_CHANGED=$(echo "$CHANGED_FILES" | grep -i 'Dockerfile' || true) - - if [[ "$BUILD_DOCKER" != "true" ]]; then - echo "Skipping Docker build as BUILD_DOCKER=$BUILD_DOCKER" - exit 0 - fi - - if [[ -z "$DOCKERFILE_CHANGED" ]]; then - echo "Skipping Docker build as no Dockerfile changes detected in PR" - exit 0 - fi - - echo "Dockerfile change detected, proceeding with build..." - - echo "===== Starting Docker build =====" - sudo chown -R $USER:$USER . - chmod +x ./gha-script/build_docker.sh - bash ./gha-script/build_docker.sh - - echo "===== Docker images after build =====" - docker images - - echo "===== Saving Docker image =====" - docker save -o package-cache/image.tar "$IMAGE_NAME" - - ls -lh package-cache/image.tar + # ... (set_flags, Create scanner-env.sh, Archive, Upload — identical to main repo) From 905ee616c31e182c66ef195112a88c2f27a46bdf Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Mon, 6 Jul 2026 02:58:47 +0530 Subject: [PATCH 10/13] Update pr-build.yaml --- .github/workflows/pr-build.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pr-build.yaml b/.github/workflows/pr-build.yaml index ecf51cd83a..afbf4e6df7 100644 --- a/.github/workflows/pr-build.yaml +++ b/.github/workflows/pr-build.yaml @@ -128,6 +128,8 @@ jobs: run: echo "PR_NUMBER=${{ github.event.pull_request.number || inputs.pr_number }}" >> $GITHUB_ENV - name: Run validate_builds.py script with live logs + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | python3 -u gha-script/validate_builds.py ${PR_NUMBER:-false} 2>&1 | tee build_log my_pid_status=${PIPESTATUS[0]} From 1021485348c335ce1155750404e07912d5b81538 Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Mon, 6 Jul 2026 02:59:35 +0530 Subject: [PATCH 11/13] Update validate_builds.py --- gha-script/validate_builds.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gha-script/validate_builds.py b/gha-script/validate_builds.py index 031009426c..4654edf590 100644 --- a/gha-script/validate_builds.py +++ b/gha-script/validate_builds.py @@ -10,8 +10,8 @@ -GITHUB_BUILD_SCRIPT_BASE_REPO = "stutiibm" -GITHUB_BUILD_SCRIPT_BASE_OWNER = "ppc64le" +GITHUB_BUILD_SCRIPT_BASE_REPO = "build-scripts" +GITHUB_BUILD_SCRIPT_BASE_OWNER = "stutiibm" HOME = os.getcwd() package_data = {} From 5783cdc0f565c913366d4780febff9d3e158a27e Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Mon, 6 Jul 2026 11:40:39 +0530 Subject: [PATCH 12/13] Update pr-build.yaml --- .github/workflows/pr-build.yaml | 525 +++++++++++++++++++++++++++++++- 1 file changed, 524 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-build.yaml b/.github/workflows/pr-build.yaml index afbf4e6df7..8cc9de9ed5 100644 --- a/.github/workflows/pr-build.yaml +++ b/.github/workflows/pr-build.yaml @@ -53,16 +53,19 @@ jobs: exit 0 fi + # Fetch base branch with full history git fetch origin ${{ github.base_ref }} CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD) echo "Changed files:" echo "$CHANGED_FILES" + # Store changed files for reuse in later jobs echo "changed_files<> $GITHUB_OUTPUT echo "$CHANGED_FILES" >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT + # Check for build_info.json, .sh scripts (excluding specific directories), or Dockerfile RELEVANT_CHANGES=$(echo "$CHANGED_FILES" | grep -E '(build_info\.json|\.sh$|Dockerfile)' | grep -vE '^(gha-script|process_bom|script|templates|travis-currency-ymls|travis-ymls)/' || true) if [ -n "$RELEVANT_CHANGES" ]; then @@ -149,8 +152,10 @@ jobs: - name: Locate and parse build_info.json run: | + # Reuse changed files from check_changes job CHANGED_FILES="${{ needs.check_changes.outputs.changed_files }}" + # If workflow_dispatch, check_changes did not compute changed_files — fetch them now if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then git fetch origin ${{ github.base_ref }} --depth=1 CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD) @@ -160,12 +165,16 @@ jobs: if [ -z "$BUILD_INFO_FILE" ]; then echo "No build_info.json modified, trying to detect from changed files..." + PACKAGE_DIR=$(echo "$CHANGED_FILES" | head -n 1 | cut -d'/' -f1-2) + BUILD_INFO_FILE="$PACKAGE_DIR/build_info.json" + if [ ! -f "$BUILD_INFO_FILE" ]; then echo "Could not locate build_info.json!" exit 1 fi + echo "Using fallback build_info: $BUILD_INFO_FILE" fi @@ -185,4 +194,518 @@ jobs: chmod +x ./gha-script/read_buildinfo.sh bash ./gha-script/read_buildinfo.sh - # ... (set_flags, Create scanner-env.sh, Archive, Upload — identical to main repo) + - name: Set job control flags + id: set_flags + run: | + PACKAGE_DIR=$(jq -r '.package_dir // ""' $BUILD_INFO_FILE) + # Strip trailing slash if present — prevents double-slash in regex (e.g. "m/multidict/" → "m/multidict") + PACKAGE_DIR="${PACKAGE_DIR%/}" + WHEEL_BUILD=$(jq -r '.wheel_build // "false"' $BUILD_INFO_FILE) + + # Robust docker_build extraction + if jq -e '.docker_build == true or .docker_build == "true"' "$BUILD_INFO_FILE" > /dev/null; then + DOCKER_BUILD="true" + else + DOCKER_BUILD="false" + fi + + # Check if .sh scripts changed in this package + SH_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) + + # Check if Dockerfile changed + DOCKERFILE_CHANGED=$(echo "$CHANGED_FILES" | grep -i 'Dockerfile' || true) + + # Set outputs for wheel builds + if [ "$WHEEL_BUILD" == "true" ] && [ -n "$SH_SCRIPT_CHANGED" ]; then + echo "wheel_build_enabled=true" >> $GITHUB_OUTPUT + echo "✅ Wheel builds will run (WHEEL_BUILD=true and .sh scripts changed)" + else + echo "wheel_build_enabled=false" >> $GITHUB_OUTPUT + echo "⏭️ Wheel builds will be skipped (WHEEL_BUILD=$WHEEL_BUILD, .sh changes: ${SH_SCRIPT_CHANGED:-none})" + fi + + # Set output for sh changes + if [ -n "$SH_SCRIPT_CHANGED" ]; then + echo "has_sh_changes=true" >> $GITHUB_OUTPUT + else + echo "has_sh_changes=false" >> $GITHUB_OUTPUT + fi + + # Set outputs for docker build + if [ "$DOCKER_BUILD" == "true" ] && [ -n "$DOCKERFILE_CHANGED" ]; then + echo "docker_build_enabled=true" >> $GITHUB_OUTPUT + echo "has_dockerfile_changes=true" >> $GITHUB_OUTPUT + echo "✅ Docker build will run (BUILD_DOCKER=true and Dockerfile changed)" + else + echo "docker_build_enabled=false" >> $GITHUB_OUTPUT + if [ -n "$DOCKERFILE_CHANGED" ]; then + echo "has_dockerfile_changes=true" >> $GITHUB_OUTPUT + else + echo "has_dockerfile_changes=false" >> $GITHUB_OUTPUT + fi + echo "⏭️ Docker build will be skipped (BUILD_DOCKER=$DOCKER_BUILD, Dockerfile changes: ${DOCKERFILE_CHANGED:-none})" + fi + + # Set output for build job (runs when build_info.json or .sh scripts change) + BUILD_INFO_CHANGED=$(echo "$CHANGED_FILES" | grep 'build_info\.json' || true) + if [ -n "$BUILD_INFO_CHANGED" ] || [ -n "$SH_SCRIPT_CHANGED" ]; then + echo "build_package_enabled=true" >> $GITHUB_OUTPUT + echo "✅ Build package job will run (build_info.json or .sh scripts changed)" + else + echo "build_package_enabled=false" >> $GITHUB_OUTPUT + echo "⏭️ Build package job will be skipped (no build_info.json or .sh script changes)" + fi + + - name: Create scanner-env.sh + run: | + mkdir -p package-cache + + PACKAGE_DIR=$(jq -r '.package_dir // ""' $BUILD_INFO_FILE) + WHEEL_BUILD=$(jq -r '.wheel_build // "false"' $BUILD_INFO_FILE) + + # Robust docker_build extraction + if jq -e '.docker_build == true or .docker_build == "true"' "$BUILD_INFO_FILE" > /dev/null; then + DOCKER_BUILD="true" + else + DOCKER_BUILD="false" + fi + + cat < package-cache/scanner-env.sh + export PACKAGE_NAME=$PACKAGE_NAME + export VERSION=$VERSION + export PACKAGE_DIR=$PACKAGE_DIR + export WHEEL_BUILD=$WHEEL_BUILD + export BUILD_DOCKER=$DOCKER_BUILD + export CHANGED_FILES="$CHANGED_FILES" + EOF + + mv variable.sh package-cache/ + + - name: Archive package cache + run: tar -czf package-cache.tar.gz package-cache/ + + - name: Upload package cache + uses: actions/upload-artifact@v6 + with: + name: package-cache + path: package-cache.tar.gz + + build: + needs: build_info + if: needs.build_info.outputs.build_package_enabled == 'true' + runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} + + steps: + - uses: actions/checkout@v6 + with: + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.ref }} + + - name: Download package-cache + uses: actions/download-artifact@v7 + with: + name: package-cache + + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + + - name: Build Package + run: | + source package-cache/variable.sh + source package-cache/scanner-env.sh + + echo "------------------- variable.sh -----------------------------" + cat package-cache/variable.sh + echo "------------------- scanner-env.sh -----------------------------" + cat package-cache/scanner-env.sh + + sudo chown -R $USER:$USER . + echo "------------------- Executing changed scripts -----------------------------" + chmod +x ./gha-script/execute_changed_scripts.py + python3 ./gha-script/execute_changed_scripts.py + +# ===================== WHEEL JOBS ===================== + + + wheel_build_py310: + needs: build_info + if: needs.build_info.outputs.wheel_build_enabled == 'true' + runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} + continue-on-error: false + env: + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} + PYTHON_VERSION: "3.10" + + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.ref }} + + - name: Install system dependencies + run: | + sudo apt update -y + + - name: Download package-cache from previous step + uses: actions/download-artifact@v7 + with: + name: package-cache + + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + + - name: Run build_wheels.sh + run: | + ls + echo "---------------------updated cache-----------------------" + ls package-cache + + chmod +x package-cache/variable.sh + chmod +x package-cache/scanner-env.sh + source package-cache/variable.sh + source package-cache/scanner-env.sh + + # CONTROL FLAG HERE + if [ "$WHEEL_BUILD" != "true" ]; then + echo "Skipping wheel build as WHEEL_BUILD=false" + exit 0 + fi + + # Check if any .sh build script is modified in this package + BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) + + if [[ -z "$BUILD_SCRIPT_CHANGED" ]]; then + echo "Skipping wheel build as no .sh build script changes detected" + exit 0 + fi + + sudo chown -R $USER:$USER . + chmod +x ./gha-script/build_wheels.sh + bash ./gha-script/build_wheels.sh + + echo "===========after execution ==================" + sudo apt update -y + sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" + sudo uname -a + + + wheel_build_py311: + needs: build_info + if: needs.build_info.outputs.wheel_build_enabled == 'true' + runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} + continue-on-error: false + env: + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} + PYTHON_VERSION: "3.11" + + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.ref }} + + - name: Install system dependencies + run: | + sudo apt update -y + + - name: Download package-cache from previous step + uses: actions/download-artifact@v7 + with: + name: package-cache + + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + + - name: Run build_wheels.sh + run: | + ls + echo "---------------------updated cache-----------------------" + ls package-cache + + chmod +x package-cache/variable.sh + chmod +x package-cache/scanner-env.sh + source package-cache/variable.sh + source package-cache/scanner-env.sh + + # CONTROL FLAG HERE + if [ "$WHEEL_BUILD" != "true" ]; then + echo "Skipping wheel build as WHEEL_BUILD=false" + exit 0 + fi + + # Check if any .sh build script is modified in this package + BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) + + if [[ -z "$BUILD_SCRIPT_CHANGED" ]]; then + echo "Skipping wheel build as no .sh build script changes detected" + exit 0 + fi + + sudo chown -R $USER:$USER . + chmod +x ./gha-script/build_wheels.sh + bash ./gha-script/build_wheels.sh + + echo "===========after execution ==================" + sudo apt update -y + sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" + sudo uname -a + + + wheel_build_py312: + needs: build_info + if: needs.build_info.outputs.wheel_build_enabled == 'true' + runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} + continue-on-error: false + env: + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} + PYTHON_VERSION: "3.12" + + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.ref }} + + - name: Install system dependencies + run: | + sudo apt update -y + + - name: Download package-cache from previous step + uses: actions/download-artifact@v7 + with: + name: package-cache + + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + + - name: Run build_wheels.sh + run: | + ls + echo "---------------------updated cache-----------------------" + ls package-cache + + chmod +x package-cache/variable.sh + chmod +x package-cache/scanner-env.sh + source package-cache/variable.sh + source package-cache/scanner-env.sh + + # CONTROL FLAG HERE + if [ "$WHEEL_BUILD" != "true" ]; then + echo "Skipping wheel build as WHEEL_BUILD=false" + exit 0 + fi + + # Check if any .sh build script is modified in this package + BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) + + if [[ -z "$BUILD_SCRIPT_CHANGED" ]]; then + echo "Skipping wheel build as no .sh build script changes detected" + exit 0 + fi + + sudo chown -R $USER:$USER . + chmod +x ./gha-script/build_wheels.sh + bash ./gha-script/build_wheels.sh + + echo "===========after execution ==================" + sudo apt update -y + sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" + sudo uname -a + + + wheel_build_py313: + needs: build_info + if: needs.build_info.outputs.wheel_build_enabled == 'true' + runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} + continue-on-error: true + env: + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} + PYTHON_VERSION: "3.13" + + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.ref }} + + - name: Install system dependencies + run: | + sudo apt update -y + + - name: Download package-cache from previous step + uses: actions/download-artifact@v7 + with: + name: package-cache + + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + + - name: Run build_wheels.sh + run: | + ls + echo "---------------------updated cache-----------------------" + ls package-cache + + chmod +x package-cache/variable.sh + chmod +x package-cache/scanner-env.sh + source package-cache/variable.sh + source package-cache/scanner-env.sh + + # CONTROL FLAG HERE + if [ "$WHEEL_BUILD" != "true" ]; then + echo "Skipping wheel build as WHEEL_BUILD=false" + exit 0 + fi + + # Check if any .sh build script is modified in this package + BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) + + if [[ -z "$BUILD_SCRIPT_CHANGED" ]]; then + echo "Skipping wheel build as no .sh build script changes detected" + exit 0 + fi + + sudo chown -R $USER:$USER . + chmod +x ./gha-script/build_wheels.sh + bash ./gha-script/build_wheels.sh + + echo "===========after execution ==================" + sudo apt update -y + sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" + sudo uname -a + + + wheel_build_py314: + needs: build_info + if: needs.build_info.outputs.wheel_build_enabled == 'true' + runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} + continue-on-error: true + env: + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} + PYTHON_VERSION: "3.14" + + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.ref }} + + - name: Install system dependencies + run: | + sudo apt update -y + + - name: Download package-cache from previous step + uses: actions/download-artifact@v7 + with: + name: package-cache + + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + + - name: Run build_wheels.sh + run: | + ls + echo "---------------------updated cache-----------------------" + ls package-cache + + chmod +x package-cache/variable.sh + chmod +x package-cache/scanner-env.sh + source package-cache/variable.sh + source package-cache/scanner-env.sh + + # CONTROL FLAG HERE + if [ "$WHEEL_BUILD" != "true" ]; then + echo "Skipping wheel build as WHEEL_BUILD=false" + exit 0 + fi + + # Check if any .sh build script is modified in this package + BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) + + if [[ -z "$BUILD_SCRIPT_CHANGED" ]]; then + echo "Skipping wheel build as no .sh build script changes detected" + exit 0 + fi + + sudo chown -R $USER:$USER . + chmod +x ./gha-script/build_wheels.sh + bash ./gha-script/build_wheels.sh + + echo "===========after execution ==================" + sudo apt update -y + sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" + sudo uname -a + + + build_docker: + needs: build_info + if: needs.build_info.outputs.docker_build_enabled == 'true' + runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} + + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.ref }} + + - name: Download package-cache + uses: actions/download-artifact@v7 + with: + name: package-cache + + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + + - name: Build Docker Image + run: | + echo "===== Sourcing environment =====" + ls package-cache + + chmod +x package-cache/variable.sh + chmod +x package-cache/scanner-env.sh + source package-cache/variable.sh + source package-cache/scanner-env.sh + + BUILD_DOCKER=$(echo "$BUILD_DOCKER" | tr -d '[:space:]' | tr '[:upper:]' '[:lower:]') + DOCKERFILE_CHANGED=$(echo "$CHANGED_FILES" | grep -i 'Dockerfile' || true) + + if [[ "$BUILD_DOCKER" != "true" ]]; then + echo "Skipping Docker build as BUILD_DOCKER=$BUILD_DOCKER" + exit 0 + fi + + if [[ -z "$DOCKERFILE_CHANGED" ]]; then + echo "Skipping Docker build as no Dockerfile changes detected in PR" + exit 0 + fi + + echo "Dockerfile change detected, proceeding with build..." + + echo "===== Starting Docker build =====" + sudo chown -R $USER:$USER . + chmod +x ./gha-script/build_docker.sh + bash ./gha-script/build_docker.sh + + echo "===== Docker images after build =====" + docker images + + echo "===== Saving Docker image =====" + docker save -o package-cache/image.tar "$IMAGE_NAME" + + ls -lh package-cache/image.tar From 199b5802321c349d9f26211be1c27d5cb4135ce7 Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Mon, 6 Jul 2026 18:02:15 +0530 Subject: [PATCH 13/13] Update build_info.json with new build details --- p/param/build_info.json | 1 + 1 file changed, 1 insertion(+) diff --git a/p/param/build_info.json b/p/param/build_info.json index 1dd96f4800..0a613c9a31 100644 --- a/p/param/build_info.json +++ b/p/param/build_info.json @@ -15,3 +15,4 @@ "build_script": "param_ubi_9.3.sh" } } +