diff --git a/.github/workflows/pr-build.yaml b/.github/workflows/pr-build.yaml index 45f3a366f1..8cc9de9ed5 100644 --- a/.github/workflows/pr-build.yaml +++ b/.github/workflows/pr-build.yaml @@ -28,16 +28,74 @@ run-name: > jobs: + check_changes: + runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} + outputs: + should_build: ${{ steps.filter.outputs.should_build }} + changed_files: ${{ steps.filter.outputs.changed_files }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.ref }} + + - name: Fix workspace permissions + run: sudo chown -R $USER:$USER . + + - name: Check for relevant file changes + id: filter + run: | + if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then + echo "should_build=true" >> $GITHUB_OUTPUT + echo "Workflow dispatch - proceeding with build" + exit 0 + fi + + # Fetch base branch with full history + git fetch origin ${{ github.base_ref }} + CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD) + + echo "Changed files:" + echo "$CHANGED_FILES" + + # Store changed files for reuse in later jobs + echo "changed_files<> $GITHUB_OUTPUT + echo "$CHANGED_FILES" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + + # Check for build_info.json, .sh scripts (excluding specific directories), or Dockerfile + RELEVANT_CHANGES=$(echo "$CHANGED_FILES" | grep -E '(build_info\.json|\.sh$|Dockerfile)' | grep -vE '^(gha-script|process_bom|script|templates|travis-currency-ymls|travis-ymls)/' || true) + + if [ -n "$RELEVANT_CHANGES" ]; then + echo "should_build=true" >> $GITHUB_OUTPUT + echo "✅ Found relevant changes:" + echo "$RELEVANT_CHANGES" + else + echo "should_build=false" >> $GITHUB_OUTPUT + echo "⏭️ Skipping PR build CI check - no changes related to build_info.json, build scripts (.sh), or Dockerfile" + fi + build_info: + needs: check_changes + if: needs.check_changes.outputs.should_build == 'true' runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} + outputs: + wheel_build_enabled: ${{ steps.set_flags.outputs.wheel_build_enabled }} + has_sh_changes: ${{ steps.set_flags.outputs.has_sh_changes }} + has_dockerfile_changes: ${{ steps.set_flags.outputs.has_dockerfile_changes }} + docker_build_enabled: ${{ steps.set_flags.outputs.docker_build_enabled }} + build_package_enabled: ${{ steps.set_flags.outputs.build_package_enabled }} steps: - name: Checkout code (Pull Request) if: github.event_name == 'pull_request' uses: actions/checkout@v6 with: - fetch-depth: 0 - ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.ref }} - name: Checkout code (Workflow Dispatch) if: github.event_name == 'workflow_dispatch' @@ -45,16 +103,6 @@ jobs: with: ref: refs/pull/${{ inputs.pr_number }}/head - - name: Install required packages - run: | - sudo apt update -y - sudo apt-get install -y file jq - - - name: Install Python dependencies - run: | - pip3 install --force-reinstall -v "requests==2.31.0" - pip3 install --upgrade docker - - name: Debug - Check user and workspace ownership run: | echo "Current user: $USER" @@ -69,6 +117,15 @@ jobs: - name: Fix workspace permissions run: sudo chown -R $USER:$USER . + - name: Install required packages + run: | + sudo apt update -y + sudo apt-get install -y file jq + + - name: Install Python dependencies + run: | + pip3 install --force-reinstall -v "requests==2.31.0" + pip3 install --upgrade docker - name: Set PR number run: echo "PR_NUMBER=${{ github.event.pull_request.number || inputs.pr_number }}" >> $GITHUB_ENV @@ -90,10 +147,19 @@ jobs: echo "Script completed successfully for PR #${PR_NUMBER}" fi + - name: Fetch base branch + run: git fetch origin ${{ github.base_ref }} --depth=1 + - name: Locate and parse build_info.json run: | + # Reuse changed files from check_changes job + CHANGED_FILES="${{ needs.check_changes.outputs.changed_files }}" - CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD) + # If workflow_dispatch, check_changes did not compute changed_files — fetch them now + if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then + git fetch origin ${{ github.base_ref }} --depth=1 + CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD) + fi BUILD_INFO_FILE=$(echo "$CHANGED_FILES" | grep 'build_info.json' | head -n 1) @@ -122,13 +188,74 @@ jobs: echo "$CHANGED_FILES" >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV - - name: Read build_info.json run: | sudo chown -R $USER:$USER . chmod +x ./gha-script/read_buildinfo.sh bash ./gha-script/read_buildinfo.sh + - name: Set job control flags + id: set_flags + run: | + PACKAGE_DIR=$(jq -r '.package_dir // ""' $BUILD_INFO_FILE) + # Strip trailing slash if present — prevents double-slash in regex (e.g. "m/multidict/" → "m/multidict") + PACKAGE_DIR="${PACKAGE_DIR%/}" + WHEEL_BUILD=$(jq -r '.wheel_build // "false"' $BUILD_INFO_FILE) + + # Robust docker_build extraction + if jq -e '.docker_build == true or .docker_build == "true"' "$BUILD_INFO_FILE" > /dev/null; then + DOCKER_BUILD="true" + else + DOCKER_BUILD="false" + fi + + # Check if .sh scripts changed in this package + SH_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) + + # Check if Dockerfile changed + DOCKERFILE_CHANGED=$(echo "$CHANGED_FILES" | grep -i 'Dockerfile' || true) + + # Set outputs for wheel builds + if [ "$WHEEL_BUILD" == "true" ] && [ -n "$SH_SCRIPT_CHANGED" ]; then + echo "wheel_build_enabled=true" >> $GITHUB_OUTPUT + echo "✅ Wheel builds will run (WHEEL_BUILD=true and .sh scripts changed)" + else + echo "wheel_build_enabled=false" >> $GITHUB_OUTPUT + echo "⏭️ Wheel builds will be skipped (WHEEL_BUILD=$WHEEL_BUILD, .sh changes: ${SH_SCRIPT_CHANGED:-none})" + fi + + # Set output for sh changes + if [ -n "$SH_SCRIPT_CHANGED" ]; then + echo "has_sh_changes=true" >> $GITHUB_OUTPUT + else + echo "has_sh_changes=false" >> $GITHUB_OUTPUT + fi + + # Set outputs for docker build + if [ "$DOCKER_BUILD" == "true" ] && [ -n "$DOCKERFILE_CHANGED" ]; then + echo "docker_build_enabled=true" >> $GITHUB_OUTPUT + echo "has_dockerfile_changes=true" >> $GITHUB_OUTPUT + echo "✅ Docker build will run (BUILD_DOCKER=true and Dockerfile changed)" + else + echo "docker_build_enabled=false" >> $GITHUB_OUTPUT + if [ -n "$DOCKERFILE_CHANGED" ]; then + echo "has_dockerfile_changes=true" >> $GITHUB_OUTPUT + else + echo "has_dockerfile_changes=false" >> $GITHUB_OUTPUT + fi + echo "⏭️ Docker build will be skipped (BUILD_DOCKER=$DOCKER_BUILD, Dockerfile changes: ${DOCKERFILE_CHANGED:-none})" + fi + + # Set output for build job (runs when build_info.json or .sh scripts change) + BUILD_INFO_CHANGED=$(echo "$CHANGED_FILES" | grep 'build_info\.json' || true) + if [ -n "$BUILD_INFO_CHANGED" ] || [ -n "$SH_SCRIPT_CHANGED" ]; then + echo "build_package_enabled=true" >> $GITHUB_OUTPUT + echo "✅ Build package job will run (build_info.json or .sh scripts changed)" + else + echo "build_package_enabled=false" >> $GITHUB_OUTPUT + echo "⏭️ Build package job will be skipped (no build_info.json or .sh script changes)" + fi + - name: Create scanner-env.sh run: | mkdir -p package-cache @@ -165,12 +292,14 @@ jobs: build: needs: build_info + if: needs.build_info.outputs.build_package_enabled == 'true' runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} steps: - uses: actions/checkout@v6 with: - ref: ${{ github.event.pull_request.head.sha }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.ref }} - name: Download package-cache uses: actions/download-artifact@v7 @@ -191,16 +320,16 @@ jobs: cat package-cache/scanner-env.sh sudo chown -R $USER:$USER . - chmod +x ./gha-script/build_package.sh - bash ./gha-script/build_package.sh + echo "------------------- Executing changed scripts -----------------------------" + chmod +x ./gha-script/execute_changed_scripts.py + python3 ./gha-script/execute_changed_scripts.py # ===================== WHEEL JOBS ===================== - wheel_build_py310: needs: build_info - if: ${{ success() }} + if: needs.build_info.outputs.wheel_build_enabled == 'true' runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} continue-on-error: false env: @@ -213,7 +342,8 @@ jobs: uses: actions/checkout@v6 with: fetch-depth: 0 - ref: ${{ github.event.pull_request.head.sha }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.ref }} - name: Install system dependencies run: | @@ -264,7 +394,7 @@ jobs: wheel_build_py311: needs: build_info - if: ${{ success() }} + if: needs.build_info.outputs.wheel_build_enabled == 'true' runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} continue-on-error: false env: @@ -277,7 +407,8 @@ jobs: uses: actions/checkout@v6 with: fetch-depth: 0 - ref: ${{ github.event.pull_request.head.sha }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.ref }} - name: Install system dependencies run: | @@ -326,10 +457,9 @@ jobs: sudo uname -a - wheel_build_py312: needs: build_info - if: ${{ success() }} + if: needs.build_info.outputs.wheel_build_enabled == 'true' runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} continue-on-error: false env: @@ -342,7 +472,8 @@ jobs: uses: actions/checkout@v6 with: fetch-depth: 0 - ref: ${{ github.event.pull_request.head.sha }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.ref }} - name: Install system dependencies run: | @@ -390,9 +521,10 @@ jobs: sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" sudo uname -a + wheel_build_py313: needs: build_info - if: ${{ success() }} + if: needs.build_info.outputs.wheel_build_enabled == 'true' runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} continue-on-error: true env: @@ -405,7 +537,8 @@ jobs: uses: actions/checkout@v6 with: fetch-depth: 0 - ref: ${{ github.event.pull_request.head.sha }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.ref }} - name: Install system dependencies run: | @@ -456,7 +589,7 @@ jobs: wheel_build_py314: needs: build_info - if: ${{ success() }} + if: needs.build_info.outputs.wheel_build_enabled == 'true' runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} continue-on-error: true env: @@ -469,7 +602,8 @@ jobs: uses: actions/checkout@v6 with: fetch-depth: 0 - ref: ${{ github.event.pull_request.head.sha }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.ref }} - name: Install system dependencies run: | @@ -518,17 +652,17 @@ jobs: sudo uname -a - build_docker: needs: build_info - if: ${{ success() }} + if: needs.build_info.outputs.docker_build_enabled == 'true' runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} steps: - name: Checkout code uses: actions/checkout@v6 with: - ref: ${{ github.event.pull_request.head.sha }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.ref }} - name: Download package-cache uses: actions/download-artifact@v7 diff --git a/gha-script/download-scripts/download_file.sh b/gha-script/download-scripts/download_file.sh index 5db90bc478..7d706493ae 100644 --- a/gha-script/download-scripts/download_file.sh +++ b/gha-script/download-scripts/download_file.sh @@ -9,7 +9,7 @@ if [[ $(echo "$token_request" | jq -r '.errorCode') == "null" ]]; then token=$(echo "$token_request" | jq -r '.access_token') echo "Downloading $1 from COS..." curl -X GET -H "Authorization: bearer $token" \ - -o $1 "https://s3.us.cloud-object-storage.appdomain.cloud/ose-power-toolci-bucket-production/$PACKAGE_NAME/$VERSION/$1" + -o $1 "https://s3.us.cloud-object-storage.appdomain.cloud/ose-power-toolci-bucket-stag/$PACKAGE_NAME/$VERSION/$1" echo "Download completed: $1" else echo "Error: Token request failed. Response: $token_request" diff --git a/gha-script/download-scripts/download_wheels.sh b/gha-script/download-scripts/download_wheels.sh index 8830015303..c7c90017ae 100644 --- a/gha-script/download-scripts/download_wheels.sh +++ b/gha-script/download-scripts/download_wheels.sh @@ -11,7 +11,7 @@ token_request=$(curl -s -X POST https://iam.cloud.ibm.com/identity/token \ if [[ $(echo "$token_request" | jq -r '.errorCode') == "null" ]]; then token=$(echo "$token_request" | jq -r '.access_token') - bucket_url="https://s3.us.cloud-object-storage.appdomain.cloud/ose-power-artifacts-production" + bucket_url="https://s3.us.cloud-object-storage.appdomain.cloud/ose-power-artifacts-stag" echo "Fetching wheel list from COS..." echo "Checking prefix: $PACKAGE_NAME/$VERSION/" diff --git a/gha-script/execute_changed_scripts.py b/gha-script/execute_changed_scripts.py new file mode 100644 index 0000000000..25baa72a4a --- /dev/null +++ b/gha-script/execute_changed_scripts.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 + +import os +import sys + + +def extract_script_metadata(script_path): + """ + Extract metadata from script header comments. + Returns dict with package_version, tested_on, ci_check, etc. + """ + metadata = { + 'package_version': None, + 'tested_on': 'UBI:9.3', # default + 'ci_check': 'true', # default + 'package_name': None, + 'use_non_root_user': 'false' # default + } + + key_mapping = { + '# Package': 'package_name', + '# Version': 'package_version', + '# Tested on': 'tested_on', + '# Ci-Check': 'ci_check', + '# Use Non-Root User': 'use_non_root_user' + } + + try: + with open(script_path, 'r') as f: + for line in f: + line = line.strip() + for key, field in key_mapping.items(): + if line.startswith(key): + value = line.split(':', 1)[-1].strip() + metadata[field] = value + break + except Exception as e: + print(f"Error reading script {script_path}: {e}") + return None + + return metadata + + +def get_changed_scripts(): + """ + Get list of changed .sh scripts from CHANGED_FILES environment variable. + """ + changed_files = os.environ.get('CHANGED_FILES', '') + if not changed_files: + print("No CHANGED_FILES environment variable found") + return [] + + scripts = [] + for line in changed_files.split('\n'): + line = line.strip() + if line.endswith('.sh') and 'dockerfile' not in line.lower(): + scripts.append(line) + + return scripts + + +def main(): + print("=" * 80) + print("EXECUTING CHANGED BUILD SCRIPTS") + print("=" * 80) + + # Get changed scripts + changed_scripts = get_changed_scripts() + + if not changed_scripts: + print("No .sh scripts found in changed files") + print("Skipping script execution") + return 0 + + print(f"\nFound {len(changed_scripts)} changed script(s):") + for script in changed_scripts: + print(f" - {script}") + print() + + # Process each script + executed_count = 0 + skipped_count = 0 + + for script_path in changed_scripts: + print("=" * 80) + print(f"Processing: {script_path}") + print("=" * 80) + + # Check if script exists + if not os.path.exists(script_path): + print(f"⚠️ Script not found: {script_path}") + continue + + # Extract metadata from script header + metadata = extract_script_metadata(script_path) + if not metadata: + print(f"❌ Failed to extract metadata from {script_path}") + continue + + version = metadata['package_version'] + ci_check = metadata['ci_check'].lower() + tested_on = metadata['tested_on'] + use_non_root = metadata['use_non_root_user'].lower() + + print(f"📋 Metadata extracted:") + print(f" Package: {metadata['package_name']}") + print(f" Version: {version}") + print(f" Tested on: {tested_on}") + print(f" CI-Check: {ci_check}") + print(f" Use Non-Root: {use_non_root}") + + # Check CI-Check flag + if ci_check != "true": + print(f"⏭️ Skipping {script_path} - CI-Check flag is set to {ci_check}") + skipped_count += 1 + continue + + if not version: + print(f"⚠️ No version found in script header, skipping {script_path}") + skipped_count += 1 + continue + + # Set environment variables for build_package.sh + os.environ['PKG_DIR_PATH'] = os.path.dirname(script_path) + '/' + os.environ['BUILD_SCRIPT'] = os.path.basename(script_path) + os.environ['VERSION'] = version + os.environ['TESTED_ON'] = tested_on + os.environ['NON_ROOT_BUILD'] = use_non_root + + print(f"\n🚀 Executing: {script_path} with version {version}") + print(f" Command: bash gha-script/build_package.sh") + + # Execute build_package.sh which will run the script + exit_code = os.system('bash gha-script/build_package.sh') + + if exit_code != 0: + print(f"\n❌ Script execution failed for {script_path} with exit code {exit_code}") + return exit_code + else: + print(f"\n✅ Script execution completed successfully for {script_path}") + executed_count += 1 + + print("\n" + "=" * 80) + print("EXECUTION SUMMARY") + print("=" * 80) + print(f"✅ Executed: {executed_count}") + print(f"⏭️ Skipped: {skipped_count}") + print(f"📊 Total: {len(changed_scripts)}") + print("=" * 80) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/gha-script/upload-scripts/upload_file.sh b/gha-script/upload-scripts/upload_file.sh index 8a676f486f..31cebb26b0 100644 --- a/gha-script/upload-scripts/upload_file.sh +++ b/gha-script/upload-scripts/upload_file.sh @@ -14,7 +14,7 @@ if [[ $(echo "$token_request" | jq -r '.errorCode') == "null" ]]; then # curl command for uploading the file #response=$(curl -X PUT -H "Authorization: bearer $token" -H "Content-Type: application/gzip" -T $1 "https://s3.us.cloud-object-storage.appdomain.cloud/ose-power-toolci-bucket-prod/$PACKAGE_NAME/$VERSION/$1") - response=$(curl -X PUT -H "Authorization: bearer $token" -H "Content-Type: application/gzip" -T $1 "https://s3.us.cloud-object-storage.appdomain.cloud/ose-power-toolci-bucket-production/$PACKAGE_NAME/$VERSION/$1") + response=$(curl -X PUT -H "Authorization: bearer $token" -H "Content-Type: application/gzip" -T $1 "https://s3.us.cloud-object-storage.appdomain.cloud/ose-power-toolci-bucket-stag/$PACKAGE_NAME/$VERSION/$1") # Check if the PUT request was successful based on the absence of an block if ! echo "$response" | grep -q ""; then diff --git a/gha-script/upload-scripts/upload_wheel.sh b/gha-script/upload-scripts/upload_wheel.sh index cc1e8f1e27..9f56c10392 100644 --- a/gha-script/upload-scripts/upload_wheel.sh +++ b/gha-script/upload-scripts/upload_wheel.sh @@ -27,7 +27,7 @@ if [[ $(echo "$token_request" | jq -r '.errorCode') == "null" ]]; then token=$(echo "$token_request" | jq -r '.access_token') # curl command for uploading the file - #response=$(curl -X PUT -H "Authorization: bearer $token" -H "Content-Type: application/octet-stream" -T $1 "https://s3.us.cloud-object-storage.appdomain.cloud/ose-power-artifacts-prod/$PACKAGE_NAME/$VERSION/$1") + #response=$(curl -X PUT -H "Authorization: bearer $token" -H "Content-Type: application/octet-stream" -T $1 "https://s3.us.cloud-object-storage.appdomain.cloud/ose-power-artifacts-stag/$PACKAGE_NAME/$VERSION/$1") response=$(curl -X PUT \ -H "Authorization: bearer $token" \ -H "Content-Type: application/octet-stream" \ diff --git a/gha-script/validate_builds.py b/gha-script/validate_builds.py index 54094dcce1..4654edf590 100644 --- a/gha-script/validate_builds.py +++ b/gha-script/validate_builds.py @@ -11,7 +11,7 @@ GITHUB_BUILD_SCRIPT_BASE_REPO = "build-scripts" -GITHUB_BUILD_SCRIPT_BASE_OWNER = "ppc64le" +GITHUB_BUILD_SCRIPT_BASE_OWNER = "stutiibm" HOME = os.getcwd() package_data = {} @@ -315,14 +315,20 @@ def trigger_build_validation_ci(pr_number): # perform basic validation check trigger_basic_validation_checks(file_name) - #check ci-check from package header + #check ci-check from package header ci_check=package_data['ci_check'].lower() - if ci_check=="true": - # Build/test script files - trigger_script_validation_checks(file_name) - else: - print("Skipping Build script validation for {} as CI-Check flag is set to False".format(file_name)) + # COMMENTED OUT: Script execution moved to build job via execute_changed_scripts.py + # This avoids duplicate execution - validation happens here, actual build happens in build job + # if ci_check=="true": + # # Build/test script files + # trigger_script_validation_checks(file_name) + # else: + # print("Skipping Build script validation for {} as CI-Check flag is set to False".format(file_name)) + + print("✅ Basic validation passed for {} (headers, license, line endings)".format(file_name)) + print(" Script execution will happen in build job with proper version parameter") + # Keep track of validated files. validated_file_list.append(file_name) elif file_name.lower().endswith('build_info.json') and status != "removed": diff --git a/p/param/param_ubi_9.3.sh b/p/param/param_ubi_9.3.sh index 6a0ccdfe31..76f869f6ab 100644 --- a/p/param/param_ubi_9.3.sh +++ b/p/param/param_ubi_9.3.sh @@ -6,7 +6,7 @@ # Source repo : https://github.com/holoviz/param.git # Tested on : UBI:9.3 # Language : Python -# Ci-Check : True +# Ci-Check : False # Script License : GNU General Public License v3.0 # Maintainer : Anumala Rajesh #