android-compat.yml

name: Android compatibility

# One APK, every Android version we claim to support.
#
# `native-smoke` asks whether the templates can produce a launching app at all,
# and it asks it on ONE platform version — API 34. That leaves the failure this
# exists for completely uncovered: a host that runs on the version CI happens to
# boot and dies on the ones users have. A packaged game is built ONCE here and
# installed on one emulator per Android release, so a version-specific break is
# attributed to the version rather than to the build.
#
# The frames are collected to be LOOKED AT. There is deliberately no pixel
# judgement: a scene that is legitimately dark and a renderer that died produce
# the same dark PNG, and no threshold separates them. What this gates is what can
# be decided without a human — did it install, did the host reach `ready`, did the
# boot record name an error. Everything else is measured, tabulated, and left for
# a reviewer.
#
# The version list is DISCOVERED from the runner's own system images rather than
# written down, so a new Android release is picked up by the next run instead of
# by whoever remembers to edit this file. Preview releases are named by codename
# and are skipped on purpose — a matrix that reds on an unreleased platform gets
# ignored within a week.
#
# What a hosted runner cannot see: real GPU drivers, compressed texture support,
# thermal throttling and memory pressure. Dawn runs on the emulator's SwiftShader,
# so the frame times are a CPU rasteriser's. Comparing versions against each other
# is valid; reading any number here as device performance is not. The report says
# so on every run rather than trusting anyone to remember it.

on:
  pull_request:
    paths:
      - 'native/**'
      - 'build-tools/**'
      - 'sdk/**'
      - 'toolchain.manifest.json'
      - '.github/workflows/android-compat.yml'
      - 'tools/verify-native-boot.mjs'
      - 'tools/android-compat-report.mjs'
  workflow_dispatch:
    inputs:
      examples:
        description: 'Comma-separated example names to package and run on every version.'
        required: false
        default: 'space-shooter'
        type: string
      api-floor:
        description: 'Oldest API level to test (29 = Android 10).'
        required: false
        default: '29'
        type: string
      template-source:
        description: 'Which runtime template to wrap: the latest release, this branch (~35 min), or decide from what the PR touches.'
        required: false
        default: auto
        type: choice
        options: [auto, release, head]

permissions:
  contents: read

# One matrix per PR at a time. `paths` on a pull_request matches the PR's whole
# diff, not the latest push, so every push re-runs all eight emulators — three
# pushes queued twenty-four of them before this was here.
concurrency:
  group: android-compat-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  # Android 10. The manifest claims 26, which is a wider promise than anything
  # here checks — raising this floor is a decision about what we support, so it is
  # stated once, in one place, rather than implied by a matrix.
  ANDROID_API_FLOOR: ${{ inputs.api-floor || '29' }}
  ANDROID_PROFILE: pixel_6

jobs:
  # ---------------------------------------------------------------------------
  versions:
    name: Which Android versions this runner has images for
    runs-on: ubuntu-latest
    timeout-minutes: 10
    outputs:
      matrix: ${{ steps.find.outputs.matrix }}
      count: ${{ steps.find.outputs.count }}
    steps:
      - name: List the system images the SDK actually offers
        id: find
        run: |
          set -euo pipefail
          SDKMANAGER="$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager"
          # Digits only, so preview platforms (android-Baklava and the like) are
          # left out rather than reddening the matrix on an unreleased release.
          AVAILABLE=$("$SDKMANAGER" --list 2>/dev/null \
            | grep -oE 'system-images;android-[0-9]+;google_apis;x86_64' \
            | grep -oE 'android-[0-9]+' | grep -oE '[0-9]+' | sort -un || true)

          if [ -z "$AVAILABLE" ]; then
            echo "::error::sdkmanager listed no google_apis;x86_64 system images at all."
            exit 1
          fi

          LEVELS=$(echo "$AVAILABLE" | awk -v floor="$ANDROID_API_FLOOR" '$1 >= floor')
          if [ -z "$LEVELS" ]; then
            echo "::error::no system image at or above API $ANDROID_API_FLOOR. This runner has: $(echo $AVAILABLE | tr '\n' ' ')"
            exit 1
          fi

          # The floor itself must exist. If the runner image stops carrying the
          # oldest version we claim to support, that is a silent loss of coverage
          # unless it fails here and says so.
          if ! echo "$LEVELS" | grep -qx "$ANDROID_API_FLOOR"; then
            echo "::error::API $ANDROID_API_FLOOR (the declared floor) has no system image on this runner. Available: $(echo $LEVELS | tr '\n' ' ')"
            exit 1
          fi

          MATRIX=$(echo "$LEVELS" | jq -Rsc 'split("\n") | map(select(length > 0) | tonumber)')
          echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT"
          echo "count=$(echo "$LEVELS" | wc -l | tr -d ' ')" >> "$GITHUB_OUTPUT"
          echo "Testing API levels: $(echo $LEVELS | tr '\n' ' ')"

  # ---------------------------------------------------------------------------
  # The APKs come from the shared packaging workflow, so the emulator matrix and
  # the device farm run install a byte-identical binary. See android-canary-apk.yml.
  apk:
    uses: ./.github/workflows/android-canary-apk.yml
    permissions:
      contents: read
      pull-requests: read   # to see which files the PR touches
    with:
      examples: ${{ inputs.examples || 'space-shooter' }}
      template-source: ${{ inputs.template-source || 'auto' }}


  # ---------------------------------------------------------------------------
  compat:
    name: Android API ${{ matrix.api }}
    needs: [versions, apk]
    runs-on: ubuntu-latest
    # A version that works finishes in about three minutes and one that crashes in
    # seven, so this wants to be near what a real run costs: a stuck emulator
    # should be reported as "no data" quickly, not held open in case it recovers.
    #
    # But it has to hold TWO of them. At 20 it did not, and the retry below was
    # fiction: API 24 sat in the boot poll from 01:11 to 01:30 and the job cap
    # cancelled it mid-attempt, so "no data after two attempts" was printed after
    # one. Two bounded attempts plus the system-image download is what this covers.
    timeout-minutes: 40
    strategy:
      # A compatibility matrix whose whole output is "which versions work" must
      # run every version even after one fails. fail-fast here would report the
      # oldest break and hide everything above it.
      fail-fast: false
      matrix:
        api: ${{ fromJSON(needs.versions.outputs.matrix) }}
    steps:
      - uses: actions/checkout@v7

      - uses: ./.github/actions/setup

      - uses: actions/download-artifact@v8
        with:
          name: compat-apks
          path: apks

      # Without KVM the emulator falls back to software CPU emulation and the boot
      # alone outlasts the job.
      - name: Enable KVM
        run: |
          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \
            | sudo tee /etc/udev/rules.d/99-kvm4all.rules
          sudo udevadm control --reload-rules
          sudo udevadm trigger --name-match=kvm

      # Attempted twice, because "the app crashed" and "the emulator never came up"
      # both fail this step and only one of them is an answer. Older API levels
      # download their system image and then sometimes never reach
      # sys.boot_completed at all — API 29 and API 30 traded places across three
      # runs, costing a version each time, which a matrix built to say "every
      # version works" cannot afford to do at random.
      #
      # `continue-on-error` on the first go, and the retry is conditional on
      # NOTHING having been filed: a metrics file means the script ran and reached
      # a verdict, even a crash, and re-running that would only waste an emulator.
      - name: Install, launch and measure on API ${{ matrix.api }}
        id: attempt1
        continue-on-error: true
        uses: reactivecircus/android-emulator-runner@v2
        with:
          api-level: ${{ matrix.api }}
          target: google_apis
          arch: x86_64
          # An era-appropriate device below API 26: `pixel_6` is a 2021 profile,
          # and pairing it with a 2016 system image is one of the two things that
          # kept 24 and 25 from ever reaching sys.boot_completed.
          profile: ${{ matrix.api <= 25 && 'pixel' || env.ANDROID_PROFILE }}
          # The other one: the action boots from a snapshot by default, and an old
          # image that cannot load the one it was given hangs instead of saying so.
          emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -no-snapshot-load -camera-back none -camera-front none -memory 4096 -cores 2
          # Explicit, and short enough that two attempts fit the job cap. The
          # default (600) did not bound anything here — the poll ran nineteen
          # minutes — so this is the number that has to be believed, not inherited.
          emulator-boot-timeout: 420
          # ONE line, because this action runs `script` as a separate `sh -c` per
          # line: a loop written here arrives split and dies on its own `do`, and
          # nothing set on one line is visible on the next. The work lives in a
          # file that can also be run by hand.
          script: bash tools/android-compat-run.sh ${{ matrix.api }}

      - name: Did anything get measured
        id: measured
        run: |
          if ls build/compat/*.json >/dev/null 2>&1; then
            echo "any=true" >> "$GITHUB_OUTPUT"
          else
            echo "any=false" >> "$GITHUB_OUTPUT"
            echo "nothing was filed — the emulator never came up, so this is not yet a result"
          fi

      - name: Try API ${{ matrix.api }} once more
        id: retry
        if: steps.measured.outputs.any == 'false'
        uses: reactivecircus/android-emulator-runner@v2
        continue-on-error: true
        with:
          api-level: ${{ matrix.api }}
          target: google_apis
          arch: x86_64
          # An era-appropriate device below API 26: `pixel_6` is a 2021 profile,
          # and pairing it with a 2016 system image is one of the two things that
          # kept 24 and 25 from ever reaching sys.boot_completed.
          profile: ${{ matrix.api <= 25 && 'pixel' || env.ANDROID_PROFILE }}
          # The other one: the action boots from a snapshot by default, and an old
          # image that cannot load the one it was given hangs instead of saying so.
          emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -no-snapshot-load -camera-back none -camera-front none -memory 4096 -cores 2
          # Explicit, and short enough that two attempts fit the job cap. The
          # default (600) did not bound anything here — the poll ran nineteen
          # minutes — so this is the number that has to be believed, not inherited.
          emulator-boot-timeout: 420
          script: bash tools/android-compat-run.sh ${{ matrix.api }}

      - name: Keep the frame, the record and the numbers
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: compat-api${{ matrix.api }}
          path: build/compat
          if-no-files-found: warn

      # This job MEASURES; the report JUDGES. So it fails only when it measured
      # nothing at all — a crashed version is a successful measurement of a broken
      # platform, and letting that redden the job here would put the verdict in two
      # places that can disagree.
      - name: Fail only if this version was never measured
        if: always()
        run: |
          if ls build/compat/*.json >/dev/null 2>&1; then
            echo "measured API ${{ matrix.api }}"
          elif [ "${{ steps.retry.conclusion }}" = "skipped" ] || [ -z "${{ steps.retry.conclusion }}" ]; then
            # The retry never ran, so "two attempts" would be a lie — and the
            # difference matters: one says this Android version is broken, the
            # other says this job ran out of time before it could find out.
            echo "::error::API ${{ matrix.api }} produced no data and the second attempt never ran — the job hit its own cap first"
            exit 1
          else
            echo "::error::API ${{ matrix.api }} produced no data after two attempts"
            exit 1
          fi

  # ---------------------------------------------------------------------------
  report:
    name: Report into the PR
    needs: [versions, apk, compat]
    # always(): the versions that FAILED are the rows this report exists to show.
    # A summariser that only runs on success reports exactly the case nobody needs.
    if: always() && needs.versions.result == 'success'
    runs-on: ubuntu-latest
    timeout-minutes: 15
    permissions:
      contents: write        # publishes the frames to a branch the PR can render
      pull-requests: write   # posts the table
    steps:
      - uses: actions/checkout@v7

      # The report itself is node builtins only, but the toolchain version it runs
      # under is this action's to decide, not the runner image's.
      - uses: ./.github/actions/setup

      - uses: actions/download-artifact@v8
        with:
          pattern: compat-api*
          path: build/compat

      # A fork's token is read-only, so neither the push nor the comment can work.
      # Say that in the log and still build the report, rather than failing with a
      # permissions error that looks like a broken workflow.
      - name: Can this run write to the repo
        id: writable
        run: |
          if [ "${{ github.event_name }}" = 'pull_request' ] \
             && [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]; then
            echo "same-repo=false" >> "$GITHUB_OUTPUT"
            echo "PR is from a fork — the token cannot push frames or comment."
          else
            echo "same-repo=true" >> "$GITHUB_OUTPUT"
          fi

      # GitHub renders an image in a comment only from a URL. Artifacts have none,
      # so the frames go to an orphan branch and are referenced from there. The
      # github.com/raw form (not raw.githubusercontent) is used because it resolves
      # for a signed-in reviewer even when the repository is private.
      - name: Publish the frames where the PR can show them
        id: shots
        if: steps.writable.outputs.same-repo == 'true'
        env:
          BRANCH: ci-android-shots
          DEST: pr-${{ github.event.pull_request.number || 'dispatch' }}/${{ github.run_id }}
        run: |
          set -euo pipefail
          shopt -s nullglob globstar
          FRAMES=(build/compat/**/*.png)
          if [ ${#FRAMES[@]} -eq 0 ]; then
            echo "no frames were captured — nothing to publish"
            exit 0
          fi
          git config user.name  'github-actions[bot]'
          git config user.email '41898282+github-actions[bot]@users.noreply.github.com'
          if git ls-remote --exit-code --heads origin "$BRANCH" >/dev/null 2>&1; then
            git fetch --depth=1 origin "$BRANCH:$BRANCH"
            git worktree add shots "$BRANCH"
          else
            git worktree add --detach shots
            git -C shots checkout --orphan "$BRANCH"
            git -C shots rm -rf --cached . >/dev/null 2>&1 || true
            rm -rf shots/* 2>/dev/null || true
          fi
          mkdir -p "shots/$DEST"
          cp "${FRAMES[@]}" "shots/$DEST/"
          git -C shots add -A
          git -C shots commit -q -m "ci: android compat frames for ${DEST}"
          git -C shots push -q origin "$BRANCH"
          echo "base=https://github.com/${{ github.repository }}/raw/$BRANCH/$DEST" >> "$GITHUB_OUTPUT"

      # continue-on-error because a non-zero exit here IS the verdict — it means
      # some version is broken or untested — and the comment still has to be
      # posted. The gate at the end of this job reads the outcome.
      - name: Build the table
        id: table
        continue-on-error: true
        run: |
          node tools/android-compat-report.mjs \
            --dir build/compat \
            --template-source '${{ needs.apk.outputs.template-from }}' \
            --expect '${{ needs.versions.outputs.matrix }}' \
            ${{ steps.shots.outputs.base && format('--shots-base {0}', steps.shots.outputs.base) || '' }} \
            --out compat-comment.md
          cat compat-comment.md >> "$GITHUB_STEP_SUMMARY"

      # Edited in place on re-runs: a PR that pushes six times should carry one
      # current table, not six stale ones.
      - name: Post it on the PR
        if: github.event_name == 'pull_request' && steps.writable.outputs.same-repo == 'true'
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          PR: ${{ github.event.pull_request.number }}
        run: |
          set -euo pipefail
          EXISTING=$(gh api "repos/${{ github.repository }}/issues/$PR/comments" --paginate \
            --jq '.[] | select(.body | startswith("<!-- android-compat -->")) | .id' | head -1)
          if [ -n "$EXISTING" ]; then
            gh api -X PATCH "repos/${{ github.repository }}/issues/comments/$EXISTING" \
              -F body=@compat-comment.md >/dev/null
            echo "updated comment $EXISTING"
          else
            gh api -X POST "repos/${{ github.repository }}/issues/$PR/comments" \
              -F body=@compat-comment.md >/dev/null
            echo "posted a new comment"
          fi

      # The gate, after the report rather than before it: a red check with no table
      # tells a reviewer that something broke but not what. It reads the DATA, not
      # the matrix jobs' conclusions — those cannot tell a crashed app from an
      # emulator that never booted, and this distinction is the whole point.
      - name: Fail if any version is broken or untested
        if: steps.table.outcome == 'failure'
        run: |
          echo "::error::some Android version is broken or was never measured — see the table in the PR comment"
          exit 1