[fix][broker] Handle flow_or_qps bundle split requests in ELM

.github/actions/clean-disk/action.yml

@@ -23,44 +23,98 @@ inputs:
   mode:
     description: "Use 'full' to clean as much as possible"
     required: false
+  clean-threshold-gb:
+    description: "Skip cleaning if available disk space exceeds this threshold (in GB)"
+    required: false
+    default: '20'
+  full-clean-threshold-gb:
+    description: "In 'full' mode, skip additional cleaning steps if available disk space exceeds this threshold (in GB)"
+    required: false
+    default: '40'
 runs:
   using: composite
   steps:
-    - run: |
+    - shell: bash
+      env:
+        CLEAN_MODE: ${{ inputs.mode }}
+        CLEAN_THRESHOLD_GB: ${{ inputs.clean-threshold-gb }}
+        FULL_CLEAN_THRESHOLD_GB: ${{ inputs.full-clean-threshold-gb }}
+      run: |
         if [[ "$OSTYPE" == "linux-gnu"* ]]; then
-          directories=(/usr/local/lib/android /opt/ghc)
-          if [[ "${{ inputs.mode }}" == "full" ]]; then
-            # remove these directories only when mode is 'full'
-            directories+=(/usr/share/dotnet /opt/hostedtoolcache/CodeQL)
+
+          available_gb() {
+            df --output=avail -BG / | tail -1 | tr -d ' G'
+          }
+
+          show_disk() {
+            if mountpoint -q /mnt; then df -BM / /mnt; else df -BM /; fi
+          }
+
+          remove_dir() {
+            local directory=$1
+            if [ -d "$directory" ]; then
+              echo "::group::Removing $directory"
+              local emptydir=/tmp/empty$$/
+              mkdir -p "$emptydir"
+              # fast way to delete a lot of files on linux
+              time sudo eatmydata rsync -a --delete "$emptydir" "${directory}/"
+              time sudo eatmydata rm -rf "${directory}"
+              rmdir "$emptydir" 2>/dev/null || true
+              time show_disk
+              echo "::endgroup::"
+            fi
+          }
+
+          # Remove directories one at a time, stopping when threshold is met
+          clean_dirs() {
+            local threshold=$1
+            shift
+            for directory in "$@"; do
+              if (( $(available_gb) >= threshold )); then
+                return
+              fi
+              remove_dir "$directory"
+            done
+          }
+
+          if [[ "$CLEAN_MODE" == "full" ]]; then
+            threshold=$FULL_CLEAN_THRESHOLD_GB
+          else
+            threshold=$CLEAN_THRESHOLD_GB
           fi
-          emptydir=/tmp/empty$$/
-          mkdir $emptydir
-          echo "::group::Available diskspace"
-          time df -BM / /mnt
+
+          echo "::group::Available diskspace (threshold: ${threshold}GB)"
+          time show_disk
           echo "::endgroup::"
-          for directory in "${directories[@]}"; do
-            echo "::group::Removing $directory"
-            # fast way to delete a lot of files on linux
-            time sudo eatmydata rsync -a --delete $emptydir ${directory}/
-            time sudo eatmydata rm -rf ${directory}
-            time df -BM / /mnt
-            echo "::endgroup::"
-          done
-          if [[ "${{ inputs.mode }}" == "full" ]]; then
-            echo "::group::Moving /var/lib/docker to /mnt/docker"
-            sudo systemctl stop docker
-            echo '{"data-root": "/mnt/docker"}' | sudo tee /etc/docker/daemon.json
-            sudo mv /var/lib/docker /mnt/docker
-            sudo systemctl start docker
-            time df -BM / /mnt
+
+          if (( $(available_gb) >= threshold )); then
+            echo "Available disk space meets threshold, skipping cleanup."
+            exit 0
+          fi
+
+          clean_dirs "$threshold" /usr/local/lib/android /opt/ghc
+
+          if [[ "$CLEAN_MODE" == "full" ]]; then
+            clean_dirs "$threshold" /usr/share/dotnet /opt/hostedtoolcache/CodeQL
+            if (( $(available_gb) < threshold )) && mountpoint -q /mnt; then
+              echo "::group::Moving /var/lib/docker to /mnt/docker"
+              sudo systemctl stop docker
+              echo '{"data-root": "/mnt/docker"}' | sudo tee /etc/docker/daemon.json
+              sudo mv /var/lib/docker /mnt/docker
+              sudo systemctl start docker
+              time show_disk
+              echo "::endgroup::"
+            fi
+          fi
+
+          if (( $(available_gb) < threshold )); then
+            echo "::group::Cleaning apt state"
+            time sudo bash -c "apt-get clean; apt-get autoclean; apt-get -y --purge autoremove"
+            time show_disk
             echo "::endgroup::"
           fi
-          echo "::group::Cleaning apt state"
-          time sudo bash -c "apt-get clean; apt-get autoclean; apt-get -y --purge autoremove"
-          time df -BM / /mnt
+
+          echo "::group::Available diskspace"
+          time show_disk
           echo "::endgroup::"
         fi
-        echo "::group::Available diskspace"
-        time df -BM / /mnt
-        echo "::endgroup::"
-      shell: bash

.github/actions/setup-gradle/action.yml

@@ -0,0 +1,60 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: Setup Gradle
+description: Sets up Gradle with Develocity or public build scan publishing by default
+inputs:
+  develocity-access-key:
+    description: 'Develocity access key for authenticated build scans'
+    required: false
+    default: ''
+  build-scan-publish:
+    description: 'Whether to publish build scans or use Develocity when the access key is set'
+    required: false
+    default: 'true'
+  cache-read-only:
+    description: 'Whether the Gradle cache is read-only'
+    required: false
+    default: ${{ github.event.repository != null && github.ref_name != github.event.repository.default_branch }}
+  add-job-summary:
+    description: 'When to add a job summary'
+    required: false
+    default: 'always'
+runs:
+  using: composite
+  steps:
+    - name: Setup Gradle with Develocity
+      if: ${{ inputs.develocity-access-key != '' && inputs.build-scan-publish == 'true' }}
+      uses: gradle/actions/setup-gradle@39e147cb9de83bb9910b8ef8bd7fff0ee20fcd6f
+      with:
+        develocity-injection-enabled: true
+        develocity-url: https://develocity.apache.org
+        develocity-access-key: ${{ inputs.develocity-access-key }}
+        cache-read-only: ${{ inputs.cache-read-only }}
+        add-job-summary: ${{ inputs.add-job-summary }}
+
+    - name: Setup Gradle
+      if: ${{ !(inputs.develocity-access-key != '' && inputs.build-scan-publish == 'true') }}
+      uses: gradle/actions/setup-gradle@39e147cb9de83bb9910b8ef8bd7fff0ee20fcd6f
+      with:
+        build-scan-publish: ${{ inputs.build-scan-publish }}
+        build-scan-terms-of-use-url: 'https://gradle.com/terms-of-service'
+        build-scan-terms-of-use-agree: 'yes'
+        cache-read-only: ${{ inputs.cache-read-only }}
+        add-job-summary: ${{ inputs.add-job-summary }}

.github/actions/ssh-access/action.yml

@@ -138,15 +138,16 @@ runs:
             if command -v upterm &>/dev/null; then
                 shopt -s nullglob
                 echo "SSH connection information"
-                upterm session current --admin-socket ~/.upterm/*.sock || {
+                export UPTERM_ADMIN_SOCKET=$(find $HOME/.upterm $XDG_RUNTIME_DIR/upterm /run/user/$(id -u)/upterm -name "*.sock" | head -n 1)
+                upterm session current || {
                     echo "upterm isn't running. Not waiting any longer."
                     exit 0
                 }
                 timeout=${{ inputs.timeout }}
                 echo "Waiting $timeout seconds..."
                 sleep $timeout
                 echo "Keep waiting as long as there's a connected session"
-                while upterm session current --admin-socket ~/.upterm/*.sock|grep Connected &>/dev/null; do
+                while upterm session current|grep Connected &>/dev/null; do
                     sleep 30
                 done
                 echo "No session is connected. Not waiting any longer."

.github/actions/tune-runner-vm/action.yml

@@ -22,7 +22,8 @@ description: tunes the GitHub Runner VM operation system
 runs:
   using: composite
   steps:
-    - run: |
+    - shell: bash
+      run: |
         if [[ "$OSTYPE" == "linux-gnu"* ]]; then
             echo "::group::Configure and tune OS"
             # Ensure that reverse lookups for current hostname are handled properly
@@ -33,49 +34,38 @@ runs:
             # consumption is high.
             # Set vm.swappiness=1 to avoid swapping and allow high RAM usage
             echo 1 | sudo tee /proc/sys/vm/swappiness
-            (
-              shopt -s nullglob
-              # Set swappiness to 1 for all cgroups and sub-groups
-              for swappiness_file in /sys/fs/cgroup/memory/*/memory.swappiness /sys/fs/cgroup/memory/*/*/memory.swappiness; do
-                echo 1 | sudo tee $swappiness_file > /dev/null
-              done
-            ) || true
 
             # use "madvise" Linux Transparent HugePages (THP) setting
             # https://www.kernel.org/doc/html/latest/admin-guide/mm/transhuge.html
             # "madvise" is generally a better option than the default "always" setting
-            # Based on Azul instructions from https://docs.azul.com/prime/Enable-Huge-Pages#transparent-huge-pages-thp
+            # recommendation from https://netflixtechblog.com/bending-pause-times-to-your-will-with-generational-zgc-256629c9386b
             echo madvise | sudo tee /sys/kernel/mm/transparent_hugepage/enabled
             echo advise | sudo tee /sys/kernel/mm/transparent_hugepage/shmem_enabled
-            echo defer+madvise | sudo tee /sys/kernel/mm/transparent_hugepage/defrag
+            echo defer | sudo tee /sys/kernel/mm/transparent_hugepage/defrag
             echo 1 | sudo tee /sys/kernel/mm/transparent_hugepage/khugepaged/defrag
 
             # tune filesystem mount options, https://www.kernel.org/doc/Documentation/filesystems/ext4.txt
             # commit=999999, effectively disables automatic syncing to disk (default is every 5 seconds)
             # nobarrier/barrier=0, loosen data consistency on system crash (no negative impact to empheral CI nodes)
             sudo mount -o remount,nodiscard,commit=999999,barrier=0 / || true
-            sudo mount -o remount,nodiscard,commit=999999,barrier=0 /mnt || true
+            if mountpoint -q /mnt; then
+              sudo mount -o remount,nodiscard,commit=999999,barrier=0 /mnt || true
+            fi
             # disable discard/trim at device level since remount with nodiscard doesn't seem to be effective
             # https://www.spinics.net/lists/linux-ide/msg52562.html
             for i in /sys/block/sd*/queue/discard_max_bytes; do
               echo 0 | sudo tee $i
             done
-            # disable any background jobs that run SSD discard/trim
-            sudo systemctl disable fstrim.timer || true
-            sudo systemctl stop fstrim.timer || true
-            sudo systemctl disable fstrim.service || true
-            sudo systemctl stop fstrim.service || true
+            # disable unnecessary timers
+            sudo systemctl stop fstrim.timer fstrim.service \
+              podman-auto-update.timer sysstat-collect.timer sysstat-summary.timer \
+              phpsessionclean.timer man-db.timer motd-news.timer \
+              dpkg-db-backup.timer e2scrub_all.timer \
+              update-notifier-download.timer update-notifier-motd.timer || true
 
-            # stop php-fpm
-            sudo systemctl stop php8.0-fpm.service || true
-            sudo systemctl stop php7.4-fpm.service || true
-            # stop mono-xsp4
-            sudo systemctl disable mono-xsp4.service || true
-            sudo systemctl stop mono-xsp4.service || true
-            sudo killall mono || true
-
-            # stop Azure Linux agent to save RAM
-            sudo systemctl stop walinuxagent.service || true
+            # stop unnecessary services
+            sudo systemctl stop php8.3-fpm.service ModemManager.service \
+              multipathd.service udisks2.service walinuxagent.service || true
 
             echo '::endgroup::'
 
@@ -87,10 +77,4 @@ runs:
             echo "::group::Available diskspace"
             df -BM
             echo "::endgroup::"
-            # show cggroup
-            echo "::group::Cgroup settings for current cgroup $CURRENT_CGGROUP"
-            CURRENT_CGGROUP=$(cat /proc/self/cgroup | grep '0::' | awk -F: '{ print $3 }')
-            sudo cgget -a $CURRENT_CGGROUP || true
-            echo '::endgroup::'
-        fi
-      shell: bash
+        fi

.github/workflows/ci-go-functions.yaml

@@ -31,9 +31,6 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
-env:
-  MAVEN_OPTS: -Xss1500k -Xmx1024m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
-
 jobs:
   preconditions:
     name: Preconditions

.github/workflows/ci-gradle-cache-update.yaml

@@ -36,13 +36,18 @@ on:
   # trigger on a schedule so that the cache will be rebuilt if it happens to expire
   schedule:
     - cron: '30 */12 * * *'
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 env:
   JDK_DISTRIBUTION: corretto
 
 jobs:
-  update-gradle-dependencies-cache:
-    name: Update Gradle dependency cache
+  update-gradle-cache:
+    name: Update Gradle cache
     runs-on: ubuntu-24.04
     timeout-minutes: 45
 
@@ -53,31 +58,18 @@ jobs:
       - name: Tune Runner VM
         uses: ./.github/actions/tune-runner-vm
 
-      - name: Detect changed files
-        if: ${{ github.event_name != 'schedule' }}
-        id: changes
-        uses: apache/pulsar-test-infra/paths-filter@master
-        with:
-          filters: |
-            build_files:
-              - '*.gradle.kts'
-              - '**/*.gradle.kts'
-              - 'gradle/**'
-              - 'gradle.properties'
-
-      - name: Set up JDK 17
+      - name: Set up JDK 21
         uses: actions/setup-java@v5
-        if: ${{ github.event_name == 'schedule' || steps.changes.outputs.build_files == 'true' }}
         with:
           distribution: ${{ env.JDK_DISTRIBUTION }}
-          java-version: 17
+          java-version: 21
 
       - name: Setup Gradle
-        if: ${{ github.event_name == 'schedule' || steps.changes.outputs.build_files == 'true' }}
-        uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c
+        uses: ./.github/actions/setup-gradle
+        with:
+          develocity-access-key: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
+          cache-read-only: false
 
-      - name: Download dependencies
-        if: ${{ github.event_name == 'schedule' || steps.changes.outputs.build_files == 'true' }}
+      - name: Run Gradle build
         run: |
-          # resolve all dependencies to populate Gradle cache
-          ./gradlew dependencies --no-configuration-cache || true
+          ./gradlew assemble

.github/workflows/ci-semantic-pull-request.yml

@@ -34,7 +34,7 @@ jobs:
     name: Check pull request title
     runs-on: ubuntu-latest
     steps:
-      - uses: amannn/action-semantic-pull-request@v5.5.2
+      - uses: amannn/action-semantic-pull-request@48f256284bd46cdaab1048c3721360e808335d50 # v6.1.1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with: