ci: batch masternode rollouts, peering jobs, RPC updates, devnet-k8 c…

.github/workflows/ci.yml

@@ -126,45 +126,83 @@ jobs:
     runs-on: ubuntu-latest
     if: github.ref == 'refs/heads/dev-upgrade' && !startsWith(github.ref, 'refs/tags/')
     needs: devnet_build_push
+    concurrency:
+      group: devnet-k8-apply
+      cancel-in-progress: false
     steps:
       - uses: actions/checkout@v4
       - name: Set up kubectl
         uses: azure/setup-kubectl@v4
         with:
           version: "v1.19.11"
-      - name: Apply new image version to Kubernetes deployments
+      - name: Apply new image version to Kubernetes deployments in batches
         run: |
           mkdir -p $HOME/.kube
           echo "${{ secrets.KUBE_CONFIG }}" | base64 --decode > $HOME/.kube/config
           chmod 600 $HOME/.kube/config
+
           git_hash=$(git rev-parse --short "$GITHUB_SHA")
-          masternodes=$(kubectl get deployment -o json | jq -r '.items[] | select(.metadata.name | test("^masternode[0-9]")) | .metadata.name' | sort)
-          rpcs=$(kubectl get deployment -o json| jq -r '.items[] | select(.metadata.name | test("rpc")) | "\(.metadata.name)"')
-          echo "Masternodes: $masternodes"
-          echo "RPCs: $rpcs"
+          image="xinfinorg/devnet:dev-upgrade-${git_hash}"
 
-          # Skip masternode1 through masternode25, update the rest.
+          # Sort masternodes numerically by the trailing number (handles masternode1..masternode125)
+          masternodes=$(kubectl get deployment -o json \
+            | jq -r '.items[] | select(.metadata.name | test("^masternode[0-9]")) | .metadata.name' \
+            | sort -V)
           readarray -t mn_array <<< "$masternodes"
           total=${#mn_array[@]}
-          echo "Total masternode deployments: $total, skipping masternode1..masternode25"
-
-          for name in "${mn_array[@]}"; do
-            if [[ $name =~ ^masternode[0-9] ]]; then
-              num=${name#masternode}
-              num=${num%%-*}
-              if [[ $num =~ ^[0-9]+$ ]] && (( num >= 1 && num <= 25 )); then
-                echo "Skipping deployment: $name"
-                continue
-              fi
-            fi
+          echo "Total masternode deployments: $total"
 
-            echo "Updating deployment: $name"
-            kubectl set image "deployment/$name" "$name=xinfinorg/devnet:dev-upgrade-${git_hash}" || true
-          done
+          BATCH_SIZE=25
+          batch_num=0
+
+          for (( start=0; start<total; start+=BATCH_SIZE )); do
+            batch_num=$(( batch_num + 1 ))
+            end=$(( start + BATCH_SIZE ))
+            (( end > total )) && end=$total
+            echo "=== Batch ${batch_num}: nodes $((start+1))–${end} ==="
+
+            batch_names=()
+            for (( i=start; i<end; i++ )); do
+              name="${mn_array[$i]}"
+              echo "  Updating deployment: $name"
+              kubectl set image "deployment/$name" "$name=$image" || true
+              batch_names+=("$name")
+            done
 
+            # Wait for all pods in this batch to be ready before peering
+            for name in "${batch_names[@]}"; do
+              kubectl rollout status "deployment/$name" --timeout=300s
+            done
+
+            # Trigger peering-cronjob and wait for it to finish before next batch
+            job_name="peering-batch-${batch_num}-$(date +%s)"
+            echo "  Creating peering job: $job_name"
+            kubectl create job "$job_name" --from=cronjob/peering-cronjob -n default
+
+            echo "  Waiting for $job_name to complete..."
+            while true; do
+              complete=$(kubectl get job "$job_name" -n default \
+                -o jsonpath='{.status.conditions[?(@.type=="Complete")].status}')
+              failed=$(kubectl get job "$job_name" -n default \
+                -o jsonpath='{.status.conditions[?(@.type=="Failed")].status}')
+              if [[ "$complete" == "True" ]]; then
+                echo "  Batch ${batch_num} peering done."
+                break
+              elif [[ "$failed" == "True" ]]; then
+                echo "  Peering job $job_name failed — aborting."
+                exit 1
+              fi
+              echo "  Still running, checking again in 30s..."
+              sleep 30
+            done
+          done
 
+          # Update RPC deployments (no peering needed)
+          rpcs=$(kubectl get deployment -o json \
+            | jq -r '.items[] | select(.metadata.name | test("rpc")) | .metadata.name')
+          echo "Updating RPC deployments..."
           while read -r name; do
-            kubectl set image deployment/$name $name=xinfinorg/devnet:dev-upgrade-${git_hash} || true
+            kubectl set image "deployment/$name" "$name=$image" || true
           done <<< "$rpcs"
 
   devnet_send_notification: