[debug] Isolate replicateBlobV2MultipleCases flake

.github/workflows/github-actions.yml

@@ -38,6 +38,7 @@ jobs:
   # ============================================================
 
   unit-test-clustermap:
+    if: false  # DEBUG branch — only server-int-test runs
     runs-on: ubuntu-latest
     timeout-minutes: 60
     steps:
@@ -49,6 +50,7 @@ jobs:
           job-id-suffix: clustermap
 
   unit-test-network:
+    if: false  # DEBUG branch — only server-int-test runs
     runs-on: ubuntu-latest
     timeout-minutes: 60
     steps:
@@ -60,6 +62,7 @@ jobs:
           job-id-suffix: network
 
   unit-test-frontend:
+    if: false  # DEBUG branch — only server-int-test runs
     runs-on: ubuntu-latest
     timeout-minutes: 60
     steps:
@@ -71,6 +74,7 @@ jobs:
           job-id-suffix: frontend
 
   unit-test-mysql-stack:
+    if: false  # DEBUG branch — only server-int-test runs
     runs-on: ubuntu-latest
     timeout-minutes: 60
     steps:
@@ -85,6 +89,7 @@ jobs:
           needs-mysql: 'true'
 
   unit-test-azure-stack:
+    if: false  # DEBUG branch — only server-int-test runs
     runs-on: ubuntu-latest
     timeout-minutes: 60
     steps:
@@ -97,6 +102,7 @@ jobs:
           needs-azurite: 'true'
 
   unit-test-protocols:
+    if: false  # DEBUG branch — only server-int-test runs
     runs-on: ubuntu-latest
     timeout-minutes: 60
     steps:
@@ -108,6 +114,7 @@ jobs:
           job-id-suffix: protocols
 
   unit-test-utility-modules:
+    if: false  # DEBUG branch — only server-int-test runs
     runs-on: ubuntu-latest
     timeout-minutes: 60
     steps:
@@ -123,10 +130,8 @@ jobs:
   # a per-module unit-test-file-transfer job here once the path is operational.
 
   store-test:
-
+    if: false  # DEBUG branch — only server-int-test runs
     runs-on: ubuntu-latest
-    # Hard cap matches unit-test's: prevents runaway hangs from consuming
-    # full GitHub-default 6h timeout if a test wedges.
     timeout-minutes: 60
     steps:
       - name: Checkout Ambry
@@ -156,7 +161,7 @@ jobs:
         timeout-minutes: 2
 
   int-test:
-
+    if: false  # DEBUG branch — only server-int-test runs
     runs-on: ubuntu-latest
     timeout-minutes: 60
     steps:
@@ -246,7 +251,10 @@ jobs:
         name: Run integration tests
         with:
           job-id: jdk11
-          arguments: --scan --warning-mode=summary :ambry-server:intTest codeCoverageReport
+          # DEBUG branch: isolate replicateBlobV2MultipleCases. No codeCoverageReport
+          # (skips coverage build), --info for verbose gradle output, --tests filter
+          # to run only the failing test class+method across all parameter variants.
+          arguments: --scan --info --warning-mode=summary :ambry-server:intTest --tests "com.github.ambry.server.ServerHttp2Test.replicateBlobV2MultipleCases*"
           gradle-version: wrapper
 
       - name: Upload coverage to Codecov

build.gradle

@@ -253,16 +253,11 @@ subprojects {
                 logger.lifecycle "    suite total: ${result.testCount} tests, ${result.successfulTestCount} passed, ${result.failedTestCount} failed, ${result.skippedTestCount} skipped"
             }
         }
-        // Allow for retrying flaky integration tests.
+        // DEBUG branch (snalli/debug-replicate-blobv2): retries off so the FIRST
+        // failure surfaces with full debug logs, not the retried-and-masked one.
         retry {
-            // The maximum number of times to retry an individual test
-            maxRetries = 2
-            // The maximum number of test failures that are allowed (per module) before retrying is disabled. The count applies to
-            // each round of test execution. For example, if maxFailures is 5 and 4 tests initially fail and then 3
-            // again on retry, this will not be considered too many failures and retrying will continue (if maxRetries {@literal >} 1).
-            // If 5 or more tests were to fail initially then no retry would be attempted.
+            maxRetries = 0
             maxFailures = 5
-            // Whether tests that initially fail and then pass on retry should fail the task.
             failOnPassedAfterRetry = false
         }
         maxHeapSize = "6g"

log4j-test-config/src/main/resources/log4j2.xml

@@ -11,7 +11,12 @@
         </File>
     </Appenders>
     <Loggers>
-        <Root level="info">
+        <!-- DEBUG branch (snalli/debug-replicate-blobv2): root at debug + the three
+             previously-silenced loggers re-enabled. Goal is to capture replication
+             trail when ServerHttp2Test.replicateBlobV2MultipleCases fails with
+             "expected:<BlobNotFound> but was:<NoError>" (and the other ReplicaUnavailable
+             pattern) so we can see what state the test sees vs what it expects. -->
+        <Root level="debug">
             <AppenderRef ref="consoleAppender"/>
         </Root>
 
@@ -22,11 +27,10 @@
         <Logger name="org.apache.helix" level="warn" />
         <Logger name="org.apache.zookeeper" level="warn" />
 
-
-        <!-- these three are disabled because MockClusterMap does not currently mark nodes offline
-        when requests fail, which happens a lot in the cases that our end to end tests cover. -->
-        <Logger name="com.github.ambry.network.BlockingChannelConnectionPool" level="off" />
-        <Logger name="com.github.ambry.network.BlockingChannelInfo" level="off" />
-        <Logger name="com.github.ambry.replication.ReplicThread" level="off" />
+        <!-- Re-enabled for debug — these are the three loggers most relevant to
+             replicateBlobV2 flow. -->
+        <Logger name="com.github.ambry.network.BlockingChannelConnectionPool" level="debug" />
+        <Logger name="com.github.ambry.network.BlockingChannelInfo" level="debug" />
+        <Logger name="com.github.ambry.replication.ReplicThread" level="debug" />
     </Loggers>
 </Configuration>