diff --git a/apiml-common/src/main/java/org/zowe/apiml/product/eureka/client/ApimlPeerEurekaNode.java b/apiml-common/src/main/java/org/zowe/apiml/product/eureka/client/ApimlPeerEurekaNode.java index 2f92f8e22f..4aaf51ef2a 100644 --- a/apiml-common/src/main/java/org/zowe/apiml/product/eureka/client/ApimlPeerEurekaNode.java +++ b/apiml-common/src/main/java/org/zowe/apiml/product/eureka/client/ApimlPeerEurekaNode.java @@ -364,7 +364,7 @@ class NetworkIssueCounter { final AtomicInteger counter = new AtomicInteger(0); - private String getCountText() { + String getCountText() { int count = counter.get(); StringBuilder sb = new StringBuilder(); @@ -418,6 +418,10 @@ public ProcessingResult process(ReplicationTask task) { } catch (Throwable e) { networkIssueCounter.fail(e.getLocalizedMessage()); if (maybeReadTimeOut(e)) { + if (networkIssueCounter.hasReachedMax()) { + log.error("Socket read timeout has occurred repeatedly and reached the maximum retry count ({}). The replication task will be dropped as a permanent error and the peer will catch up via periodic registry sync.", networkIssueCounter.getCountText()); + return ProcessingResult.PermanentError; + } log.error("It seems to be a socket read timeout exception, it will retry later. if it continues to happen and some eureka node occupied all the cpu time, you should set property 'eureka.server.peer-node-read-timeout-ms' to a bigger value", e); //read timeout exception is more Congestion than TransientError, return Congestion for longer delay return ProcessingResult.Congestion; @@ -455,6 +459,10 @@ public ProcessingResult process(List tasks) { } catch (Throwable e) { networkIssueCounter.fail(e.getLocalizedMessage()); if (maybeReadTimeOut(e)) { + if (networkIssueCounter.hasReachedMax()) { + log.error("Socket read timeout has occurred repeatedly and reached the maximum retry count ({}). Batch replication tasks will be dropped as a permanent error and the peer will catch up via periodic registry sync.", networkIssueCounter.getCountText()); + return ProcessingResult.PermanentError; + } log.error("It seems to be a socket read timeout exception, it will retry later. if it continues to happen and some eureka node occupied all the cpu time, you should set property 'eureka.server.peer-node-read-timeout-ms' to a bigger value", e); //read timeout exception is more Congestion than TransientError, return Congestion for longer delay return ProcessingResult.Congestion; diff --git a/apiml-common/src/test/java/org/zowe/apiml/product/eureka/client/ReplicationTaskProcessorTest.java b/apiml-common/src/test/java/org/zowe/apiml/product/eureka/client/ReplicationTaskProcessorTest.java index 496d68a430..642078e06b 100644 --- a/apiml-common/src/test/java/org/zowe/apiml/product/eureka/client/ReplicationTaskProcessorTest.java +++ b/apiml-common/src/test/java/org/zowe/apiml/product/eureka/client/ReplicationTaskProcessorTest.java @@ -18,6 +18,7 @@ import org.junit.jupiter.api.Test; import javax.net.ssl.SSLException; +import java.net.SocketTimeoutException; import java.util.Collections; import java.util.List; import java.util.stream.IntStream; @@ -130,6 +131,51 @@ void whenNetworkProblemRepeatedMultipleTimes_thenResetCounterAfterSuccessfulConn status = replicationTaskProcessor.process(task2); assertThat(status, is(ProcessingResult.PermanentError)); } + + @Test + void whenReadTimeoutRepeatedMultipleTimes_thenEscalatesToPermanentError() { + TestableInstanceReplicationTask task = aReplicationTask() + .withAction(Action.Heartbeat) + .withException(new SocketTimeoutException("Read timed out")) + .withNetworkFailures(DEFAULT_MAX_RETRIES) + .build(); + + // First read timeout should cause Congestion + ProcessingResult status = replicationTaskProcessor.process(task); + assertThat(status, is(ProcessingResult.Congestion)); + + IntStream.range(1, DEFAULT_MAX_RETRIES - 2).forEach(n -> replicationTaskProcessor.process(task)); + + // 9th read timeout should still cause Congestion + status = replicationTaskProcessor.process(task); + assertThat(status, is(ProcessingResult.Congestion)); + + // 10th read timeout should escalate to PermanentError + status = replicationTaskProcessor.process(task); + assertThat(status, is(ProcessingResult.PermanentError)); + } + + @Test + void whenConnectionFailureRepeatedMultipleTimes_thenBehaviorUnchanged() { + TestableInstanceReplicationTask task = aReplicationTask() + .withAction(Action.Heartbeat) + .withNetworkFailures(DEFAULT_MAX_RETRIES) + .build(); + + // First connection failure should cause TransientError + ProcessingResult status = replicationTaskProcessor.process(task); + assertThat(status, is(ProcessingResult.TransientError)); + + IntStream.range(1, DEFAULT_MAX_RETRIES - 2).forEach(n -> replicationTaskProcessor.process(task)); + + // 9th connection failure should still cause TransientError + status = replicationTaskProcessor.process(task); + assertThat(status, is(ProcessingResult.TransientError)); + + // 10th connection failure should cause PermanentError + status = replicationTaskProcessor.process(task); + assertThat(status, is(ProcessingResult.PermanentError)); + } } @Nested @@ -265,5 +311,50 @@ void whenNetworkProblemRepeatedMultipleTimes_thenResetCounterAfterSuccessfulConn status = replicationTaskProcessor.process(tasks); assertThat(status, is(ProcessingResult.TransientError)); } + + @Test + void whenReadTimeoutRepeatedMultipleTimes_thenEscalatesToPermanentError() { + TestableInstanceReplicationTask task = aReplicationTask().build(); + List tasks = Collections.singletonList(task); + replicationClient.withReadtimeOut(DEFAULT_MAX_RETRIES); + + // First read timeout should cause Congestion + ProcessingResult status = replicationTaskProcessor.process(tasks); + assertThat(status, is(ProcessingResult.Congestion)); + + IntStream.range(1, DEFAULT_MAX_RETRIES - 2).forEach(n -> replicationTaskProcessor.process(tasks)); + + // 9th read timeout should still cause Congestion + status = replicationTaskProcessor.process(tasks); + assertThat(status, is(ProcessingResult.Congestion)); + + // 10th read timeout should escalate to PermanentError + status = replicationTaskProcessor.process(tasks); + assertThat(status, is(ProcessingResult.PermanentError)); + } + + @Test + void whenSuccessfulReplicationResetsReadTimeoutCounter() { + TestableInstanceReplicationTask task = aReplicationTask().build(); + List tasks = Collections.singletonList(task); + + // Accumulate 5 read timeouts (not enough to reach max of 10) + replicationClient.withReadtimeOut(5); + IntStream.range(0, 5).forEach(n -> replicationTaskProcessor.process(tasks)); + + // Successful replication should reset the counter + replicationClient.withReadtimeOut(0); + replicationClient.withBatchReply(200); + replicationClient.withNetworkStatusCode(200); + ProcessingResult status = replicationTaskProcessor.process(tasks); + assertThat(status, is(ProcessingResult.Success)); + + // Now 5 more read timeouts should start fresh (Congestion, not PermanentError) + replicationClient.withReadtimeOut(10); // client counter at 5, so 5 more timeouts fire + for (int i = 0; i < 5; i++) { + status = replicationTaskProcessor.process(tasks); + assertThat(status, is(ProcessingResult.Congestion)); + } + } } } diff --git a/apiml/src/main/resources/application.yml b/apiml/src/main/resources/application.yml index 5c5fe501da..c9e35d8bd8 100644 --- a/apiml/src/main/resources/application.yml +++ b/apiml/src/main/resources/application.yml @@ -16,7 +16,7 @@ eureka: server: max-threads-for-peer-replication: 6 useReadOnlyResponseCache: false - peer-node-read-timeout-ms: 15000 + peer-node-read-timeout-ms: 30000 spring: cloud: gateway: diff --git a/discovery-service/src/main/resources/application.yml b/discovery-service/src/main/resources/application.yml index 43ab0468de..0e8582db5a 100644 --- a/discovery-service/src/main/resources/application.yml +++ b/discovery-service/src/main/resources/application.yml @@ -79,7 +79,7 @@ eureka: server: max-threads-for-peer-replication: 6 useReadOnlyResponseCache: false - peer-node-read-timeout-ms: 15000 + peer-node-read-timeout-ms: 30000 management: endpoints: