diff --git a/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/session/TransferFailureOrderingTest.java b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/session/TransferFailureOrderingTest.java new file mode 100644 index 0000000000..9137a1aa6b --- /dev/null +++ b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/session/TransferFailureOrderingTest.java @@ -0,0 +1,406 @@ +/* + * Copyright (c) 2026 the Eclipse Milo Authors + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.milo.opcua.sdk.client.session; + +import static org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.Unsigned.uint; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import org.eclipse.milo.opcua.sdk.client.OpcUaClient; +import org.eclipse.milo.opcua.sdk.client.OpcUaSession; +import org.eclipse.milo.opcua.sdk.client.SessionActivityListener; +import org.eclipse.milo.opcua.sdk.client.UaSession; +import org.eclipse.milo.opcua.sdk.client.subscriptions.OpcUaSubscription; +import org.eclipse.milo.opcua.sdk.server.EndpointConfig; +import org.eclipse.milo.opcua.sdk.server.OpcUaServer; +import org.eclipse.milo.opcua.sdk.test.DelegatingSessionServiceSet; +import org.eclipse.milo.opcua.sdk.test.ScriptableSubscriptionServiceSet; +import org.eclipse.milo.opcua.sdk.test.TestClient; +import org.eclipse.milo.opcua.sdk.test.TestServer; +import org.eclipse.milo.opcua.stack.core.StatusCodes; +import org.eclipse.milo.opcua.stack.core.UaException; +import org.eclipse.milo.opcua.stack.core.types.builtin.DateTime; +import org.eclipse.milo.opcua.stack.core.types.builtin.StatusCode; +import org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.UInteger; +import org.eclipse.milo.opcua.stack.core.types.structured.ActivateSessionRequest; +import org.eclipse.milo.opcua.stack.core.types.structured.ActivateSessionResponse; +import org.eclipse.milo.opcua.stack.core.types.structured.ResponseHeader; +import org.eclipse.milo.opcua.stack.core.types.structured.TransferResult; +import org.eclipse.milo.opcua.stack.core.types.structured.TransferSubscriptionsRequest; +import org.eclipse.milo.opcua.stack.core.types.structured.TransferSubscriptionsResponse; +import org.eclipse.milo.opcua.stack.core.util.Unit; +import org.eclipse.milo.opcua.stack.transport.server.ServiceRequestContext; +import org.junit.jupiter.api.Test; + +/** + * The ordering between failed-Subscription cleanup and reconnect initialization. + * + *

A replacement Session is forced by refusing re-activation. Its TransferSubscriptions answer + * then either contains mixed Good/Bad operation results or reports an expected unsupported-service + * failure. The failed Subscription holds its overridable callback, widening the scheduling window + * deterministically: cleanup must already be complete and the multi-threaded Session FSM must be + * free to enter Initializing while application notification remains held. + */ +public class TransferFailureOrderingTest { + + private static final long AWAIT_TIMEOUT_MILLIS = 10_000; + private static final long RECONNECT_TIMEOUT_MILLIS = 30_000; + private static final long REQUEST_TIMEOUT_MILLIS = 60_000; + private static final long DISCONNECT_TIMEOUT_MILLIS = 5_000; + + /** + * A failed TransferResult and an expected service-level transfer failure both mean the affected + * Subscription does not exist on the replacement Session. It must therefore be absent before + * Initializing can lead to onSessionActive recovery, while a successfully transferred peer is + * retained. + */ + @Test + void mixedTransferResultsAreRemovedBeforeReconnectInitialization() throws Exception { + transferFailureCallbacksDoNotDelayReconnectInitialization(TransferResponse.MIXED_RESULTS); + } + + @Test + void unsupportedTransferIsRemovedBeforeReconnectInitialization() throws Exception { + transferFailureCallbacksDoNotDelayReconnectInitialization(TransferResponse.SERVICE_UNSUPPORTED); + } + + private void transferFailureCallbacksDoNotDelayReconnectInitialization(TransferResponse response) + throws Exception { + + try (var fixture = new Fixture(response, FailureNotification.HOLD)) { + fixture.beginReplacementSessionReconnect(); + fixture.awaitTransferFailureNotification(); + + assertTrue( + fixture.awaitReconnectInitialization(RECONNECT_TIMEOUT_MILLIS), + "a blocking transfer-failure callback kept the replacement Session in Transferring"); + + List subscriptionsAtActive = fixture.awaitSubscriptionsAtSessionActive(); + List expected = + response == TransferResponse.MIXED_RESULTS + ? List.of(fixture.successfulSubscription) + : List.of(); + + assertEquals( + expected, + subscriptionsAtActive, + "onSessionActive observed Subscriptions that were not transferred to the replacement" + + " Session"); + assertTrue( + fixture.failedSubscription.getSubscriptionId().isEmpty(), + "the failed Subscription was not reset before the Session became Active"); + + fixture.releaseTransferFailureNotification(); + } + } + + /** + * Application callbacks must not be able to strand the Session FSM in Transferring. If an + * overridden notification fails before delegating, the local reset is still required before the + * reconnect proceeds. + */ + @Test + void exceptionFromTransferFailureNotificationDoesNotWedgeReconnect() throws Exception { + try (var fixture = new Fixture(TransferResponse.MIXED_RESULTS, FailureNotification.THROW)) { + + fixture.beginReplacementSessionReconnect(); + fixture.awaitTransferFailureNotification(); + + assertTrue( + fixture.awaitReconnectInitialization(RECONNECT_TIMEOUT_MILLIS), + "an exception from transfer-failure notification wedged the Session FSM"); + assertEquals( + List.of(fixture.successfulSubscription), + fixture.awaitSubscriptionsAtSessionActive(), + "the notification failed before delegating, but internal cleanup did not reset the" + + " failed Subscription"); + assertTrue( + fixture.failedSubscription.getSubscriptionId().isEmpty(), + "internal cleanup did not clear the failed SubscriptionId"); + } + } + + private enum TransferResponse { + MIXED_RESULTS, + SERVICE_UNSUPPORTED + } + + private enum FailureNotification { + HOLD, + THROW + } + + /** Refuses one re-activation, forcing the FSM to create a replacement Session. */ + private static final class RefusingSessionServiceSet extends DelegatingSessionServiceSet { + + private final AtomicBoolean refuseNextActivation = new AtomicBoolean(false); + + RefusingSessionServiceSet(OpcUaServer server) { + super(server); + } + + @Override + public ActivateSessionResponse onActivateSession( + ServiceRequestContext context, ActivateSessionRequest request) throws UaException { + + if (refuseNextActivation.compareAndSet(true, false)) { + throw new UaException(StatusCodes.Bad_SessionIdInvalid); + } + + return super.onActivateSession(context, request); + } + } + + /** Supplies either mixed operation results or an expected service-level failure. */ + private static final class TransferSubscriptionServiceSet + extends ScriptableSubscriptionServiceSet { + + private volatile TransferResponse response; + private volatile UInteger failedSubscriptionId; + + TransferSubscriptionServiceSet(OpcUaServer server) { + super(server); + } + + @Override + public TransferSubscriptionsResponse onTransferSubscriptions( + ServiceRequestContext context, TransferSubscriptionsRequest request) throws UaException { + + TransferResponse response = this.response; + + if (response == null) { + return super.onTransferSubscriptions(context, request); + } else if (response == TransferResponse.SERVICE_UNSUPPORTED) { + throw new UaException(StatusCodes.Bad_ServiceUnsupported); + } + + UInteger[] subscriptionIds = request.getSubscriptionIds(); + int count = subscriptionIds != null ? subscriptionIds.length : 0; + var results = new TransferResult[count]; + + for (int i = 0; i < count; i++) { + StatusCode status = + subscriptionIds[i].equals(failedSubscriptionId) + ? new StatusCode(StatusCodes.Bad_SubscriptionIdInvalid) + : StatusCode.GOOD; + + results[i] = new TransferResult(status, new UInteger[0]); + } + + var responseHeader = + new ResponseHeader( + DateTime.now(), + request.getRequestHeader().getRequestHandle(), + StatusCode.GOOD, + null, + null, + null); + + return new TransferSubscriptionsResponse(responseHeader, results, null); + } + } + + /** Holds or rejects the overridable notification after internal transfer cleanup has run. */ + private static final class ControllableSubscription extends OpcUaSubscription { + + private final FailureNotification behavior; + private final CountDownLatch notificationEntered = new CountDownLatch(1); + private final CountDownLatch notificationGate = new CountDownLatch(1); + + ControllableSubscription(OpcUaClient client, FailureNotification behavior) { + super(client); + this.behavior = behavior; + } + + @Override + public void notifyTransferFailed(StatusCode status) { + notificationEntered.countDown(); + + if (behavior == FailureNotification.THROW) { + throw new IllegalStateException("scripted transfer-failure notification exception"); + } + + try { + if (!notificationGate.await(RECONNECT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { + throw new IllegalStateException("timed out waiting to release transfer-failure cleanup"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } + + super.notifyTransferFailed(status); + } + } + + private static final class Fixture implements AutoCloseable { + + private final ExecutorService executor = + Executors.newFixedThreadPool(4, daemonThreadFactory("transfer-failure-ordering")); + + private final CountDownLatch sessionInactive = new CountDownLatch(1); + private final CountDownLatch reconnectInitializationStarted = new CountDownLatch(1); + private final CompletableFuture> subscriptionsAtSessionActive = + new CompletableFuture<>(); + + private final OpcUaServer server; + private final OpcUaClient client; + private final TransferSubscriptionServiceSet subscriptionServiceSet; + private final RefusingSessionServiceSet sessionServiceSet; + + private final ControllableSubscription failedSubscription; + private final OpcUaSubscription successfulSubscription; + + Fixture(TransferResponse response, FailureNotification notification) throws Exception { + server = TestServer.create().getServer(); + subscriptionServiceSet = new TransferSubscriptionServiceSet(server); + sessionServiceSet = new RefusingSessionServiceSet(server); + + for (EndpointConfig endpoint : server.getConfig().getEndpoints()) { + server.addServiceSet(endpoint.getPath(), subscriptionServiceSet); + server.addServiceSet(endpoint.getPath(), sessionServiceSet); + } + + server.startup().get(); + + client = + TestClient.create( + server, + transportConfig -> transportConfig.setExecutor(executor), + config -> + config + .setRequestTimeout(uint(REQUEST_TIMEOUT_MILLIS)) + .setKeepAliveInterval(uint(REQUEST_TIMEOUT_MILLIS)) + .setMaxPendingPublishRequests(uint(3))); + client.connect(); + + client.addSessionActivityListener( + new SessionActivityListener() { + @Override + public void onSessionInactive(UaSession session) { + sessionInactive.countDown(); + } + + @Override + public void onSessionActive(UaSession session) { + if (sessionInactive.getCount() == 0) { + subscriptionsAtSessionActive.complete(client.getSubscriptions()); + } + } + }); + client.addSessionInitializer( + (OpcUaClient ignoredClient, OpcUaSession ignoredSession) -> { + reconnectInitializationStarted.countDown(); + return CompletableFuture.completedFuture(Unit.VALUE); + }); + + failedSubscription = new ControllableSubscription(client, notification); + failedSubscription.create(); + + successfulSubscription = new OpcUaSubscription(client); + successfulSubscription.create(); + + subscriptionServiceSet.failedSubscriptionId = + failedSubscription.getSubscriptionId().orElseThrow(); + subscriptionServiceSet.response = response; + + assertTrue( + awaitTrue(() -> subscriptionServiceSet.getParkedRequestCount() > 0, AWAIT_TIMEOUT_MILLIS), + "the client did not establish a Publish pipeline before the reconnect"); + } + + void beginReplacementSessionReconnect() { + sessionServiceSet.refuseNextActivation.set(true); + subscriptionServiceSet.enqueueServiceFault(StatusCodes.Bad_SessionIdInvalid); + } + + void awaitTransferFailureNotification() throws Exception { + assertTrue( + sessionInactive.await(RECONNECT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), + "the scripted Session fault did not make the Session inactive"); + assertTrue( + failedSubscription.notificationEntered.await( + RECONNECT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), + "the replacement Session never reached failed-Subscription cleanup"); + } + + boolean awaitReconnectInitialization(long timeoutMillis) throws InterruptedException { + return reconnectInitializationStarted.await(timeoutMillis, TimeUnit.MILLISECONDS); + } + + void releaseTransferFailureNotification() { + failedSubscription.notificationGate.countDown(); + } + + List awaitSubscriptionsAtSessionActive() throws Exception { + return subscriptionsAtSessionActive.get(RECONNECT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); + } + + @Override + public void close() throws Exception { + releaseTransferFailureNotification(); + subscriptionServiceSet.failParkedRequests(StatusCodes.Bad_NoSubscription); + + try { + client.disconnectAsync().get(DISCONNECT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); + } catch (TimeoutException ignored) { + // An assertion can leave reconnect work in flight. Server and executor teardown below + // releases it without obscuring the assertion that failed. + } finally { + try { + server.shutdown().get(AWAIT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); + } finally { + executor.shutdownNow(); + } + } + } + } + + private static ThreadFactory daemonThreadFactory(String prefix) { + var sequence = new AtomicInteger(); + + return runnable -> { + var thread = new Thread(runnable, prefix + "-" + sequence.incrementAndGet()); + thread.setDaemon(true); + return thread; + }; + } + + private static boolean awaitTrue(ThrowingBooleanSupplier condition, long timeoutMillis) + throws Exception { + + long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(timeoutMillis); + + while (System.nanoTime() < deadline) { + if (condition.get()) { + return true; + } + Thread.sleep(10); + } + + return condition.get(); + } + + @FunctionalInterface + private interface ThrowingBooleanSupplier { + boolean get() throws Exception; + } +} diff --git a/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/AbandonedGapAcknowledgementTest.java b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/AbandonedGapAcknowledgementTest.java new file mode 100644 index 0000000000..f5f3dac30e --- /dev/null +++ b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/AbandonedGapAcknowledgementTest.java @@ -0,0 +1,288 @@ +/* + * Copyright (c) 2026 the Eclipse Milo Authors + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.milo.opcua.sdk.client.subscriptions; + +import static org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.Unsigned.uint; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.eclipse.milo.opcua.sdk.client.OpcUaClient; +import org.eclipse.milo.opcua.sdk.server.EndpointConfig; +import org.eclipse.milo.opcua.sdk.server.OpcUaServer; +import org.eclipse.milo.opcua.sdk.test.ScriptableSubscriptionServiceSet; +import org.eclipse.milo.opcua.sdk.test.TestClient; +import org.eclipse.milo.opcua.sdk.test.TestServer; +import org.eclipse.milo.opcua.stack.core.NodeIds; +import org.eclipse.milo.opcua.stack.core.StatusCodes; +import org.eclipse.milo.opcua.stack.core.UaException; +import org.eclipse.milo.opcua.stack.core.types.builtin.DataValue; +import org.eclipse.milo.opcua.stack.core.types.builtin.Variant; +import org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.UInteger; +import org.eclipse.milo.opcua.stack.core.types.structured.MonitoredItemNotification; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * What the client acknowledges when it gives a gap up as lost data. + * + *

Part 4 §5.14.7.1, of availableSequenceNumbers: "The Client should acknowledge all Messages in + * this list for which it will not request retransmission." An unacknowledged NotificationMessage + * stays in the Server's retransmission queue, is re-advertised in the availableSequenceNumbers of + * every subsequent PublishResponse, and holds its memory for the life of the Subscription. So when + * the client decides a gap is too large to recover — {@code missingSequenceNumbers} reports it as + * lost data and never asks for any of it — the messages it is abandoning have to be acknowledged, + * or the Server holds them forever for a retransmission that will never be requested. That + * acknowledgement is what 93bedb32c added. + * + *

This deliberately acknowledges NotificationMessages the client never received, which is + * the opposite of what {@link PublishAcknowledgementTest} and {@code + * PublishSequenceRecoveryTest.InitialKeepAlive} pin. Both rules are correct, and the distinction is + * whether the client might still want the message: + * + *

+ * + *

The two tests below are that pair: one drives a gap the client abandons and asserts the + * abandoned sequence numbers are acknowledged, the other drives a gap the client does try to + * recover and asserts that a message it failed to recover is not acknowledged. + */ +public class AbandonedGapAcknowledgementTest { + + /** How long to wait for something that must happen. */ + private static final long AWAIT_TIMEOUT_MILLIS = 5_000; + + /** + * Long enough that nothing times out on its own, so a parked Publish request stays parked and any + * failure observed below is scripted rather than incidental. + */ + private static final long REQUEST_TIMEOUT_MILLIS = 60_000; + + private TestServer testServer; + private OpcUaServer server; + private OpcUaClient client; + private ScriptableSubscriptionServiceSet scriptable; + private OpcUaSubscription subscription; + private UInteger subscriptionId; + private UInteger clientHandle; + + /** Every {@code retransmitSequenceNumber} the client has asked the Server to Republish. */ + private final List republishRequests = Collections.synchronizedList(new ArrayList<>()); + + private final AtomicInteger dataReceivedCount = new AtomicInteger(); + private final AtomicInteger notificationDataLostCount = new AtomicInteger(); + + @BeforeEach + void startClientAndServerAndCreateSubscription() throws Exception { + testServer = TestServer.create(); + server = testServer.getServer(); + + scriptable = new ScriptableSubscriptionServiceSet(server); + for (EndpointConfig endpoint : server.getConfig().getEndpoints()) { + server.addServiceSet(endpoint.getPath(), scriptable); + } + + // Models a Server that has already evicted or cannot produce the requested message, and records + // that it was asked at all. + scriptable.setRepublishResponder( + request -> { + republishRequests.add(request.getRetransmitSequenceNumber().longValue()); + + throw new UaException(StatusCodes.Bad_MessageNotAvailable); + }); + + server.startup().get(); + + client = TestClient.create(server, cfg -> cfg.setRequestTimeout(uint(REQUEST_TIMEOUT_MILLIS))); + client.connect(); + + subscription = new OpcUaSubscription(client); + subscription.setSubscriptionListener( + new OpcUaSubscription.SubscriptionListener() { + @Override + public void onDataReceived( + OpcUaSubscription subscription, + List items, + List values) { + + dataReceivedCount.incrementAndGet(); + } + + @Override + public void onNotificationDataLost(OpcUaSubscription subscription) { + notificationDataLostCount.incrementAndGet(); + } + }); + subscription.create(); + + subscriptionId = subscription.getSubscriptionId().orElseThrow(); + + // Client-side only: addMonitoredItem assigns the ClientHandle the notification fan-out looks + // notifications up by, and no Server-side item takes part in delivering a scripted one. + var item = OpcUaMonitoredItem.newDataItem(NodeIds.Server_ServerStatus_CurrentTime); + subscription.addMonitoredItem(item); + clientHandle = item.getClientHandle().orElseThrow(); + } + + @AfterEach + void stopClientAndServer() throws Exception { + scriptable.failParkedRequests(StatusCodes.Bad_NoSubscription); + try { + client.disconnectAsync().get(5, TimeUnit.SECONDS); + } finally { + server.shutdown().get(5, TimeUnit.SECONDS); + } + } + + /** + * The defect. NotificationMessage 1 is received, then 10 arrives with only 2 and 3 still in the + * Server's retransmission queue. The gap 2..9 is eight messages and the Server is advertising + * two, so it is not recoverable and {@code missingSequenceNumbers} gives it up as lost data + * without requesting any of it. The two the Server is still holding must be acknowledged, because + * nothing will ever ask for them. + */ + @Test + void sequenceNumbersAbandonedWithAnUnrecoverableGapAreAcknowledged() throws Exception { + sendDataChange(1, uint(1)); + assertTrue(awaitAcknowledged(1), "sequence 1 was never acknowledged"); + + // Sequences 2..9 were sent and lost; the Server still holds 2 and 3 and has evicted the rest. + sendDataChange(10, uint(2), uint(3), uint(10)); + + assertTrue( + awaitTrue(() -> dataReceivedCount.get() >= 2), + "the NotificationMessage that revealed the gap was never delivered"); + assertTrue(awaitAcknowledged(10), "sequence 10 was never acknowledged"); + assertTrue( + awaitTrue(() -> notificationDataLostCount.get() >= 1), + "the gap was not given up on as lost data, so the scenario under test did not happen and" + + " the assertions below prove nothing"); + assertEquals( + List.of(), + List.copyOf(republishRequests), + "a gap given up on as lost data must not be requested via Republish; if it was, this is not" + + " the abandoned-gap path"); + + assertTrue( + awaitAcknowledged(2), + "sequence 2 was advertised as available for retransmission and the client has decided never" + + " to request it, so Part 4 §5.14.7.1 requires it to be acknowledged. Unacknowledged," + + " it sits in the Server's retransmission queue — re-advertised in every" + + " PublishResponse — for the life of the Subscription"); + assertTrue( + awaitAcknowledged(3), + "sequence 3 was advertised as available for retransmission and the client has decided never" + + " to request it, so Part 4 §5.14.7.1 requires it to be acknowledged"); + } + + /** + * The companion rule, which the test above must not be read as overturning: while a gap is small + * enough for the client to try to recover it, a message the recovery failed to bring back + * is still not acknowledged. The client never had it, and the Server's copy — if the failure was + * transient — is the only one there is. + * + *

NotificationMessage 1 is received, then 3 arrives with 2 and 3 advertised: a one-message gap + * inside a two-message retransmission queue, so it is recoverable and Republish(2) is attempted. + * The Republish fails, the data is reported lost, and 2 must remain unacknowledged. + */ + @Test + void aSequenceNumberThatFailedToRepublishIsNotAcknowledged() throws Exception { + sendDataChange(1, uint(1)); + assertTrue(awaitAcknowledged(1), "sequence 1 was never acknowledged"); + + // Sequence 2 was sent and lost, and the Server still holds it: a recoverable gap. + sendDataChange(3, uint(2), uint(3)); + + assertEquals( + List.of(2L), + awaitRepublishRequests(), + "the missing NotificationMessage 2 was recoverable and must be requested via Republish; if" + + " it was not, this is not the recoverable-gap path and the assertion below proves" + + " nothing"); + + assertTrue(awaitAcknowledged(3), "sequence 3 was never acknowledged"); + + assertFalse( + acknowledged(2), + "NotificationMessage 2 was requested via Republish and not recovered, so the client never" + + " had it: acknowledging it would let the Server delete the only copy of data the" + + " client asked for. Only a gap the client has decided never to request may be" + + " acknowledged"); + } + + // region fixture helpers + + /** + * Enqueue a PublishResponse carrying a DataChangeNotification at {@code sequenceNumber}, so the + * message is a data message rather than a keep-alive and reaches {@code onDataReceived}. + */ + private void sendDataChange(long sequenceNumber, UInteger... available) { + scriptable.enqueueDataChange( + subscriptionId, + sequenceNumber, + List.of( + new MonitoredItemNotification( + clientHandle, new DataValue(Variant.ofInt32((int) sequenceNumber)))), + available); + } + + private boolean acknowledged(long sequenceNumber) { + return scriptable.getReceivedAcknowledgements().stream() + .anyMatch( + ack -> + ack.getSubscriptionId().equals(subscriptionId) + && ack.getSequenceNumber().longValue() == sequenceNumber); + } + + private boolean awaitAcknowledged(long sequenceNumber) throws Exception { + return awaitTrue(() -> acknowledged(sequenceNumber)); + } + + /** The recorded Republish requests, once at least one has arrived or the timeout has elapsed. */ + private List awaitRepublishRequests() throws Exception { + awaitTrue(() -> !republishRequests.isEmpty()); + + return List.copyOf(republishRequests); + } + + /** Polls {@code condition} until it holds or {@link #AWAIT_TIMEOUT_MILLIS} elapses. */ + private static boolean awaitTrue(ThrowingBooleanSupplier condition) throws Exception { + long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(AWAIT_TIMEOUT_MILLIS); + + while (System.nanoTime() < deadline) { + if (condition.get()) { + return true; + } + Thread.sleep(25); + } + + return condition.get(); + } + + @FunctionalInterface + private interface ThrowingBooleanSupplier { + boolean get() throws Exception; + } + + // endregion +} diff --git a/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/GapRepairSessionBindingTest.java b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/GapRepairSessionBindingTest.java new file mode 100644 index 0000000000..eb6f8855dc --- /dev/null +++ b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/GapRepairSessionBindingTest.java @@ -0,0 +1,678 @@ +/* + * Copyright (c) 2026 the Eclipse Milo Authors + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.milo.opcua.sdk.client.subscriptions; + +import static org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.Unsigned.uint; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import org.eclipse.milo.opcua.sdk.client.OpcUaClient; +import org.eclipse.milo.opcua.sdk.client.SessionActivityListener; +import org.eclipse.milo.opcua.sdk.client.UaSession; +import org.eclipse.milo.opcua.sdk.server.EndpointConfig; +import org.eclipse.milo.opcua.sdk.server.OpcUaServer; +import org.eclipse.milo.opcua.sdk.test.DelegatingSessionServiceSet; +import org.eclipse.milo.opcua.sdk.test.ScriptableSubscriptionServiceSet; +import org.eclipse.milo.opcua.sdk.test.TestClient; +import org.eclipse.milo.opcua.sdk.test.TestServer; +import org.eclipse.milo.opcua.stack.core.NodeIds; +import org.eclipse.milo.opcua.stack.core.StatusCodes; +import org.eclipse.milo.opcua.stack.core.UaException; +import org.eclipse.milo.opcua.stack.core.types.builtin.DataValue; +import org.eclipse.milo.opcua.stack.core.types.builtin.ExtensionObject; +import org.eclipse.milo.opcua.stack.core.types.builtin.Variant; +import org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.UInteger; +import org.eclipse.milo.opcua.stack.core.types.structured.ActivateSessionRequest; +import org.eclipse.milo.opcua.stack.core.types.structured.ActivateSessionResponse; +import org.eclipse.milo.opcua.stack.core.types.structured.DataChangeNotification; +import org.eclipse.milo.opcua.stack.core.types.structured.MonitoredItemNotification; +import org.eclipse.milo.opcua.stack.core.types.structured.RepublishRequest; +import org.eclipse.milo.opcua.stack.core.types.structured.RepublishResponse; +import org.eclipse.milo.opcua.stack.transport.server.ServiceRequestContext; +import org.junit.jupiter.api.Test; + +/** + * Which Session the reactive gap repair sends its Republish requests on. + * + *

When a PublishResponse reveals a gap in a Subscription's NotificationMessage sequence, the + * client recovers the missing messages with Republish (Part 4 §5.14.6) while that Subscription's + * processing queue is paused, which is what orders the recovered messages ahead of the one that + * revealed the gap. The repair is a chain of round trips, and a Session can disappear in the middle + * of it — the fault that took it away may well be what created the gap. + * + *

Resolving the Session again for each request in the chain is what makes that dangerous. A + * request issued while the Session is being re-established finds no Session and waits for the next + * one, and a repair that waits is a processing queue that stays paused. Two things are then stuck + * behind it. The Subscription's own gap never closes, so the NotificationMessage that revealed it + * is never delivered — the client is waiting for a Session in order to ask for data the Session it + * already had could have given it. And the reconnect recovery of Part 4 §6.7 is queued on that same + * paused queue, so the recovery of every Subscription on the client is incomplete, the + * Publish suspension gate never opens, and Subscriptions with nothing missing get no Publish + * traffic either. + * + *

The Session outage is bounded by the test, not by timing: the Server holds the reconnect's + * ActivateSession until the test releases it, so "while no Session is available" is a window with + * ends the test controls. + */ +public class GapRepairSessionBindingTest { + + /** How long to wait for something that must happen. */ + private static final long AWAIT_TIMEOUT_MILLIS = 10_000; + + /** + * How long to wait for work a parked repair would be holding up. A repair that sends its + * remaining requests on the Session it already has gets through a loopback Republish round trip + * in single-digit milliseconds, so this is generous by three orders of magnitude — but a repair + * waiting for a Session that the test is holding back does not finish at all, so no larger value + * would change an outcome. + */ + private static final long STALL_WINDOW_MILLIS = 5_000; + + /** + * The Session FSM waits one second in {@code ReactivatingWait} before its first re-activation + * attempt and doubles the wait on every failure; this window allows for several attempts. + */ + private static final long RECONNECT_TIMEOUT_MILLIS = 30_000; + + /** + * Long enough that nothing times out on its own: no parked Publish request, no Republish, and no + * Session keep-alive. Every stall asserted against below is therefore the client's own doing. + */ + private static final long REQUEST_TIMEOUT_MILLIS = 60_000; + + /** Upper bound on how long a gated request is held, so nothing hangs indefinitely. */ + private static final long GATE_TIMEOUT_MILLIS = 30_000; + + private static final long DISCONNECT_TIMEOUT_MILLIS = 5_000; + + /** The last NotificationMessage each Subscription accounts for before the gap is created. */ + private static final long FIRST_SEQUENCE_NUMBER = 1; + + /** NotificationMessages 2, 3 and 4 are missing when 5 arrives. */ + private static final long GAP_REVEALED_BY = 5; + + private static final long FIRST_MISSING = 2; + private static final long SECOND_MISSING = 3; + private static final long LAST_MISSING = 4; + + /** + * A repair that outlives the Session it began on must finish on the Session it has, not wait for + * the one being established. Everything it still needs is data the Server was already holding for + * it, and the Session that can ask for it is the one the gap was found on. + */ + @Test + void gapRepairFinishesOnTheSessionInHandWhileTheNextOneIsBeingEstablished() throws Exception { + try (var fixture = new Fixture()) { + Sub repairing = fixture.repairing(); + + fixture.holdFirstRepublish(); + fixture.holdReconnectActivateSession(); + + fixture.revealGap(repairing); + fixture.awaitFirstRepublishHeld(); + + fixture.faultSession(); + fixture.awaitReconnectActivateSessionHeld(); + + fixture.releaseFirstRepublish(); + + assertTrue( + fixture.awaitRepublished(repairing, LAST_MISSING, STALL_WINDOW_MILLIS), + "the repair stopped after its first Republish was answered: the requests still to be made" + + " went looking for a Session, found the one they began on gone, and waited for the" + + " one being established. Republish requests received: " + + fixture.republishRequests()); + + assertEquals( + List.of(1, 2, 3, 4, 5), + fixture.deliveredValues(repairing), + "every recovered NotificationMessage, and then the one that revealed the gap, must reach" + + " the application while the repair's own Session is still the only one there is;" + + " nothing here needs the Session that is being established"); + + assertEquals( + 1, + fixture.sessionReactivatedCount(), + "precondition: the Server is still holding the reconnect's ActivateSession, so the" + + " assertions above must have been met with no new Session available at all"); + } + } + + /** + * The repair pauses one Subscription's processing queue, and the reconnect recovery of Part 4 + * §6.7 is queued on that same queue. A repair that never finishes therefore holds a recovery that + * never finishes, and the Publish suspension gate — which waits for every Subscription's recovery + * — never opens for any of them. A Subscription with nothing missing stops receiving data because + * a different one lost a message. + * + *

The Server stops answering the repair's remaining Republish requests at the moment the + * reconnect completes, so a repair that has already finished by then is unaffected and a repair + * that resumes on the new Session is held. That is the difference the assertion measures. + */ + @Test + void publishTrafficForOtherSubscriptionsIsNotHeldBehindAParkedGapRepair() throws Exception { + try (var fixture = new Fixture()) { + Sub repairing = fixture.repairing(); + Sub healthy = fixture.healthy(); + + fixture.holdFirstRepublish(); + fixture.holdReconnectActivateSession(); + + fixture.revealGap(repairing); + fixture.awaitFirstRepublishHeld(); + + fixture.faultSession(); + fixture.awaitReconnectActivateSessionHeld(); + + fixture.releaseFirstRepublish(); + + // Not asserted: a repair bound to the Session it began on is done by now, and a repair + // waiting + // for the next Session is not. Either way, what follows only holds the requests a repair has + // not made yet. + fixture.awaitRepublished(repairing, LAST_MISSING, STALL_WINDOW_MILLIS); + + fixture.holdRepublishesFor(repairing, SECOND_MISSING, LAST_MISSING); + fixture.releaseReconnectActivateSession(); + fixture.awaitReactivation(); + + fixture.enqueueDataChange(healthy, FIRST_SEQUENCE_NUMBER + 1); + + assertTrue( + fixture.awaitDeliveredValues( + healthy, + List.of((int) FIRST_SEQUENCE_NUMBER, (int) FIRST_SEQUENCE_NUMBER + 1), + STALL_WINDOW_MILLIS), + "a Subscription with nothing missing received no NotificationMessage after the reconnect:" + + " the other Subscription's gap repair resumed on the new Session and is holding its" + + " processing queue paused, so the reconnect recovery queued there cannot run, the" + + " Publish suspension gate never opens, and no PublishRequest is sent for any" + + " Subscription. Delivered: " + + fixture.deliveredValues(healthy)); + } + } + + /** + * The control for the test above: the identical reconnect, with no gap and therefore no repair in + * flight when the Session went away. It proves the fixture's second Subscription does receive + * NotificationMessages once a reconnect is over, so a failure above is about the parked repair + * rather than about a fixture that never delivers anything. + */ + @Test + void publishTrafficForOtherSubscriptionsResumesAfterAReconnectWithNoRepairInFlight() + throws Exception { + + try (var fixture = new Fixture()) { + Sub healthy = fixture.healthy(); + + fixture.holdReconnectActivateSession(); + + fixture.faultSession(); + fixture.awaitReconnectActivateSessionHeld(); + + fixture.releaseReconnectActivateSession(); + fixture.awaitReactivation(); + + fixture.enqueueDataChange(healthy, FIRST_SEQUENCE_NUMBER + 1); + + assertTrue( + fixture.awaitDeliveredValues( + healthy, + List.of((int) FIRST_SEQUENCE_NUMBER, (int) FIRST_SEQUENCE_NUMBER + 1), + STALL_WINDOW_MILLIS), + "control: a Subscription with nothing missing must receive NotificationMessages again" + + " once the reconnect is over. Delivered: " + + fixture.deliveredValues(healthy)); + } + } + + // region fixture + + /** One Subscription: its Server-assigned id and what its listener has observed. */ + private record Sub( + UInteger subscriptionId, UInteger clientHandle, List deliveredValues) {} + + /** + * A {@link DelegatingSessionServiceSet} that can hold one ActivateSession until the test releases + * it, which is how the tests above bound the window during which the client has no Session. + */ + private static final class GatingSessionServiceSet extends DelegatingSessionServiceSet { + + private final AtomicBoolean holdNext = new AtomicBoolean(false); + + private final CountDownLatch heldActivateSession = new CountDownLatch(1); + private final CountDownLatch activateSessionGate = new CountDownLatch(1); + + GatingSessionServiceSet(OpcUaServer server) { + super(server); + } + + @Override + public ActivateSessionResponse onActivateSession( + ServiceRequestContext context, ActivateSessionRequest request) throws UaException { + + if (holdNext.compareAndSet(true, false)) { + heldActivateSession.countDown(); + + await(activateSessionGate); + } + + return super.onActivateSession(context, request); + } + } + + /** + * A running Server whose Publish and Republish responses are scripted and whose reconnect + * ActivateSession can be held, plus a connected client with two Subscriptions: one that will be + * made to lose NotificationMessages, and one that has nothing missing throughout. + * + *

The client is configured for two pending PublishRequests, which is what makes the pipeline + * state unambiguous once a gap has been revealed and the Session faulted: one request carries the + * PublishResponse that reveals the gap, the other carries the Session fault, and nothing is left + * parked at the Server that could answer a later scripted response without the client having sent + * a new request for it. + */ + private static final class Fixture implements AutoCloseable { + + private static final long MAX_PENDING_PUBLISH_REQUESTS = 2; + + /** Every Republish the Server has received, as {@code subscriptionId:sequenceNumber}. */ + private final List republishRequests = Collections.synchronizedList(new ArrayList<>()); + + /** Counted down when a Republish the test asked to have held reaches the Server. */ + private final CountDownLatch firstRepublishHeld = new CountDownLatch(1); + + /** Releases the first held Republish. */ + private final CountDownLatch firstRepublishGate = new CountDownLatch(1); + + /** Releases every Republish held by {@link #holdRepublishesFor}. */ + private final CountDownLatch repairGate = new CountDownLatch(1); + + private final AtomicBoolean holdFirstRepublish = new AtomicBoolean(false); + + /** + * Republish requests, as {@code subscriptionId:sequenceNumber}, held by {@link #repairGate}. + */ + private final Set heldRepairRequests = ConcurrentHashMap.newKeySet(); + + /** + * The sequence numbers the Server is holding for retransmission, per Subscription. Anything + * else is answered Bad_MessageNotAvailable. + */ + private final Map> retransmissionQueues = new ConcurrentHashMap<>(); + + private final CountDownLatch sessionInactive = new CountDownLatch(1); + private final CountDownLatch sessionReactivated = new CountDownLatch(1); + + private final OpcUaServer server; + private final OpcUaClient client; + private final ScriptableSubscriptionServiceSet scriptable; + private final GatingSessionServiceSet sessionServiceSet; + + private final Sub repairing; + private final Sub healthy; + + Fixture() throws Exception { + TestServer testServer = TestServer.create(); + server = testServer.getServer(); + + scriptable = new ScriptableSubscriptionServiceSet(server); + sessionServiceSet = new GatingSessionServiceSet(server); + + for (EndpointConfig endpoint : server.getConfig().getEndpoints()) { + server.addServiceSet(endpoint.getPath(), scriptable); + server.addServiceSet(endpoint.getPath(), sessionServiceSet); + } + + server.startup().get(); + + client = + TestClient.create( + server, + cfg -> + cfg.setRequestTimeout(uint(REQUEST_TIMEOUT_MILLIS)) + // No Session keep-alive traffic: the only requests in flight during a test + // are + // the ones it scripts. + .setKeepAliveInterval(uint(REQUEST_TIMEOUT_MILLIS)) + .setMaxPendingPublishRequests(uint(MAX_PENDING_PUBLISH_REQUESTS))); + client.connect(); + + client.addSessionActivityListener( + new SessionActivityListener() { + @Override + public void onSessionInactive(UaSession session) { + sessionInactive.countDown(); + } + + @Override + public void onSessionActive(UaSession session) { + if (sessionInactive.getCount() == 0) { + sessionReactivated.countDown(); + } + } + }); + + scriptable.setRepublishResponder(this::respondToRepublish); + + repairing = createSubscription(); + healthy = createSubscription(); + + // The Server is holding every NotificationMessage the gap will be made of, and nothing for + // the + // Subscription that has nothing missing. + retransmissionQueues.put( + repairing.subscriptionId(), Set.of(FIRST_MISSING, SECOND_MISSING, LAST_MISSING)); + retransmissionQueues.put(healthy.subscriptionId(), Set.of()); + + deliverFirstNotification(repairing); + deliverFirstNotification(healthy); + + assertTrue( + awaitTrue( + () -> scriptable.getParkedRequestCount() == MAX_PENDING_PUBLISH_REQUESTS, + AWAIT_TIMEOUT_MILLIS), + "the client did not refill its Publish pipeline"); + } + + Sub repairing() { + return repairing; + } + + Sub healthy() { + return healthy; + } + + private Sub createSubscription() throws UaException { + List deliveredValues = Collections.synchronizedList(new ArrayList<>()); + + var subscription = new OpcUaSubscription(client); + subscription.setSubscriptionListener( + new OpcUaSubscription.SubscriptionListener() { + @Override + public void onDataReceived( + OpcUaSubscription s, List items, List values) { + + for (DataValue value : values) { + deliveredValues.add((Integer) value.getValue().getValue()); + } + } + }); + subscription.create(); + + // The MonitoredItem only has to exist on the client: addMonitoredItem assigns the + // ClientHandle + // the notification fan-out looks scripted notifications up by, and no Server-side item + // participates in delivering one. + OpcUaMonitoredItem item = + OpcUaMonitoredItem.newDataItem(NodeIds.Server_ServerStatus_CurrentTime); + subscription.addMonitoredItem(item); + + return new Sub( + subscription.getSubscriptionId().orElseThrow(), + item.getClientHandle().orElseThrow(), + deliveredValues); + } + + /** + * Deliver NotificationMessage {@value #FIRST_SEQUENCE_NUMBER}, leaving this Subscription's last + * accounted-for sequence number at it. + */ + private void deliverFirstNotification(Sub sub) throws Exception { + enqueueDataChange(sub, FIRST_SEQUENCE_NUMBER); + + assertTrue( + awaitTrue(() -> !deliveredValues(sub).isEmpty(), AWAIT_TIMEOUT_MILLIS), + "the first NotificationMessage was never delivered"); + } + + /** + * Script the next PublishResponse as a data message for {@code sub} carrying {@code + * sequenceNumber}, whose value identifies the NotificationMessage it came from. + */ + void enqueueDataChange(Sub sub, long sequenceNumber) { + scriptable.enqueue( + request -> + CompletableFuture.completedFuture( + scriptable.buildPublishResponse( + request, + sub.subscriptionId(), + sequenceNumber, + notificationData(sub, sequenceNumber), + new UInteger[] {uint(sequenceNumber)}, + false))); + } + + /** + * Script a PublishResponse for {@code sub} carrying sequence number {@value #GAP_REVEALED_BY}, + * leaving {@value #FIRST_MISSING} to {@value #LAST_MISSING} missing and advertising every one + * of them as available for retransmission, so the whole gap is recoverable. + */ + void revealGap(Sub sub) { + scriptable.enqueue( + request -> + CompletableFuture.completedFuture( + scriptable.buildPublishResponse( + request, + sub.subscriptionId(), + GAP_REVEALED_BY, + notificationData(sub, GAP_REVEALED_BY), + new UInteger[] { + uint(FIRST_MISSING), + uint(SECOND_MISSING), + uint(LAST_MISSING), + uint(GAP_REVEALED_BY) + }, + false))); + } + + /** Hold the next Republish request the Server receives until the test releases it. */ + void holdFirstRepublish() { + holdFirstRepublish.set(true); + } + + void awaitFirstRepublishHeld() throws Exception { + assertTrue( + firstRepublishHeld.await(AWAIT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), + "precondition: the gap was never detected, so no repair is in flight"); + } + + void releaseFirstRepublish() { + firstRepublishGate.countDown(); + } + + /** + * Stop answering these Republish requests, if they have not been made yet, until teardown. Used + * once the reconnect is about to complete, so that only a repair which resumes on the new + * Session is affected. + */ + void holdRepublishesFor(Sub sub, long... sequenceNumbers) { + for (long sequenceNumber : sequenceNumbers) { + heldRepairRequests.add(republishKey(sub.subscriptionId(), sequenceNumber)); + } + } + + /** Hold the ActivateSession of the next reconnect until the test releases it. */ + void holdReconnectActivateSession() { + sessionServiceSet.holdNext.set(true); + } + + void awaitReconnectActivateSessionHeld() throws Exception { + assertTrue( + sessionInactive.await(RECONNECT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), + "the scripted Bad_SessionIdInvalid Publish fault did not take the Session out of Active"); + assertTrue( + sessionServiceSet.heldActivateSession.await( + RECONNECT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), + "the client never tried to activate a Session again"); + } + + void releaseReconnectActivateSession() { + sessionServiceSet.activateSessionGate.countDown(); + } + + void awaitReactivation() throws Exception { + assertTrue( + sessionReactivated.await(RECONNECT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), + "the Session never became Active again"); + } + + /** + * Answer one parked PublishRequest with a Bad_SessionIdInvalid ServiceFault, which {@code + * SessionFsmFactory}'s SessionFaultListener classifies as a Session error and turns into a + * reconnect. The Server-side Session is untouched, so re-activation succeeds and both + * Subscriptions survive it. + */ + void faultSession() { + scriptable.enqueueServiceFault(StatusCodes.Bad_SessionIdInvalid); + } + + /** + * Answer a Republish the way a Server holding a bounded retransmission queue would: with the + * NotificationMessage if it is holding one with that sequence number for that Subscription, and + * Bad_MessageNotAvailable if it is not — which is also what terminates the Republish loop Part + * 4 §6.7 describes. + */ + private RepublishResponse respondToRepublish(RepublishRequest request) throws UaException { + UInteger subscriptionId = request.getSubscriptionId(); + long sequenceNumber = request.getRetransmitSequenceNumber().longValue(); + String key = republishKey(subscriptionId, sequenceNumber); + + republishRequests.add(key); + + if (holdFirstRepublish.compareAndSet(true, false)) { + firstRepublishHeld.countDown(); + + await(firstRepublishGate); + } else if (heldRepairRequests.contains(key)) { + await(repairGate); + } + + if (!retransmissionQueues.getOrDefault(subscriptionId, Set.of()).contains(sequenceNumber)) { + throw new UaException(StatusCodes.Bad_MessageNotAvailable); + } + + return scriptable.buildRepublishResponse( + request, + sequenceNumber, + notificationData(subscriptionOf(subscriptionId), sequenceNumber)); + } + + private Sub subscriptionOf(UInteger subscriptionId) { + return repairing.subscriptionId().equals(subscriptionId) ? repairing : healthy; + } + + private ExtensionObject[] notificationData(Sub sub, long sequenceNumber) { + var notification = + new MonitoredItemNotification( + sub.clientHandle(), new DataValue(Variant.ofInt32((int) sequenceNumber))); + + return new ExtensionObject[] { + scriptable.encode( + new DataChangeNotification(new MonitoredItemNotification[] {notification}, null)) + }; + } + + List republishRequests() { + return List.copyOf(republishRequests); + } + + boolean awaitRepublished(Sub sub, long sequenceNumber, long timeoutMillis) throws Exception { + String key = republishKey(sub.subscriptionId(), sequenceNumber); + + return awaitTrue(() -> republishRequests.contains(key), timeoutMillis); + } + + List deliveredValues(Sub sub) { + return List.copyOf(sub.deliveredValues()); + } + + boolean awaitDeliveredValues(Sub sub, List expected, long timeoutMillis) + throws Exception { + + return awaitTrue(() -> deliveredValues(sub).equals(expected), timeoutMillis); + } + + long sessionReactivatedCount() { + return sessionReactivated.getCount(); + } + + /** Polls {@code condition} until it holds or the timeout elapses. */ + boolean awaitTrue(ThrowingBooleanSupplier condition, long timeoutMillis) throws Exception { + long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(timeoutMillis); + + while (System.nanoTime() < deadline) { + if (condition.get()) { + return true; + } + Thread.sleep(10); + } + + return condition.get(); + } + + @Override + public void close() throws Exception { + repairGate.countDown(); + firstRepublishGate.countDown(); + sessionServiceSet.activateSessionGate.countDown(); + scriptable.failParkedRequests(StatusCodes.Bad_NoSubscription); + try { + client.disconnectAsync().get(DISCONNECT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); + } catch (TimeoutException ignored) { + // A client whose processing queue is held by a repair that cannot finish may not manage the + // disconnect it is asked for; shutting the Server down below is what releases it. Tolerated + // here so teardown does not mask the assertion that detected the stall. + } finally { + server.shutdown().get(10, TimeUnit.SECONDS); + } + } + } + + private static String republishKey(UInteger subscriptionId, long sequenceNumber) { + return subscriptionId + ":" + sequenceNumber; + } + + /** + * Suspend the calling Server dispatch thread until {@code gate} is released. Bounded, so a test + * that goes wrong fails rather than hangs; the Server dispatches every service request on its own + * executor, so only the thread handling this one request waits. + */ + private static void await(CountDownLatch gate) throws UaException { + try { + if (!gate.await(GATE_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { + throw new UaException(StatusCodes.Bad_Timeout); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + + throw new UaException(StatusCodes.Bad_UnexpectedError, e); + } + } + + @FunctionalInterface + private interface ThrowingBooleanSupplier { + boolean get() throws Exception; + } + + // endregion +} diff --git a/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/MonitoredItemDeletionTest.java b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/MonitoredItemDeletionTest.java new file mode 100644 index 0000000000..00c79c0e36 --- /dev/null +++ b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/MonitoredItemDeletionTest.java @@ -0,0 +1,507 @@ +/* + * Copyright (c) 2026 the Eclipse Milo Authors + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.milo.opcua.sdk.client.subscriptions; + +import static org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.Unsigned.uint; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import org.eclipse.milo.opcua.sdk.client.OpcUaClient; +import org.eclipse.milo.opcua.sdk.server.EndpointConfig; +import org.eclipse.milo.opcua.sdk.server.OpcUaServer; +import org.eclipse.milo.opcua.sdk.test.DelegatingMonitoredItemServiceSet; +import org.eclipse.milo.opcua.sdk.test.TestClient; +import org.eclipse.milo.opcua.sdk.test.TestServer; +import org.eclipse.milo.opcua.stack.core.NodeIds; +import org.eclipse.milo.opcua.stack.core.StatusCodes; +import org.eclipse.milo.opcua.stack.core.UaException; +import org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.UInteger; +import org.eclipse.milo.opcua.stack.core.types.structured.DeleteMonitoredItemsRequest; +import org.eclipse.milo.opcua.stack.core.types.structured.DeleteMonitoredItemsResponse; +import org.eclipse.milo.opcua.stack.transport.server.ServiceRequestContext; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +/** + * Verifies the bookkeeping {@link OpcUaSubscription} keeps for MonitoredItems that have been + * removed from the Subscription but not yet deleted from the Server. + * + *

Removing a MonitoredItem only queues it for deletion; the DeleteMonitoredItems service call + * happens later, in {@link OpcUaSubscription#deleteMonitoredItems()}. Two things must hold for that + * queue: a deletion that did not happen must stay queued, and a deletion that can no longer refer + * to anything on the Server must be discarded. Neither costs anything when everything works, and + * both are the difference between a leaked item and a wrongly deleted one when it doesn't. + */ +public class MonitoredItemDeletionTest { + + /** The queue of pending deletions after a DeleteMonitoredItems service fault. */ + @Nested + class ServiceFault { + + /** + * A DeleteMonitoredItems service fault leaves the MonitoredItem alive on the Server, so it must + * stay queued for deletion: a service fault is precisely the transient failure a caller is + * expected to retry, and the retry has to send the deletion the failed attempt did not perform. + * Dropping it strands the item on the Server, publishing notifications that no longer route to + * any client-side item, with no way for the application to delete it short of deleting the + * whole Subscription. + */ + @Test + void failedDeleteStaysQueuedForTheNextDelete() throws Exception { + try (var fixture = new Fixture()) { + var subscription = new OpcUaSubscription(fixture.client); + subscription.create(); + + OpcUaMonitoredItem item = fixture.createItem(subscription); + UInteger monitoredItemId = item.getMonitoredItemId().orElseThrow(); + + subscription.removeMonitoredItem(item); + + fixture.serviceSet.failNextDeleteMonitoredItems(1); + + List failedResults = + subscription.deleteMonitoredItems(); + + assertEquals(1, failedResults.size()); + assertEquals(item, failedResults.get(0).monitoredItem()); + assertEquals( + StatusCodes.Bad_TooManyOperations, + failedResults.get(0).serviceResult().value(), + "control: the scripted service fault reached the caller"); + assertTrue( + failedResults.get(0).operationResult().isEmpty(), + "control: a service fault means there is no operation-level result"); + + assertFalse( + subscription.isMonitoredItemsSynchronized(), + "the MonitoredItem is still alive on the Server after the failed delete, so the" + + " Subscription's MonitoredItems are not synchronized"); + + // The retry must send the deletion the first attempt failed to perform. + List retryResults = + subscription.deleteMonitoredItems(); + + assertEquals( + 1, retryResults.size(), "the failed deletion must be retried by the next delete"); + assertEquals(item, retryResults.get(0).monitoredItem()); + assertTrue( + retryResults.get(0).isGood(), + "the retried deletion should succeed: " + retryResults.get(0).serviceResult()); + + List> deleteRequests = fixture.serviceSet.getDeletedMonitoredItemIds(); + assertEquals( + List.of(List.of(monitoredItemId), List.of(monitoredItemId)), + deleteRequests, + "the Server should have seen the deletion attempted twice"); + assertTrue(subscription.isMonitoredItemsSynchronized()); + } + } + + /** + * Control for {@link #failedDeleteStaysQueuedForTheNextDelete()}: when the delete succeeds the + * item must leave the pending-deletion queue, so a later deleteMonitoredItems() is a no-op. + * Without this, "keep failed deletions queued" could be satisfied by never dequeuing anything. + */ + @Test + void successfulDeleteLeavesNothingQueued() throws Exception { + try (var fixture = new Fixture()) { + var subscription = new OpcUaSubscription(fixture.client); + subscription.create(); + + OpcUaMonitoredItem item = fixture.createItem(subscription); + UInteger monitoredItemId = item.getMonitoredItemId().orElseThrow(); + + subscription.removeMonitoredItem(item); + + List results = subscription.deleteMonitoredItems(); + + assertEquals(1, results.size()); + assertTrue(results.get(0).isGood()); + assertTrue(subscription.isMonitoredItemsSynchronized()); + + assertEquals(List.of(), subscription.deleteMonitoredItems()); + assertEquals( + List.of(List.of(monitoredItemId)), + fixture.serviceSet.getDeletedMonitoredItemIds(), + "a MonitoredItem that was deleted must not be deleted a second time"); + } + } + + /** + * A MonitoredItem whose deletion failed is still owned by the Subscription, so adding it back + * has to restore it. If the failed deletion is dropped instead, the item keeps a ClientHandle + * that no longer maps to anything, and {@code addMonitoredItem} silently ignores it: the + * application is left holding an item it can neither delete nor use. + */ + @Test + void itemWhoseDeleteFailedCanBeAddedBack() throws Exception { + try (var fixture = new Fixture()) { + var subscription = new OpcUaSubscription(fixture.client); + subscription.create(); + + OpcUaMonitoredItem item = fixture.createItem(subscription); + + subscription.removeMonitoredItem(item); + + fixture.serviceSet.failNextDeleteMonitoredItems(1); + assertTrue( + subscription.deleteMonitoredItems().get(0).serviceResult().isBad(), + "control: the delete attempt failed"); + + subscription.addMonitoredItem(item); + + assertEquals( + List.of(item), + subscription.getMonitoredItems(), + "an item whose deletion failed is still alive on the Server and must be restored to" + + " the Subscription when it is added back"); + assertEquals(OpcUaMonitoredItem.SyncState.SYNCHRONIZED, item.getSyncState()); + } + } + + /** + * Control for {@link #itemWhoseDeleteFailedCanBeAddedBack()}: adding back an item that is still + * queued for deletion cancels the pending deletion. This is the path the failed-delete case has + * to end up on, so it must work independently of any delete attempt. + */ + @Test + void itemQueuedForDeletionCanBeAddedBack() throws Exception { + try (var fixture = new Fixture()) { + var subscription = new OpcUaSubscription(fixture.client); + subscription.create(); + + OpcUaMonitoredItem item = fixture.createItem(subscription); + + subscription.removeMonitoredItem(item); + subscription.addMonitoredItem(item); + + assertEquals(List.of(item), subscription.getMonitoredItems()); + assertEquals(List.of(), subscription.deleteMonitoredItems()); + assertEquals( + List.of(), + fixture.serviceSet.getDeletedMonitoredItemIds(), + "an item that was added back must no longer be pending deletion"); + } + } + + /** + * Adding an item back while DeleteMonitoredItems is in flight makes it desired client state + * again, but cannot retract the request already at the Server. When that request succeeds, the + * item must remain mapped with a valid ClientHandle and become pending creation; otherwise the + * old map key points at an item whose handle was cleared, and the item can neither be recreated + * nor safely added again. + */ + @Test + void itemAddedBackDuringSuccessfulDeleteRemainsMappedForRecreation() throws Exception { + try (var fixture = new Fixture()) { + var subscription = new OpcUaSubscription(fixture.client); + subscription.create(); + + OpcUaMonitoredItem item = fixture.createItem(subscription); + UInteger originalClientHandle = item.getClientHandle().orElseThrow(); + + subscription.removeMonitoredItem(item); + fixture.serviceSet.gateNextDeleteMonitoredItems(); + + CompletableFuture> delete = + CompletableFuture.supplyAsync(subscription::deleteMonitoredItems); + + fixture.serviceSet.awaitDeleteMonitoredItems(); + subscription.addMonitoredItem(item); + + assertEquals( + List.of(item), + subscription.getMonitoredItems(), + "control: the item was added back before the Server answered the deletion"); + + fixture.serviceSet.releaseDeleteMonitoredItems(); + + List deleteResults = delete.get(5, TimeUnit.SECONDS); + + assertEquals(1, deleteResults.size()); + assertTrue(deleteResults.get(0).isGood()); + assertEquals( + originalClientHandle, + item.getClientHandle().orElseThrow(), + "the mapped item must retain the ClientHandle used as its map key"); + assertEquals(List.of(item), subscription.getMonitoredItems()); + assertEquals( + OpcUaMonitoredItem.SyncState.INITIAL, + item.getSyncState(), + "the successful deletion means the re-added item must be recreated on the Server"); + + List createResults = + subscription.createMonitoredItems(); + + assertEquals(1, createResults.size(), "the re-added item must be created exactly once"); + assertTrue(createResults.get(0).isGood()); + assertEquals(List.of(item), subscription.getMonitoredItems()); + assertTrue(subscription.isMonitoredItemsSynchronized()); + } + } + } + + /** The queue of pending deletions across a Subscription's deletion and re-creation. */ + @Nested + class SubscriptionRecreated { + + /** + * MonitoredItemIds are scoped to a Subscription, and Milo's Server assigns them from 1 for + * every new Subscription. A deletion queued against a Subscription that no longer exists must + * therefore be discarded when the Subscription is reset: replaying that MonitoredItemId against + * the replacement Subscription does not fail, it deletes whichever item happens to hold the id + * now. + */ + @Test + void staleMonitoredItemIdIsNotDeletedFromTheNewSubscription() throws Exception { + try (var fixture = new Fixture()) { + var subscription = new OpcUaSubscription(fixture.client); + subscription.create(); + UInteger firstSubscriptionId = subscription.getSubscriptionId().orElseThrow(); + + OpcUaMonitoredItem itemA = fixture.createItem(subscription); + assertEquals( + uint(1), + itemA.getMonitoredItemId().orElseThrow(), + "control: the Server assigns MonitoredItemIds from 1 within a Subscription"); + + // Queue A for deletion, then destroy the Subscription it belongs to before the + // deletion is sent. + subscription.removeMonitoredItem(itemA); + subscription.delete(); + + subscription.create(); + UInteger secondSubscriptionId = subscription.getSubscriptionId().orElseThrow(); + assertNotEquals(firstSubscriptionId, secondSubscriptionId); + + OpcUaMonitoredItem itemB = fixture.createItem(subscription); + assertEquals( + uint(1), + itemB.getMonitoredItemId().orElseThrow(), + "pins the collision: B holds the MonitoredItemId A held in the previous Subscription"); + + fixture.serviceSet.clearDeletedMonitoredItemIds(); + + List results = subscription.deleteMonitoredItems(); + + assertEquals( + List.of(), + fixture.serviceSet.getDeletedMonitoredItemIds(), + "a MonitoredItem belonging to a Subscription that no longer exists must not be" + + " deleted from its replacement"); + assertEquals( + List.of(), + results.stream().map(r -> r.monitoredItem().getReadValueId().getNodeId()).toList(), + "there is nothing left to delete after the Subscription was deleted"); + + // B, the item that held the recycled MonitoredItemId, must still exist on the Server. + subscription.removeMonitoredItem(itemB); + List deleteB = subscription.deleteMonitoredItems(); + + assertEquals(1, deleteB.size()); + assertTrue( + deleteB.get(0).isGood(), + "B should still exist on the Server, but deleting it returned " + + deleteB.get(0).operationResult().orElse(deleteB.get(0).serviceResult())); + } + } + + /** + * A MonitoredItem queued for deletion when the Subscription is reset keeps its SyncState, so + * the Subscription reports itself as never synchronized: {@code isMonitoredItemsSynchronized()} + * stays {@code false} for a Subscription whose items are all created, which is a state no + * caller can act on. + */ + @Test + void newSubscriptionIsSynchronizedAfterAnItemWasQueuedForDeletion() throws Exception { + try (var fixture = new Fixture()) { + var subscription = new OpcUaSubscription(fixture.client); + subscription.create(); + + OpcUaMonitoredItem itemA = fixture.createItem(subscription); + + subscription.removeMonitoredItem(itemA); + subscription.delete(); + subscription.create(); + + assertTrue( + subscription.isMonitoredItemsSynchronized(), + "the new Subscription has no MonitoredItems, and the item queued for deletion no" + + " longer exists on the Server, so there is nothing left to synchronize"); + } + } + + /** + * Control for {@link #newSubscriptionIsSynchronizedAfterAnItemWasQueuedForDeletion()}: a + * delete/create cycle with no pending deletion leaves the Subscription synchronized once its + * items are created again. + */ + @Test + void newSubscriptionIsSynchronizedAfterItemsAreCreatedAgain() throws Exception { + try (var fixture = new Fixture()) { + var subscription = new OpcUaSubscription(fixture.client); + subscription.create(); + + fixture.createItem(subscription); + + subscription.delete(); + subscription.create(); + + assertFalse( + subscription.isMonitoredItemsSynchronized(), + "control: the item still has to be created on the new Subscription"); + assertTrue(subscription.createMonitoredItems().get(0).isGood()); + assertTrue(subscription.isMonitoredItemsSynchronized()); + } + } + } + + // region fixture + + /** + * A {@link DelegatingMonitoredItemServiceSet} that records the MonitoredItemIds of every + * DeleteMonitoredItems request it receives and can fail the next {@code n} of them with a service + * fault. + */ + private static final class RecordingMonitoredItemServiceSet + extends DelegatingMonitoredItemServiceSet { + + private final List> deletedMonitoredItemIds = + Collections.synchronizedList(new ArrayList<>()); + + private final AtomicInteger deleteFailuresRemaining = new AtomicInteger(0); + private final AtomicBoolean gateNextDelete = new AtomicBoolean(false); + private final CountDownLatch deleteArrived = new CountDownLatch(1); + private final CountDownLatch deleteGate = new CountDownLatch(1); + + RecordingMonitoredItemServiceSet(OpcUaServer server) { + super(server); + } + + /** The MonitoredItemIds of each DeleteMonitoredItems request, in the order received. */ + List> getDeletedMonitoredItemIds() { + synchronized (deletedMonitoredItemIds) { + return List.copyOf(deletedMonitoredItemIds); + } + } + + void clearDeletedMonitoredItemIds() { + deletedMonitoredItemIds.clear(); + } + + void failNextDeleteMonitoredItems(int count) { + deleteFailuresRemaining.set(count); + } + + void gateNextDeleteMonitoredItems() { + gateNextDelete.set(true); + } + + void awaitDeleteMonitoredItems() throws InterruptedException { + assertTrue( + deleteArrived.await(5, TimeUnit.SECONDS), + "the gated DeleteMonitoredItems request never reached the Server"); + } + + void releaseDeleteMonitoredItems() { + deleteGate.countDown(); + } + + @Override + public DeleteMonitoredItemsResponse onDeleteMonitoredItems( + ServiceRequestContext context, DeleteMonitoredItemsRequest request) throws UaException { + + UInteger[] ids = request.getMonitoredItemIds(); + deletedMonitoredItemIds.add( + ids == null ? List.of() : Arrays.stream(ids).collect(Collectors.toList())); + + if (deleteFailuresRemaining.getAndDecrement() > 0) { + throw new UaException( + StatusCodes.Bad_TooManyOperations, "scripted DeleteMonitoredItems failure"); + } + + if (gateNextDelete.compareAndSet(true, false)) { + deleteArrived.countDown(); + + try { + if (!deleteGate.await(30, TimeUnit.SECONDS)) { + throw new UaException( + StatusCodes.Bad_Timeout, "the DeleteMonitoredItems gate was never opened"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new UaException(StatusCodes.Bad_UnexpectedError, e); + } + } + + return super.onDeleteMonitoredItems(context, request); + } + } + + /** A running Server whose MonitoredItem service calls are recorded, and a connected client. */ + private static final class Fixture implements AutoCloseable { + + private final OpcUaServer server; + private final OpcUaClient client; + private final RecordingMonitoredItemServiceSet serviceSet; + + Fixture() throws Exception { + TestServer testServer = TestServer.create(); + server = testServer.getServer(); + + serviceSet = new RecordingMonitoredItemServiceSet(server); + for (EndpointConfig endpoint : server.getConfig().getEndpoints()) { + server.addServiceSet(endpoint.getPath(), serviceSet); + } + + server.startup().get(); + + client = TestClient.create(server, cfg -> {}); + client.connect(); + } + + /** Add a MonitoredItem to {@code subscription} and create it on the Server. */ + OpcUaMonitoredItem createItem(OpcUaSubscription subscription) { + var item = OpcUaMonitoredItem.newDataItem(NodeIds.Server_ServerStatus_CurrentTime); + subscription.addMonitoredItem(item); + + List results = subscription.createMonitoredItems(); + assertEquals(1, results.size()); + assertTrue(results.get(0).isGood(), "failed to create the MonitoredItem"); + + return item; + } + + @Override + public void close() throws Exception { + serviceSet.releaseDeleteMonitoredItems(); + try { + client.disconnectAsync().get(5, TimeUnit.SECONDS); + } finally { + server.shutdown().get(5, TimeUnit.SECONDS); + } + } + } + + // endregion +} diff --git a/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/PublishAcknowledgementTest.java b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/PublishAcknowledgementTest.java new file mode 100644 index 0000000000..91a764f3ca --- /dev/null +++ b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/PublishAcknowledgementTest.java @@ -0,0 +1,530 @@ +/* + * Copyright (c) 2026 the Eclipse Milo Authors + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.milo.opcua.sdk.client.subscriptions; + +import static org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.Unsigned.uint; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayOutputStream; +import java.io.OutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import org.eclipse.milo.opcua.sdk.client.OpcUaClient; +import org.eclipse.milo.opcua.sdk.server.EndpointConfig; +import org.eclipse.milo.opcua.sdk.server.OpcUaServer; +import org.eclipse.milo.opcua.sdk.test.ScriptableSubscriptionServiceSet; +import org.eclipse.milo.opcua.sdk.test.TestClient; +import org.eclipse.milo.opcua.sdk.test.TestServer; +import org.eclipse.milo.opcua.stack.core.NodeIds; +import org.eclipse.milo.opcua.stack.core.StatusCodes; +import org.eclipse.milo.opcua.stack.core.UaException; +import org.eclipse.milo.opcua.stack.core.types.builtin.DataValue; +import org.eclipse.milo.opcua.stack.core.types.builtin.ExtensionObject; +import org.eclipse.milo.opcua.stack.core.types.builtin.StatusCode; +import org.eclipse.milo.opcua.stack.core.types.builtin.Variant; +import org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.UInteger; +import org.eclipse.milo.opcua.stack.core.types.structured.DataChangeNotification; +import org.eclipse.milo.opcua.stack.core.types.structured.MonitoredItemNotification; +import org.eclipse.milo.opcua.stack.core.types.structured.PublishRequest; +import org.eclipse.milo.opcua.stack.core.types.structured.PublishResponse; +import org.eclipse.milo.opcua.stack.core.types.structured.SubscriptionAcknowledgement; +import org.eclipse.milo.opcua.stack.transport.server.ServiceRequestContext; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.slf4j.LoggerFactory; + +/** + * What {@code PublishingManager} does with SubscriptionAcknowledgements once it has queued them. + * + *

An acknowledgement is the client's permission for the Server to forget a NotificationMessage: + * Part 4 §5.14.5.2 says of subscriptionAcknowledgements that "the Server may delete the Message + * with this sequence number from its retransmission queue", and §5.14.7.1 that "the Client should + * acknowledge all Messages in this list for which it will not request retransmission". So an + * acknowledgement must be sent for every NotificationMessage the client actually has, and must be + * sent exactly once it can be — never for a message the client does not have, and never + * abandoned, because an acknowledgement that is dropped leaves the Server holding a message + * forever (until its retransmission queue evicts it) and leaves availableSequenceNumbers growing. + * + *

{@code sendPublishRequest} drains the queued acknowledgements into the outgoing PublishRequest + * and clears them. That hands ownership of them to a single network request, and the two nested + * classes below cover the two ways the Server's answer to that request can contradict the client's + * assumption that they arrived: + * + *

    + *
  1. the request fails outright, so the acknowledgements it carried never took effect; + *
  2. the request succeeds but the response's per-acknowledgement results say the Server rejected + * one of them (Part 4 §5.14.5.2: "List of results for the acknowledgements... The size and + * order of the list matches the size and order of the subscriptionAcknowledgements request + * parameter"). + *
+ */ +public class PublishAcknowledgementTest { + + /** How long to wait for something that must happen. */ + private static final long AWAIT_TIMEOUT_MILLIS = 10_000; + + /** + * How long to wait for a client that has lost an acknowledgement to send it again, and for a + * client that has been told an acknowledgement failed to say so. A client that re-queues a lost + * acknowledgement puts it on the very next PublishRequest, which is sent within milliseconds of + * the failed one completing, so this is generous by three orders of magnitude. + */ + private static final long RECOVERY_WINDOW_MILLIS = 5_000; + + /** + * Long enough that nothing times out on its own, so a parked Publish request stays parked and any + * failure observed below is scripted rather than incidental. + */ + private static final long REQUEST_TIMEOUT_MILLIS = 60_000; + + /** + * The client keeps one more PublishRequest in flight than it has Subscriptions, so a single + * Subscription means two. + */ + private static final int PIPELINE_DEPTH = 2; + + /** Distinctive text used to prove the log capture is observing this JVM's SLF4J output. */ + private static final String CAPTURE_PROBE = "publish-acknowledgement-log-capture-probe"; + + /** + * A PublishRequest can fail — a timeout, a dropped connection, a transient ServiceFault — and the + * acknowledgements it was carrying fail with it. They were removed from the client's queue when + * the request was built, so unless they are put back the client has silently decided never to + * acknowledge those NotificationMessages: the Server keeps them in its retransmission queue until + * it evicts them, and re-advertises them in availableSequenceNumbers on every subsequent + * PublishResponse. + * + *

The failure modelled here is the strong one — the request is failed before the Server + * records what it carried, so the acknowledgement is genuinely lost in flight and the Server + * never saw it. That is the case where re-sending is unambiguously required; the weaker case (the + * Server processed the acknowledgement and the response was lost on the way back) merely costs a + * duplicate acknowledgement, which the Server answers with a per-acknowledgement result rather + * than a fault. + */ + @Nested + class AcknowledgementLostWithItsPublishRequest { + + @Test + void acknowledgementIsSentAgainAfterThePublishRequestCarryingItFails() throws Exception { + try (var fixture = new Fixture()) { + fixture.loseNextPublishRequestCarryingAcknowledgement(1); + + fixture.sendDataChange(1, uint(1)); + + assertTrue(fixture.awaitDelivered(1), "NotificationMessage 1 was never delivered"); + assertTrue( + fixture.awaitAcknowledgementLost(), + "no PublishRequest carrying the acknowledgement for sequence 1 ever reached the" + + " Server, so the scenario under test did not happen"); + + // Keep the Publish pipeline turning so a re-queued acknowledgement has requests to ride on + // whether it is restored when the failure is handled or only when the next response is. + fixture.sendDataChange(2, uint(1), uint(2)); + assertTrue(fixture.awaitDelivered(2), "NotificationMessage 2 was never delivered"); + + assertTrue( + fixture.awaitAcknowledged(1, RECOVERY_WINDOW_MILLIS), + "NotificationMessage 1 was received and delivered, but the only acknowledgement for it" + + " went out on a PublishRequest that failed and was never sent again: the Server" + + " is left holding a message the client already has, and re-advertising it in" + + " availableSequenceNumbers, until its retransmission queue evicts it"); + } + } + + /** + * The control that keeps the assertion above from passing vacuously: the identical script with + * nothing failed. If the client did not acknowledge sequence 1 here either, the test above + * would be measuring the fixture rather than the loss. + */ + @Test + void acknowledgementIsSentWhenThePublishRequestCarryingItSucceeds() throws Exception { + try (var fixture = new Fixture()) { + fixture.sendDataChange(1, uint(1)); + + assertTrue(fixture.awaitDelivered(1), "NotificationMessage 1 was never delivered"); + + fixture.sendDataChange(2, uint(1), uint(2)); + assertTrue(fixture.awaitDelivered(2), "NotificationMessage 2 was never delivered"); + + assertTrue( + fixture.awaitAcknowledged(1, RECOVERY_WINDOW_MILLIS), + "control: with no induced failure the acknowledgement for sequence 1 must reach the" + + " Server"); + } + } + } + + /** + * Part 4 §5.14.5.2 gives the PublishResponse a results array: "List of results for the + * acknowledgements. The size and order of the list matches the size and order of the + * subscriptionAcknowledgements request parameter." It is how a Server reports + * Bad_SequenceNumberUnknown or Bad_SubscriptionIdInvalid for an individual acknowledgement while + * the Publish call itself succeeds. + * + *

{@code PublishingManager} never reads it, so an acknowledgement the Server refused is + * indistinguishable from one it accepted. §5.14.5.2 also says of availableSequenceNumbers that + * "this information is for diagnostic purpose and Clients should log differences to the expected + * sequence numbers" — the same expectation applies to a refused acknowledgement, and today + * nothing at all is emitted. + */ + @Nested + class AcknowledgementResults { + + /** + * There is no listener, status, or other API surface on which a refused acknowledgement could + * be observed, so this asserts on what the client logs. It therefore requires the diagnostic to + * be logged at a level enabled by this module's {@code simplelogger.properties} (INFO or + * above); WARN is the appropriate level for a Server refusing a client's acknowledgement. + */ + @Test + void acknowledgementRefusedByTheServerIsReported() throws Exception { + try (var fixture = new Fixture()) { + fixture.sendDataChange(1, uint(1)); + + assertTrue(fixture.awaitDelivered(1), "NotificationMessage 1 was never delivered"); + assertTrue( + fixture.awaitAcknowledged(1, AWAIT_TIMEOUT_MILLIS), + "the client never acknowledged sequence 1, so there is no acknowledgement for the" + + " Server to refuse"); + + try (var stderr = new StderrCapture()) { + // Control for the capture itself: unless a WARN emitted through this JVM's SLF4J binding + // is visible here, the absence asserted below would prove nothing at all. + LoggerFactory.getLogger(PublishAcknowledgementTest.class).warn(CAPTURE_PROBE); + + assertTrue( + awaitTrue(() -> stderr.text().contains(CAPTURE_PROBE), RECOVERY_WINDOW_MILLIS), + "control: WARN records are not reaching the captured System.err, so this test cannot" + + " observe anything the client does or does not log"); + + fixture.refuseAcknowledgements(StatusCodes.Bad_SequenceNumberUnknown); + + assertTrue( + fixture.awaitDelivered(2), + "the PublishResponse refusing the acknowledgement was never processed"); + + assertTrue( + awaitTrue( + () -> stderr.text().contains("Bad_SequenceNumberUnknown"), + RECOVERY_WINDOW_MILLIS), + "the Server refused the acknowledgement for sequence 1 with" + + " Bad_SequenceNumberUnknown in the PublishResponse results array, and the" + + " client reported nothing: the results array is never read, so no" + + " acknowledgement failure is detectable by an application or an operator"); + } + } + } + } + + // region fixture + + /** + * A running Server whose Publish and Republish responses are scripted, a connected client, and a + * Subscription with one client-side MonitoredItem whose values are recorded as they are + * delivered. + * + *

Construction stops before any NotificationMessage has been sent, so a test can arrange the + * fate of the first acknowledgement before the notification that produces it. + */ + private static final class Fixture implements AutoCloseable { + + /** Every value handed to {@code onDataReceived}, in delivery order. */ + private final List deliveredValues = Collections.synchronizedList(new ArrayList<>()); + + private final OpcUaServer server; + private final OpcUaClient client; + private final LossyPublishServiceSet scriptable; + private final UInteger subscriptionId; + private final UInteger clientHandle; + + Fixture() throws Exception { + TestServer testServer = TestServer.create(); + server = testServer.getServer(); + + scriptable = new LossyPublishServiceSet(server); + for (EndpointConfig endpoint : server.getConfig().getEndpoints()) { + server.addServiceSet(endpoint.getPath(), scriptable); + } + + server.startup().get(); + + client = + TestClient.create( + server, + cfg -> + cfg + // Long request timeout so parked Publish requests do not time out. + .setRequestTimeout(uint(REQUEST_TIMEOUT_MILLIS)) + // No Session keep-alive traffic: the only requests in flight during a test + // are the ones it scripts. + .setKeepAliveInterval(uint(REQUEST_TIMEOUT_MILLIS))); + client.connect(); + + var subscription = new OpcUaSubscription(client); + subscription.setSubscriptionListener( + new OpcUaSubscription.SubscriptionListener() { + @Override + public void onDataReceived( + OpcUaSubscription subscription, + List items, + List values) { + + values.forEach(value -> deliveredValues.add((Integer) value.getValue().getValue())); + } + }); + subscription.create(); + + subscriptionId = subscription.getSubscriptionId().orElseThrow(); + + // Client-side only: addMonitoredItem() assigns the ClientHandle the notification fan-out + // looks values up by, which is all a scripted notification needs. + var item = OpcUaMonitoredItem.newDataItem(NodeIds.Server_ServerStatus_CurrentTime); + subscription.addMonitoredItem(item); + clientHandle = item.getClientHandle().orElseThrow(); + + assertTrue( + awaitTrue( + () -> scriptable.getParkedRequestCount() >= PIPELINE_DEPTH, AWAIT_TIMEOUT_MILLIS), + "the client did not fill its Publish pipeline"); + } + + /** + * Answer a parked Publish request with a data-change NotificationMessage carrying {@code + * sequenceNumber} as its sequence number and as the value of its MonitoredItem. + */ + void sendDataChange(long sequenceNumber, UInteger... available) { + scriptable.enqueueDataChange( + subscriptionId, sequenceNumber, notifications((int) sequenceNumber), available); + } + + /** + * Arrange for the next PublishRequest carrying an acknowledgement of {@code sequenceNumber} to + * fail before the Server records or answers it. + */ + void loseNextPublishRequestCarryingAcknowledgement(long sequenceNumber) { + scriptable.loseNextRequestCarrying(subscriptionId, uint(sequenceNumber)); + } + + boolean awaitAcknowledgementLost() throws Exception { + return scriptable.awaitRequestLost(AWAIT_TIMEOUT_MILLIS); + } + + /** + * Answer every PublishRequest still parked, refusing each acknowledgement it carries with + * {@code statusCode} in the positionally matching entry of the response's results array. + * + *

The request carrying an acknowledgement is answered with a data-change NotificationMessage + * so that its delivery is an observable barrier proving the response was processed; the others + * get a keep-alive that leaves the sequence accounting where it was. + */ + void refuseAcknowledgements(long statusCode) { + for (int i = 0; i < PIPELINE_DEPTH; i++) { + scriptable.enqueue(request -> refusingResponse(request, statusCode)); + } + } + + private CompletableFuture refusingResponse( + PublishRequest request, long statusCode) { + + SubscriptionAcknowledgement[] acks = request.getSubscriptionAcknowledgements(); + int ackCount = acks != null ? acks.length : 0; + + var results = new StatusCode[ackCount]; + Arrays.fill(results, StatusCode.of(statusCode)); + + // Part 4 §5.14.1.1: a keep-alive carries "the sequence number of the next NotificationMessage + // that is to be sent", so a keep-alive numbered 2 accounts for nothing beyond sequence 1. + ExtensionObject[] notificationData = ackCount > 0 ? encodedNotifications(2) : null; + UInteger[] available = ackCount > 0 ? new UInteger[] {uint(2)} : new UInteger[0]; + + return CompletableFuture.completedFuture( + scriptable.buildPublishResponse( + request, subscriptionId, 2, notificationData, available, false, results)); + } + + boolean awaitDelivered(int value) throws Exception { + return awaitTrue(() -> deliveredValues.contains(value), AWAIT_TIMEOUT_MILLIS); + } + + boolean awaitAcknowledged(long sequenceNumber, long timeoutMillis) throws Exception { + return awaitTrue( + () -> + scriptable.getReceivedAcknowledgements().stream() + .anyMatch( + ack -> + subscriptionId.equals(ack.getSubscriptionId()) + && ack.getSequenceNumber().longValue() == sequenceNumber), + timeoutMillis); + } + + /** A DataChangeNotification carrying {@code value} for this Fixture's MonitoredItem. */ + private List notifications(int value) { + return List.of( + new MonitoredItemNotification(clientHandle, new DataValue(Variant.ofInt32(value)))); + } + + private ExtensionObject[] encodedNotifications(int value) { + return new ExtensionObject[] { + scriptable.encode( + new DataChangeNotification( + notifications(value).toArray(MonitoredItemNotification[]::new), null)) + }; + } + + @Override + public void close() throws Exception { + scriptable.failParkedRequests(StatusCodes.Bad_NoSubscription); + try { + client.disconnectAsync().get(5, TimeUnit.SECONDS); + } finally { + server.shutdown().get(10, TimeUnit.SECONDS); + } + } + } + + /** + * A {@link ScriptableSubscriptionServiceSet} that can drop one PublishRequest on the floor. + * + *

The armed request is failed before {@code super.onPublish} records its acknowledgements, so + * from the Server's point of view the request — and everything it was carrying — never arrived. + * That is what makes {@code getReceivedAcknowledgements()} a sound record of what the client + * actually managed to acknowledge. + */ + private static final class LossyPublishServiceSet extends ScriptableSubscriptionServiceSet { + + private final AtomicReference target = new AtomicReference<>(); + private final CountDownLatch lost = new CountDownLatch(1); + + LossyPublishServiceSet(OpcUaServer server) { + super(server); + } + + void loseNextRequestCarrying(UInteger subscriptionId, UInteger sequenceNumber) { + target.set(new SubscriptionAcknowledgement(subscriptionId, sequenceNumber)); + } + + boolean awaitRequestLost(long timeoutMillis) throws InterruptedException { + return lost.await(timeoutMillis, TimeUnit.MILLISECONDS); + } + + @Override + public CompletableFuture onPublish( + ServiceRequestContext context, PublishRequest request) { + + SubscriptionAcknowledgement armed = target.get(); + + if (armed != null && carries(request, armed) && target.compareAndSet(armed, null)) { + lost.countDown(); + + return CompletableFuture.failedFuture(new UaException(StatusCodes.Bad_Timeout)); + } + + return super.onPublish(context, request); + } + + private static boolean carries(PublishRequest request, SubscriptionAcknowledgement armed) { + SubscriptionAcknowledgement[] acks = request.getSubscriptionAcknowledgements(); + + if (acks == null) { + return false; + } + + return Arrays.stream(acks) + .anyMatch( + ack -> + armed.getSubscriptionId().equals(ack.getSubscriptionId()) + && armed.getSequenceNumber().equals(ack.getSequenceNumber())); + } + } + + /** + * Captures everything written to {@link System#err} while installed, passing it through to the + * original stream so the output a failing test needs is not swallowed. + * + *

slf4j-simple, the SLF4J binding on this module's test classpath, is configured with its + * default uncached {@code System.err} target and resolves {@code System.err} on every write, so + * installing this stream is enough to observe log records emitted afterwards. + */ + private static final class StderrCapture implements AutoCloseable { + + private final ByteArrayOutputStream captured = new ByteArrayOutputStream(); + private final PrintStream original = System.err; + private final PrintStream installed; + + StderrCapture() { + installed = new PrintStream(new Tee(), true, StandardCharsets.UTF_8); + + System.setErr(installed); + } + + String text() { + synchronized (captured) { + return captured.toString(StandardCharsets.UTF_8); + } + } + + @Override + public void close() { + System.setErr(original); + installed.flush(); + } + + private final class Tee extends OutputStream { + + @Override + public void write(int b) { + synchronized (captured) { + captured.write(b); + } + original.write(b); + } + + @Override + public void write(byte[] b, int off, int len) { + synchronized (captured) { + captured.write(b, off, len); + } + original.write(b, off, len); + } + } + } + + /** Polls {@code condition} until it holds or {@code timeoutMillis} elapses. */ + private static boolean awaitTrue(ThrowingBooleanSupplier condition, long timeoutMillis) + throws Exception { + + long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(timeoutMillis); + + while (System.nanoTime() < deadline) { + if (condition.get()) { + return true; + } + Thread.sleep(25); + } + + return condition.get(); + } + + @FunctionalInterface + private interface ThrowingBooleanSupplier { + boolean get() throws Exception; + } + + // endregion +} diff --git a/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/PublishCeilingRecoveryTest.java b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/PublishCeilingRecoveryTest.java new file mode 100644 index 0000000000..ae4f2f8308 --- /dev/null +++ b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/PublishCeilingRecoveryTest.java @@ -0,0 +1,823 @@ +/* + * Copyright (c) 2026 the Eclipse Milo Authors + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.milo.opcua.sdk.client.subscriptions; + +import static org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.Unsigned.uint; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.ReentrantLock; +import org.eclipse.milo.opcua.sdk.client.OpcUaClient; +import org.eclipse.milo.opcua.sdk.server.EndpointConfig; +import org.eclipse.milo.opcua.sdk.server.OpcUaServer; +import org.eclipse.milo.opcua.sdk.test.ScriptableSubscriptionServiceSet; +import org.eclipse.milo.opcua.sdk.test.TestClient; +import org.eclipse.milo.opcua.sdk.test.TestServer; +import org.eclipse.milo.opcua.stack.core.StatusCodes; +import org.eclipse.milo.opcua.stack.core.UaException; +import org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.UInteger; +import org.eclipse.milo.opcua.stack.core.types.structured.PublishRequest; +import org.eclipse.milo.opcua.stack.core.types.structured.PublishResponse; +import org.eclipse.milo.opcua.stack.transport.server.ServiceRequestContext; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +/** + * Recovery of the Publish pipeline ceiling {@code PublishingManager} learns from + * Bad_TooManyPublishRequests. + * + *

Part 4 §5.14.5.1: once a Server has answered that error a Client "shall not issue another + * Publish request before one of its outstanding Publish requests is returned", and the client also + * remembers how many the Server was in fact holding, so that the next returning request does not + * restore the very outstanding count that drew the fault. The clamp is right. Keeping it forever is + * not: a momentary server-side condition — a queue briefly full, a cap raised again a second later + * — costs a long-lived client a permanently shallower pipeline, because the only two events that + * clear the clamp today are a Subscription being added and a Session being activated, and neither + * need ever happen again for the rest of that client's life. + * + *

What is required instead, and what these tests pin: + * + *

    + *
  1. sustained successful Publish activity lets the ceiling recover to the natural target, + * {@code min(subscriptionCount + 1, maxPendingPublishRequests)}; + *
  2. recovery is incremental — one request at a time, never straight back to the target; + *
  3. a probe costs a cooldown counted in successful Publish round trips, not in elapsed time, so + * that it is the Server's own answers which pay for the next attempt; + *
  4. a probe that draws Bad_TooManyPublishRequests again re-clamps the ceiling and + * lengthens the next cooldown geometrically, so that a Server which always refuses sees a + * number of probes growing only logarithmically in the number of responses it delivers; + *
  5. at most one probe is in flight at a time, and the natural target is never exceeded. + *
+ * + *

The bounds asserted below deliberately leave the implementation room to choose its constants, + * but they do constrain them: the first probe must come no earlier than {@link + * #QUIET_PREFIX_DELIVERIES} + 1 successful round trips (so that the immediate one-replacement-per- + * return behaviour pinned by {@link PublishPipelineRefillTest}, whose window is a single returning + * request, is untouched), no later than a few tens of them, and against a Server that always + * refuses the schedule must fit within {@link #MAX_PROBES} probes over {@link + * #PROBE_WINDOW_DELIVERIES} delivered NotificationMessages, with the interval between probes never + * shrinking. A base cooldown of 8 successful round trips, doubling on each refused probe and capped + * in the hundreds, satisfies all of them with room to spare. + * + *

Every test drives a Server whose Publish requests are parked by {@link + * ScriptableSubscriptionServiceSet} — nothing completes one unless the test says so — and answers + * them one at a time, so the number of Publish requests the Server is left holding after each + * answer is the client's current ceiling, observed rather than inferred, and every wait is + * on a Publish request arriving rather than on time passing. + * + * @see PublishPipelineRefillTest for the immediate response to the fault, which this must not + * change. + */ +public class PublishCeilingRecoveryTest { + + /** + * The natural target of the one-Subscription fixtures: {@code min(subscriptionCount + 1, + * maxPendingPublishRequests)} = min(1 + 1, 2). maxPendingPublishRequests is configured to the + * same value rather than left at its default so that the target is stated by the test. + */ + private static final int SINGLE_SUBSCRIPTION_TARGET = 2; + + /** Three Subscriptions, and a natural target of min(3 + 1, 4) = 4. */ + private static final int DEEP_PIPELINE_SUBSCRIPTIONS = 3; + + private static final int DEEP_PIPELINE_TARGET = 4; + + /** + * The ceiling one rejection leaves behind when the whole one-Subscription pipeline was + * outstanding: {@code max(1, outstanding)} with the refused request accounted for, i.e. max(1, 2 + * - 1). + */ + private static final int SINGLE_SUBSCRIPTION_CLAMP = 1; + + /** + * The ceiling two rejections leave behind when the whole three-Subscription pipeline was + * outstanding: the first is accounted for with three still outstanding, the second with two, and + * the ceiling is the smallest of them. + */ + private static final int DEEP_PIPELINE_CLAMP = 2; + + /** + * The number of Publish requests the capped Server will hold for a Session before answering + * Bad_TooManyPublishRequests. One is below every natural target used here, so the Server refuses + * whenever the client tries to deepen its pipeline at all. + */ + private static final int SERVER_PUBLISH_CAP = 1; + + /** + * The number of successful Publish round trips a recovery run is given to reach the natural + * target. Three raises at a doubling cooldown starting from 8 cost 8 + 16 + 32 = 56, so this + * leaves the implementation an order of magnitude of slack while still failing rather than + * hanging if the ceiling never moves. + */ + private static final int MAX_RECOVERY_DELIVERIES = 256; + + /** + * Successful Publish round trips the anti-hammering run delivers. Long enough that a cooldown + * which doubles from a base of a few produces several probes, so that the intervals between them + * can be compared and not merely counted. + */ + private static final int PROBE_WINDOW_DELIVERIES = 256; + + /** + * How long, after a Publish request has arrived to replace the one just answered, to keep waiting + * for another before answering the next. + * + *

Not a synchronization mechanism: the round trip itself is waited for, and this is only the + * window in which a second request — the probe that accompanies a replacement — is allowed + * to land. The client sends both from one loop, so the gap between them is tens of microseconds; + * this is three orders of magnitude more than that, and is what makes a probe meet a Server whose + * queue is still occupied rather than one that has just been emptied. + */ + private static final long BURST_SETTLE_MILLIS = 5; + + /** + * The most refused probes a Server that always answers Bad_TooManyPublishRequests may see over + * {@link #PROBE_WINDOW_DELIVERIES} delivered NotificationMessages. + * + *

This is the anti-hammering guarantee. Without a cooldown the client probes once per + * returning request, which is one refusal per delivered NotificationMessage — 256 of them. With a + * cooldown that does not grow it probes 256/cooldown times, still linear in the traffic. With a + * cooldown that doubles on each refusal it probes log2(256/base) times: at most 6 for any base of + * 4 or more and any cooldown cap of 64 or more. Ten leaves room for an implementation that counts + * its successes a little differently without leaving room for one that hammers. + */ + private static final int MAX_PROBES = 10; + + /** + * The fewest refused probes that same run must see. + * + *

One would be satisfied by a client that tried once and gave up, which is the clamp again + * under another name; the guarantee is that it keeps asking, only ever more rarely. Two also + * keeps the test honest about the Server: every probe against this Server is refused, so a run + * that recorded one refusal would mean a probe had slipped past the cap and the ceiling had been + * allowed to stand. + */ + private static final int MIN_PROBES = 2; + + /** + * Successful Publish round trips over which, immediately after the clamp, one returning request + * must still buy exactly one replacement and nothing more. + * + *

{@link PublishPipelineRefillTest} pins that for the first of them; this pins that the + * cooldown before the first probe is longer than a handful, so that the two cannot be confused + * for one another and a probe cannot appear inside that test's window. + */ + private static final int QUIET_PREFIX_DELIVERIES = 4; + + /** + * Keep-alives all carry sequence number 1: Part 4 §5.14.1.1 makes a keep-alive's sequence number + * "the sequence number of the next NotificationMessage that is to be sent", so repeating it says + * the Server has still sent nothing, leaves the client's sequence accounting exactly where it + * started, and produces neither an acknowledgement nor a Republish. Every one of them is + * nevertheless a successful Publish round trip: a request answered, a NotificationMessage + * delivered, a permit released and a replacement request sent. + */ + private static final long KEEP_ALIVE_SEQUENCE_NUMBER = 1L; + + private static final long AWAIT_TIMEOUT_MILLIS = 10_000; + + /** How long to watch for a Publish request that must not be sent. */ + private static final long QUIET_WINDOW_MILLIS = 1_000; + + /** + * Long enough that nothing times out on its own: the PublishRequests these tests park at the + * Server stay parked until the test decides how they end. + */ + private static final long REQUEST_TIMEOUT_MILLIS = 60_000; + + /** + * A transient rejection — the Server refuses while its queue is briefly full, then stops refusing + * — must not cost the client its pipeline depth for good. + */ + @Nested + class RecoveryAfterATransientRejection { + + /** + * Control: the same three-Subscription fixture with a Server that never refuses. It proves the + * fixture can observe a four-deep pipeline at all, so {@link + * #theCeilingClimbsBackOneRequestAtATimeRatherThanInOneJump} fails because the ceiling did not + * recover and not because four outstanding requests are unobservable here. + */ + @Test + void theWholePipelineIsOutstandingWhenTheServerNeverRefuses() throws Exception { + try (Fixture fixture = Fixture.withThreeSubscriptions()) { + assertTrue( + fixture.scriptable.awaitParked(DEEP_PIPELINE_TARGET, AWAIT_TIMEOUT_MILLIS), + "the Publish pipeline never filled to its natural target"); + } + } + + /** + * The property the clamp costs today: a client that took one Bad_TooManyPublishRequests keeps a + * pipeline one request shallower than the Server would now accept, for as long as it lives. + */ + @Test + void theCeilingRecoversToTheNaturalTargetOnceTheServerStopsRefusing() throws Exception { + try (Fixture fixture = Fixture.withOneSubscription(SERVER_PUBLISH_CAP)) { + // Filling a two-deep pipeline against a Server that holds one costs exactly one rejection: + // the first request is held, the second refused, and the refused one is not replaced. + fixture.assertPipelineSettlesAt(SINGLE_SUBSCRIPTION_CLAMP); + assertEquals( + 1, + fixture.scriptable.getRejectionCount(), + "the clamp under test is the one a single Bad_TooManyPublishRequests installs"); + + // The transient condition is over; from here the Server accepts everything the client cares + // to send. + fixture.scriptable.setCap(CappingSubscriptionServiceSet.UNCAPPED); + + List steps = + fixture.deliverUntilPipelineReaches( + SINGLE_SUBSCRIPTION_CLAMP, SINGLE_SUBSCRIPTION_TARGET, MAX_RECOVERY_DELIVERIES); + + assertEquals( + SINGLE_SUBSCRIPTION_TARGET, + steps.isEmpty() ? SINGLE_SUBSCRIPTION_CLAMP : steps.get(steps.size() - 1), + "the ceiling learned from one Bad_TooManyPublishRequests never recovered: after " + + MAX_RECOVERY_DELIVERIES + + " successful Publish round trips against a Server that refuses nothing, the" + + " client still keeps a shallower pipeline than min(subscriptionCount + 1," + + " maxPendingPublishRequests). Nothing in the traffic itself can lift the clamp," + + " so it lasts until a Subscription is added or a Session is activated — for a" + + " long-lived client, forever"); + } + } + + /** + * Recovery must be a probe, not a jump. Restoring the whole deficit at once would send the + * Server exactly the burst that drew the fault, which is what the clamp exists to prevent; + * raising the ceiling by one asks the question with a single request. + * + *

Two rejections against a four-deep pipeline leave a ceiling of two, so an incremental + * recovery is observable as the pipeline going three deep before it goes four deep. + */ + @Test + void theCeilingClimbsBackOneRequestAtATimeRatherThanInOneJump() throws Exception { + try (Fixture fixture = Fixture.withThreeSubscriptions()) { + assertTrue( + fixture.scriptable.awaitParked(DEEP_PIPELINE_TARGET, AWAIT_TIMEOUT_MILLIS), + "the Publish pipeline never filled to its natural target"); + + // Two of the four outstanding requests are refused. Their failures are accounted for + // independently — each decrements the outstanding count once — so the ceiling ends up at + // the smaller of the two counts they saw, whichever order they are handled in. + fixture.scriptable.enqueueServiceFault(StatusCodes.Bad_TooManyPublishRequests); + fixture.scriptable.enqueueServiceFault(StatusCodes.Bad_TooManyPublishRequests); + + fixture.assertPipelineSettlesAt(DEEP_PIPELINE_CLAMP); + + List steps = + fixture.deliverUntilPipelineReaches( + DEEP_PIPELINE_CLAMP, DEEP_PIPELINE_TARGET, MAX_RECOVERY_DELIVERIES); + + assertEquals( + List.of(DEEP_PIPELINE_CLAMP + 1, DEEP_PIPELINE_TARGET), + steps, + "the ceiling did not climb back one request at a time: the numbers of Publish requests" + + " the Server was left holding, in the order they were first observed, were " + + steps + + ", starting from a clamp of " + + DEEP_PIPELINE_CLAMP + + " and aiming at a natural target of " + + DEEP_PIPELINE_TARGET); + } + } + } + + /** + * The other half of leniency, and the one that decides whether it is safe: a Server that keeps + * refusing must not be asked over and over. + */ + @Nested + class AntiHammering { + + /** + * A Server whose cap really is below what the client wants answers every probe with + * Bad_TooManyPublishRequests. The client must keep asking — otherwise a transient condition is + * indistinguishable from a permanent one and the clamp is permanent again — but the asking must + * cost the Server almost nothing: with a cooldown that doubles on each refusal, the number of + * probes grows logarithmically in the number of responses the Server delivers, so doubling the + * traffic adds a probe or two rather than doubling the probes. + */ + @Test + void aServerThatAlwaysRefusesSeesOnlyABoundedAndSlowlyGrowingNumberOfProbes() throws Exception { + try (Fixture fixture = Fixture.withOneSubscription(SERVER_PUBLISH_CAP)) { + fixture.assertPipelineSettlesAt(SINGLE_SUBSCRIPTION_CLAMP); + + assertEquals( + 1, + fixture.scriptable.getRejectionCount(), + "filling the pipeline against the capped Server cost one refusal"); + + List refusedAt = fixture.deliverAndRecordRefusals(PROBE_WINDOW_DELIVERIES); + + assertTrue( + refusedAt.size() >= MIN_PROBES, + "the client stopped probing the ceiling: over " + + PROBE_WINDOW_DELIVERIES + + " successful Publish round trips the Server was asked only " + + refusedAt.size() + + " time(s) whether it would now queue another PublishRequest, so a condition that" + + " had long since passed would go on costing the client pipeline depth for as long" + + " as it lived. The round trips the refusals were observed at were " + + refusedAt); + + assertTrue( + refusedAt.size() <= MAX_PROBES, + "the client hammered the Server: " + + refusedAt.size() + + " refused probes over " + + PROBE_WINDOW_DELIVERIES + + " delivered NotificationMessages, more than the " + + MAX_PROBES + + " a cooldown that grows geometrically on each refusal allows. A Server that" + + " always refuses must see a number of probes that grows only logarithmically in" + + " the number of responses it delivers. The round trips they were observed at were" + + " " + + refusedAt); + + // The intervals between probes, the first measured from the clamp. Geometric backoff makes + // them grow, and the assertion is that the window ends with a longer wait than it began + // with. Strict step-by-step monotonicity is deliberately not asserted: a refusal and the + // delivery it is attributed to are two independently timed events, so which round trip a + // refusal is recorded against can jitter by one either way, and an inversion between + // adjacent intervals says nothing about the cooldown. The bound on the probe count above is + // what pins the guarantee; this pins its direction. + List intervals = intervals(refusedAt); + + if (intervals.size() >= 2) { + assertTrue( + intervals.get(intervals.size() - 1) > intervals.get(0), + "the cooldown did not lengthen across the window: the intervals between probes, in" + + " successful Publish round trips, were " + + intervals + + ". Each refusal must make the next wait longer, or a Server that always refuses" + + " is asked at a constant rate forever"); + } + } + } + } + + /** + * The immediate response to the fault, which recovery must leave exactly as it was. + * + *

Part 4 §5.14.5.1 buys one replacement per returning request, and {@link + * PublishPipelineRefillTest} pins that for the first request to return after the fault. A probe + * is a later and separate event; a cooldown short enough to fire inside that window would turn + * this change into a regression of that one. + */ + @Nested + class ImmediateResponseToTheFault { + + /** + * Passes both before and after recovery is implemented — deliberately. It is the guard that + * keeps the cooldown clear of the window {@link PublishPipelineRefillTest} measures. + */ + @Test + void theFirstFewReturningRequestsStillBuyExactlyOneReplacementEach() throws Exception { + try (Fixture fixture = Fixture.withOneSubscription(SERVER_PUBLISH_CAP)) { + fixture.assertPipelineSettlesAt(SINGLE_SUBSCRIPTION_CLAMP); + + // Nothing the Server does from here can refuse a request, so an extra request going out is + // the client's own decision and is visible as one arriving at the Server. + fixture.scriptable.setCap(CappingSubscriptionServiceSet.UNCAPPED); + + int arrivalsBefore = fixture.scriptable.getArrivalCount(); + + int highWater = fixture.deliverKeepAlives(QUIET_PREFIX_DELIVERIES); + + // Let anything already in flight arrive before the count is compared, so that an extra + // request cannot be missed by having been sent a moment too late. + assertFalse( + fixture.scriptable.awaitArrivals( + arrivalsBefore + QUIET_PREFIX_DELIVERIES + 1, QUIET_WINDOW_MILLIS), + "more Publish requests were sent than the number of requests that returned"); + + assertEquals( + QUIET_PREFIX_DELIVERIES, + fixture.scriptable.getArrivalCount() - arrivalsBefore, + "one returning request bought more than one replacement within the first " + + QUIET_PREFIX_DELIVERIES + + " successful Publish round trips after the clamp, which is the window Part 4" + + " §5.14.5.1's one-replacement-per-return rule is measured in"); + + assertEquals( + SINGLE_SUBSCRIPTION_CLAMP, + highWater, + "the clamp was lifted within the first " + + QUIET_PREFIX_DELIVERIES + + " successful Publish round trips after the fault; the cooldown before a probe" + + " must be longer than that"); + } + } + } + + // region helpers + + /** + * @param points the round trip counts at which something was observed, in order. + * @return the intervals between them, the first measured from the start of the run. + */ + private static List intervals(List points) { + var intervals = new ArrayList(points.size()); + int previous = 0; + + for (int point : points) { + intervals.add(point - previous); + previous = point; + } + + return intervals; + } + + /** + * A {@link ScriptableSubscriptionServiceSet} that also models a Server which will not queue more + * than a fixed number of PublishRequests for a Session: one that would take its queue past the + * cap is answered Bad_TooManyPublishRequests, which is what Part 4 §5.14.5.1 has a Server do when + * a Client has more requests queued than it supports. The cap is settable, so a test can model a + * condition that passes. + * + *

The queue is the set of requests the script is holding, which is every request that has + * arrived and not been answered — this harness answers nothing on its own. A Server queues a + * PublishRequest from the moment it arrives until it has something to send in answer, so a test + * that answers one request at a time and waits for the client's reply to land before answering + * the next holds the queue exactly as a Server with data to send at a steady rate would. + */ + private static final class CappingSubscriptionServiceSet + extends ScriptableSubscriptionServiceSet { + + /** A Server that refuses nothing. */ + static final int UNCAPPED = Integer.MAX_VALUE; + + /** + * Guards the counters below and, held across the decision, makes "measure the queue, then join + * it" atomic: two requests arriving together must not both find room in a queue with one place + * left. + */ + private final ReentrantLock lock = new ReentrantLock(); + + /** Signalled whenever a Publish request arrives, refused or not. */ + private final Condition arrived = lock.newCondition(); + + /** Publish requests received, refused ones included. Guarded by {@link #lock}. */ + private int arrivals = 0; + + /** Publish requests refused for exceeding the cap. Guarded by {@link #lock}. */ + private int rejections = 0; + + private volatile int cap; + + CappingSubscriptionServiceSet(OpcUaServer server, int cap) { + super(server); + + this.cap = cap; + } + + void setCap(int cap) { + this.cap = cap; + } + + int getArrivalCount() { + lock.lock(); + try { + return arrivals; + } finally { + lock.unlock(); + } + } + + int getRejectionCount() { + lock.lock(); + try { + return rejections; + } finally { + lock.unlock(); + } + } + + /** Wait until at least {@code count} Publish requests have arrived, refused ones included. */ + boolean awaitArrivals(int count, long timeoutMillis) throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(timeoutMillis); + + lock.lock(); + try { + while (arrivals < count) { + long remaining = deadline - System.nanoTime(); + if (remaining <= 0) { + return false; + } + arrived.awaitNanos(remaining); + } + + return true; + } finally { + lock.unlock(); + } + } + + /** + * Wait until at least {@code count} Publish requests are parked, i.e. until the client has that + * many outstanding for the script to answer. + */ + boolean awaitParked(int count, long timeoutMillis) throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(timeoutMillis); + + lock.lock(); + try { + while (getParkedRequestCount() < count) { + long remaining = deadline - System.nanoTime(); + if (remaining <= 0) { + return false; + } + arrived.awaitNanos(remaining); + } + + return true; + } finally { + lock.unlock(); + } + } + + @Override + public CompletableFuture onPublish( + ServiceRequestContext context, PublishRequest request) { + + lock.lock(); + try { + arrivals++; + arrived.signalAll(); + + if (getParkedRequestCount() >= cap) { + rejections++; + + return CompletableFuture.failedFuture( + new UaException(StatusCodes.Bad_TooManyPublishRequests)); + } + + return super.onPublish(context, request); + } finally { + lock.unlock(); + } + } + } + + /** + * A running Server whose Publish requests are all parked by {@link CappingSubscriptionServiceSet} + * — nothing completes one unless the test says so, and nothing is refused unless its cap says so + * — plus a connected client with its Subscriptions already created. + */ + private static final class Fixture implements AutoCloseable { + + private final OpcUaServer server; + private final OpcUaClient client; + private final CappingSubscriptionServiceSet scriptable; + + private final List subscriptions = new ArrayList<>(); + + /** min(subscriptionCount + 1, maxPendingPublishRequests) for this fixture's client. */ + private final int naturalTarget; + + static Fixture withOneSubscription(int cap) throws Exception { + return new Fixture(1, SINGLE_SUBSCRIPTION_TARGET, cap); + } + + /** + * A fixture whose four-deep pipeline fills against a Server that refuses nothing, so the tests + * that need a deep clamp can install one themselves, request by request. + */ + static Fixture withThreeSubscriptions() throws Exception { + return new Fixture( + DEEP_PIPELINE_SUBSCRIPTIONS, + DEEP_PIPELINE_TARGET, + CappingSubscriptionServiceSet.UNCAPPED); + } + + private Fixture(int subscriptionCount, int naturalTarget, int cap) throws Exception { + this.naturalTarget = naturalTarget; + + TestServer testServer = TestServer.create(); + server = testServer.getServer(); + + scriptable = new CappingSubscriptionServiceSet(server, cap); + for (EndpointConfig endpoint : server.getConfig().getEndpoints()) { + server.addServiceSet(endpoint.getPath(), scriptable); + } + + server.startup().get(); + + client = + TestClient.create( + server, + cfg -> + cfg + // Long request timeout so parked Publish requests do not time out. + .setRequestTimeout(uint(REQUEST_TIMEOUT_MILLIS)) + // No Session keep-alive traffic: the only requests in flight during a + // test are the ones it scripts. + .setKeepAliveInterval(uint(REQUEST_TIMEOUT_MILLIS)) + // States the natural target in the test rather than inheriting the config + // default: min(subscriptionCount + 1, naturalTarget) == naturalTarget. + .setMaxPendingPublishRequests(uint(naturalTarget))); + + client.connect(); + + for (int i = 0; i < subscriptionCount; i++) { + var subscription = new OpcUaSubscription(client); + subscription.create(); + + subscriptions.add(subscription); + } + } + + /** + * The Subscription every keep-alive is addressed to. The others exist only to raise the natural + * target; a Subscription that receives nothing has nothing to say about the pipeline's depth, + * and its watchdog timer merely notifies a listener it has none of. + */ + UInteger notifiedSubscriptionId() { + return subscriptions.get(0).getSubscriptionId().orElseThrow(); + } + + /** + * Assert the pipeline has settled at {@code expected} outstanding Publish requests: the traffic + * has stopped, and what the Server is left holding when it does is the ceiling the client is + * working to. + * + *

Quiescence is waited for rather than merely sampled, because the client's opening burst is + * one request per unit of its natural target and the tail of that burst is traffic which has + * not stopped yet rather than traffic which never will. + */ + void assertPipelineSettlesAt(int expected) throws Exception { + assertTrue( + scriptable.awaitArrivals(naturalTarget, AWAIT_TIMEOUT_MILLIS), + "the client never sent the Publish requests its natural target calls for"); + + long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(AWAIT_TIMEOUT_MILLIS); + + int arrivals; + do { + arrivals = scriptable.getArrivalCount(); + } while (scriptable.awaitArrivals(arrivals + 1, QUIET_WINDOW_MILLIS) + && System.nanoTime() < deadline); + + assertEquals( + arrivals, + scriptable.getArrivalCount(), + "the Publish pipeline never settled: the client kept sending requests without being" + + " answered. Recovery must be driven by successful Publish round trips, not by time" + + " passing"); + + assertEquals( + expected, + scriptable.getParkedRequestCount(), + "the Publish pipeline settled at a depth the test did not set up"); + } + + /** + * Answer one parked Publish request with a keep-alive and wait until the client has dealt with + * it, i.e. until every Publish request it sends in reply has arrived. + * + *

A request is answered only once one is parked, never by leaving a responder ready for a + * request that has not arrived yet: a responder waiting in the script would answer the next + * arrival immediately, and the Server's queue would never hold the two requests a probe + * consists of at the same time. + * + *

The same reason is why the reply is waited out rather than merely waited for. A client + * probing its ceiling sends two requests from one loop, microseconds apart; answering the first + * before the second arrives would empty the queue in between and let the probe through a cap it + * should have hit. {@link #BURST_SETTLE_MILLIS} is orders of magnitude more than that gap, and + * is spent only on the reply that never comes — the second request of a burst that was not a + * probe. + */ + private void deliverOneKeepAlive() throws Exception { + assertTrue( + scriptable.awaitParked(1, AWAIT_TIMEOUT_MILLIS), + "the client has no Publish request outstanding: the pipeline has stalled"); + + int arrivals = scriptable.getArrivalCount(); + + scriptable.enqueueKeepAlive(notifiedSubscriptionId(), KEEP_ALIVE_SEQUENCE_NUMBER); + + assertTrue( + scriptable.awaitArrivals(arrivals + 1, AWAIT_TIMEOUT_MILLIS), + "the answered Publish request was never replaced: the pipeline has stalled"); + + scriptable.awaitArrivals(arrivals + 2, BURST_SETTLE_MILLIS); + } + + /** + * Deliver {@code count} keep-alives, one at a time. + * + * @return the greatest number of Publish requests the Server was left holding after any of + * them, i.e. the highest ceiling the client reached. + */ + int deliverKeepAlives(int count) throws Exception { + int highWater = 0; + + for (int i = 0; i < count; i++) { + deliverOneKeepAlive(); + + highWater = Math.max(highWater, sampleOutstanding()); + } + + return highWater; + } + + /** + * Deliver {@code count} keep-alives, one at a time, watching for the Server refusing one. + * + * @return the number of successful Publish round trips that had been delivered when each + * refused request was observed, in order. Each entry is a probe: an extra PublishRequest + * the client sent to find out whether the Server would now queue one more. + */ + List deliverAndRecordRefusals(int count) throws Exception { + var refusedAt = new ArrayList(); + int rejections = scriptable.getRejectionCount(); + + for (int delivered = 1; delivered <= count; delivered++) { + deliverOneKeepAlive(); + + int total = scriptable.getRejectionCount(); + + for (int i = rejections; i < total; i++) { + refusedAt.add(delivered); + } + + rejections = total; + } + + return refusedAt; + } + + /** + * Deliver keep-alives, one at a time, until the Server is left holding {@code target} Publish + * requests or {@code maxDeliveries} have been delivered. + * + * @return every increase in the number of Publish requests the Server was left holding, in the + * order they were first observed: the path the ceiling took from {@code clamp} upwards. + */ + List deliverUntilPipelineReaches(int clamp, int target, int maxDeliveries) + throws Exception { + + var steps = new ArrayList(); + int outstanding = clamp; + + for (int i = 0; i < maxDeliveries && outstanding < target; i++) { + deliverOneKeepAlive(); + + int sampled = sampleOutstanding(); + + if (sampled > outstanding) { + outstanding = sampled; + steps.add(sampled); + } + } + + return steps; + } + + /** + * @return the number of Publish requests the Server is holding, which between two answers is + * the number the client is keeping outstanding. + */ + private int sampleOutstanding() { + int outstanding = scriptable.getParkedRequestCount(); + + assertTrue( + outstanding <= naturalTarget, + "the client kept " + + outstanding + + " Publish requests outstanding, more than the natural target of " + + naturalTarget); + + return outstanding; + } + + @Override + public void close() throws Exception { + scriptable.setCap(CappingSubscriptionServiceSet.UNCAPPED); + scriptable.failParkedRequests(StatusCodes.Bad_NoSubscription); + try { + client.disconnectAsync().get(5, TimeUnit.SECONDS); + } finally { + server.shutdown().get(10, TimeUnit.SECONDS); + } + } + } + + // endregion +} diff --git a/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/PublishFirstActivationRecoveryTest.java b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/PublishFirstActivationRecoveryTest.java new file mode 100644 index 0000000000..0281c80eed --- /dev/null +++ b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/PublishFirstActivationRecoveryTest.java @@ -0,0 +1,219 @@ +/* + * Copyright (c) 2026 the Eclipse Milo Authors + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.milo.opcua.sdk.client.subscriptions; + +import static org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.Unsigned.uint; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import org.eclipse.milo.opcua.sdk.client.OpcUaClient; +import org.eclipse.milo.opcua.sdk.client.SessionActivityListener; +import org.eclipse.milo.opcua.sdk.client.UaSession; +import org.eclipse.milo.opcua.sdk.server.EndpointConfig; +import org.eclipse.milo.opcua.sdk.server.OpcUaServer; +import org.eclipse.milo.opcua.sdk.test.ScriptableSubscriptionServiceSet; +import org.eclipse.milo.opcua.sdk.test.TestClient; +import org.eclipse.milo.opcua.sdk.test.TestServer; +import org.eclipse.milo.opcua.stack.core.StatusCodes; +import org.eclipse.milo.opcua.stack.core.UaException; +import org.junit.jupiter.api.Test; + +/** + * What the Part 4 §6.7 Republish drain is allowed to do on the client's first Session + * activation. + * + *

The drain exists to collect the NotificationMessages a Server generated while a Session was + * unusable. On the first activation there is no such window: the Session and every Subscription + * registered against it were created moments earlier, on that Session. Running the drain anyway + * spends a Republish round trip per Subscription to be told Bad_MessageNotAvailable, and because + * the drain holds Publish traffic suspended until it ends, it also delays every Subscription's + * first PublishRequest by that much. + * + *

The window this guards is reachable rather than theoretical: {@code connect()} returns when + * the Session future completes, and that happens in a task submitted before the {@code + * onSessionActive} fan-out — so a Subscription created immediately afterwards can register itself + * before the callbacks run, and be included in a recovery snapshot that has nothing to recover. + * + *

Note on what these tests do and do not prove. They are a guard, not a reproduction: entering + * that window requires the calling thread to beat the fan-out task, which it loses on an idle + * machine, so this class passes against the code from before the drain was made conditional. What + * demonstrated the defect was the whole integration-tests module in one JVM fork on two CPUs, where + * the window opens often enough to produce a spurious Republish in {@code + * PublishSequenceRecoveryTest} and {@code PublishResponseOrderingTest}, a reset publish ceiling in + * {@code PublishCeilingRecoveryTest}, and — because a drain that is held holds the publish gate + * with it — a Publish that is never sent in {@code SubscriptionWatchdogRecoveryStarvationTest}. + * These assertions pin the intended behaviour so that a future change making recovery unconditional + * again fails here deterministically instead of as a flake somewhere else. + */ +public class PublishFirstActivationRecoveryTest { + + private static final long AWAIT_TIMEOUT_MILLIS = 10_000; + + /** + * How long to watch for a Republish that must not happen. Generous relative to the round trip it + * would take against a loopback Server. + */ + private static final long QUIET_PERIOD_MILLIS = 2_000; + + /** + * A Subscription created in the window between {@code connect()} returning and the activation + * callbacks running must not be dragged through a drain that cannot find anything. + */ + @Test + void noRepublishIsRequestedOnTheFirstSessionActivation() throws Exception { + try (var fixture = new Fixture()) { + var subscription = new OpcUaSubscription(fixture.client); + subscription.create(); + + // The client pipelines PublishRequests as soon as publishing is allowed, so their arrival is + // the signal that the gate opened and any recovery is over. + assertTrue( + fixture.awaitPublishRequests(2), + "no PublishRequest was sent, so the publish gate never opened: either recovery is still" + + " running or it never finished"); + + Thread.sleep(QUIET_PERIOD_MILLIS); + + assertEquals( + List.of(), + fixture.republishes(), + "the Subscription was created on the Session that has just become Active for the first" + + " time, so the Server cannot be holding a NotificationMessage the client has not" + + " collected and the §6.7 drain must not ask for one"); + } + } + + /** + * The control that keeps the assertion above from being vacuous: once a Session has actually been + * lost, the drain is exactly what §6.7 requires and must run. + */ + @Test + void republishIsRequestedAfterTheSessionIsReactivated() throws Exception { + try (var fixture = new Fixture()) { + var subscription = new OpcUaSubscription(fixture.client); + subscription.create(); + + assertTrue(fixture.awaitPublishRequests(2), "the publish gate never opened"); + assertEquals(List.of(), fixture.republishes(), "premise: no drain on the first activation"); + + // Bad_SessionIdInvalid is classified as a Session error and turned into a reconnect. The + // Server-side Session is untouched, so re-activation succeeds and the Subscription survives. + fixture.scriptable.enqueueServiceFault(StatusCodes.Bad_SessionIdInvalid); + + assertTrue( + fixture.sessionInactive.await(AWAIT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), + "the scripted Bad_SessionIdInvalid Publish fault did not take the Session out of Active"); + assertTrue( + fixture.sessionReactivated.await(AWAIT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), + "the Session never became Active again"); + + assertTrue( + fixture.awaitRepublish(), + "the Session was lost and re-activated, so Part 4 §6.7 requires the client to Republish" + + " from the next expected sequence number before resuming Publish handling, and it" + + " asked for nothing"); + } + } + + private static class Fixture implements AutoCloseable { + + final CountDownLatch sessionInactive = new CountDownLatch(1); + final CountDownLatch sessionReactivated = new CountDownLatch(1); + + private final List republishes = new CopyOnWriteArrayList<>(); + private final CountDownLatch republishRequested = new CountDownLatch(1); + + private final OpcUaServer server; + final OpcUaClient client; + final ScriptableSubscriptionServiceSet scriptable; + + Fixture() throws Exception { + TestServer testServer = TestServer.create(); + server = testServer.getServer(); + + scriptable = new ScriptableSubscriptionServiceSet(server); + for (EndpointConfig endpoint : server.getConfig().getEndpoints()) { + server.addServiceSet(endpoint.getPath(), scriptable); + } + + server.startup().get(); + + client = + TestClient.create( + server, + cfg -> + // Long enough that a parked PublishRequest does not time out, and no Session + // keep-alive traffic competes with the requests these tests script. + cfg.setRequestTimeout(uint(60_000)).setKeepAliveInterval(uint(60_000))); + + scriptable.setRepublishResponder( + request -> { + republishes.add(request.getRetransmitSequenceNumber().longValue()); + republishRequested.countDown(); + + // The termination condition of the §6.7 loop: the Server holds nothing to retransmit. + throw new UaException(StatusCodes.Bad_MessageNotAvailable); + }); + + client.connect(); + + client.addSessionActivityListener( + new SessionActivityListener() { + @Override + public void onSessionInactive(UaSession session) { + sessionInactive.countDown(); + } + + @Override + public void onSessionActive(UaSession session) { + if (sessionInactive.getCount() == 0) { + sessionReactivated.countDown(); + } + } + }); + } + + List republishes() { + return List.copyOf(republishes); + } + + boolean awaitRepublish() throws InterruptedException { + return republishRequested.await(AWAIT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); + } + + boolean awaitPublishRequests(int count) throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(AWAIT_TIMEOUT_MILLIS); + + while (System.nanoTime() < deadline) { + if (scriptable.getPublishRequestCount() >= count) { + return true; + } + Thread.sleep(25); + } + + return scriptable.getPublishRequestCount() >= count; + } + + @Override + public void close() throws Exception { + scriptable.failParkedRequests(StatusCodes.Bad_NoSubscription); + try { + client.disconnectAsync().get(5, TimeUnit.SECONDS); + } finally { + server.shutdown().get(5, TimeUnit.SECONDS); + } + } + } +} diff --git a/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/PublishPermitLeakTest.java b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/PublishPermitLeakTest.java new file mode 100644 index 0000000000..15af8ca3b0 --- /dev/null +++ b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/PublishPermitLeakTest.java @@ -0,0 +1,378 @@ +/* + * Copyright (c) 2026 the Eclipse Milo Authors + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.milo.opcua.sdk.client.subscriptions; + +import static org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.Unsigned.uint; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.eclipse.milo.opcua.sdk.client.OpcUaClient; +import org.eclipse.milo.opcua.sdk.server.EndpointConfig; +import org.eclipse.milo.opcua.sdk.server.OpcUaServer; +import org.eclipse.milo.opcua.sdk.test.ScriptableSubscriptionServiceSet; +import org.eclipse.milo.opcua.sdk.test.TestClient; +import org.eclipse.milo.opcua.sdk.test.TestServer; +import org.eclipse.milo.opcua.stack.core.StatusCodes; +import org.eclipse.milo.opcua.stack.core.UaException; +import org.eclipse.milo.opcua.stack.core.types.UaRequestMessageType; +import org.eclipse.milo.opcua.stack.core.types.UaResponseMessageType; +import org.eclipse.milo.opcua.stack.core.types.builtin.DateTime; +import org.eclipse.milo.opcua.stack.core.types.builtin.DiagnosticInfo; +import org.eclipse.milo.opcua.stack.core.types.builtin.StatusCode; +import org.eclipse.milo.opcua.stack.core.types.structured.PublishRequest; +import org.eclipse.milo.opcua.stack.core.types.structured.ReadResponse; +import org.eclipse.milo.opcua.stack.core.types.structured.ResponseHeader; +import org.eclipse.milo.opcua.stack.transport.client.tcp.OpcTcpClientTransport; +import org.eclipse.milo.opcua.stack.transport.client.tcp.OpcTcpClientTransportConfig; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +/** + * The pending-publish permits {@code PublishingManager} uses to keep a bounded number of + * PublishRequests in flight, and the answers that must not cost it one. + * + *

A permit is taken before a PublishRequest is sent and released when its answer has been dealt + * with; the release is also what sends the replacement request. A permit that is never released is + * therefore not merely a leak: the pipeline it belonged to permanently shrinks by one, and a client + * that loses its last permit stops asking the Server for notifications altogether, silently and for + * good — no error is reported, and the Subscription stays alive on the Server. + * + *

Every test here configures maxPendingPublishRequests = 1, so that the client's single + * Subscription targets min(1 + 1, 1) = 1 outstanding request. A leaked permit is then the whole + * pipeline, and the symptom is exactly the one the application would see: no further + * PublishRequest, ever. + * + * @see PublishPipelineRefillTest for the refill rules the permits feed. + */ +public class PublishPermitLeakTest { + + /** min(subscriptionCount + 1, maxPendingPublishRequests) = min(2, 1) with one Subscription. */ + private static final int PIPELINE_TARGET = 1; + + private static final long AWAIT_TIMEOUT_MILLIS = 10_000; + + /** + * Long enough that nothing times out on its own: the PublishRequests this test parks at the + * Server must stay parked until the test decides how they end. + */ + private static final long REQUEST_TIMEOUT_MILLIS = 60_000; + + /** + * Part 4 §5.14.5.2 gives a PublishRequest a PublishResponse, and the client dispatches on the + * type it gets: anything else goes down the failure path. That path had no failure to inspect — + * the request completed successfully, just not with a PublishResponse — so extracting a + * StatusCode from the (null) exception threw, and the permit the request held was never released. + * + *

The answer is fabricated by the transport rather than by the Server, because the type of a + * Publish answer is fixed by {@code SubscriptionServiceSet.onPublish}: a Server cannot express + * one through the service set, and a ServiceFault — the one wrong-typed message a Server can send + * — is turned into an exception by the client's channel before the SDK ever sees it. What is + * under test is the client's accounting of the answer, in which the Server's copy of the request + * plays no part, so the intercepted request is not sent at all. + */ + @Nested + class AnswerThatIsNotAPublishResponse { + + /** + * Control: the same interception answering with a service failure instead, i.e. the same path + * with an exception to extract a StatusCode from. It proves the fixture observes the pipeline + * being refilled after an intercepted answer at all, so the test below fails because the answer + * carried no exception and not because a refill is unobservable here. + */ + @Test + void publishRequestsResumeAfterAFailedPublishRequest() throws Exception { + try (var fixture = new Fixture(PublishAnswer.FAILURE)) { + fixture.createSubscription(); + + assertTrue(fixture.awaitInterceptedRequest(), "the client never sent a PublishRequest"); + + assertTrue( + fixture.awaitPublishRequestAtServer(), + "the failed PublishRequest was not replaced: its permit was never released"); + } + } + + @Test + void publishRequestsResumeAfterAnAnswerThatIsNotAPublishResponse() throws Exception { + try (var fixture = new Fixture(PublishAnswer.WRONG_TYPE)) { + fixture.createSubscription(); + + assertTrue(fixture.awaitInterceptedRequest(), "the client never sent a PublishRequest"); + + assertTrue( + fixture.awaitPublishRequestAtServer(), + "the client sent no further PublishRequest after being answered with a message that" + + " was not a PublishResponse: the answer completed the request successfully, so" + + " there was no exception to take a StatusCode from, and the pending-publish" + + " permit the request held was lost along with the whole pipeline"); + } + } + } + + /** + * A NotificationMessage is handed to the Subscription's delivery queue and the permit is released + * when the application callbacks have run, which is the backpressure the Server is held to. A + * delivery queue that cannot accept the task — {@code TaskQueue.submit()} returns {@code null} + * once it is shut down, or when its max queue size would be exceeded — has no callbacks to wait + * for, so nothing will ever release the permit on the message's behalf. + * + *

{@code recoverSubscription} already treated the same rejection from a Subscription's + * processing queue as work that will not happen and completed in its place; the delivery queue's + * rejection was simply dropped. + */ + @Nested + class DeliveryQueueThatCannotAcceptTheMessage { + + /** + * Control: the identical script with a delivery queue that can accept the message. It proves + * the fixture observes the pipeline being refilled after a PublishResponse at all, so the test + * below fails because the message could not be queued and not because a refill is unobservable + * here. + */ + @Test + void publishRequestsResumeAfterADeliveredNotificationMessage() throws Exception { + try (var fixture = new Fixture(PublishAnswer.NONE)) { + OpcUaSubscription subscription = fixture.createSubscription(); + assertTrue(fixture.awaitFullPipeline(), "the Publish pipeline never filled"); + + int before = fixture.scriptable.getPublishRequestCount(); + + fixture.scriptable.enqueueKeepAlive(subscription.getSubscriptionId().orElseThrow(), 1); + + assertTrue( + fixture.awaitPublishRequestsAtServer(before + 1), + "the answered PublishRequest was not replaced once its NotificationMessage had been" + + " delivered"); + } + } + + @Test + void publishRequestsResumeWhenTheDeliveryQueueCannotAcceptTheNotificationMessage() + throws Exception { + + try (var fixture = new Fixture(PublishAnswer.NONE)) { + OpcUaSubscription subscription = fixture.createSubscription(); + assertTrue(fixture.awaitFullPipeline(), "the Publish pipeline never filled"); + + // Shut down before the response arrives, so the rejection is a property of the fixture + // rather than a race with the delivery. + subscription.getDeliveryQueue().shutdown(false); + + int before = fixture.scriptable.getPublishRequestCount(); + + fixture.scriptable.enqueueKeepAlive(subscription.getSubscriptionId().orElseThrow(), 1); + + assertTrue( + fixture.awaitPublishRequestsAtServer(before + 1), + "the client sent no further PublishRequest after a NotificationMessage its delivery" + + " queue could not accept: nothing was waiting to deliver it, so nothing released" + + " the pending-publish permit the message's PublishRequest held, and the pipeline" + + " is empty for good"); + } + } + } + + // region helpers + + /** What the transport does with the client's first PublishRequest. */ + private enum PublishAnswer { + + /** Nothing: every PublishRequest is sent to the Server. */ + NONE, + + /** Answers it, without sending it, with a response that is not a PublishResponse. */ + WRONG_TYPE, + + /** Answers it, without sending it, with a service failure. */ + FAILURE + } + + /** + * An {@link OpcTcpClientTransport} that answers the client's first PublishRequest itself. + * + *

The interception is at the transport rather than at the Server because the answer being + * scripted is one no Server can express through a {@code SubscriptionServiceSet}, whose {@code + * onPublish} returns a {@code PublishResponse} and nothing else. + */ + private static final class MisansweringTransport extends OpcTcpClientTransport { + + private final AtomicInteger intercepted = new AtomicInteger(0); + + private final PublishAnswer answer; + + MisansweringTransport(OpcTcpClientTransportConfig config, PublishAnswer answer) { + super(config); + + this.answer = answer; + } + + int interceptedCount() { + return intercepted.get(); + } + + @Override + public CompletableFuture sendRequestMessage( + UaRequestMessageType requestMessage) { + + if (answer != PublishAnswer.NONE + && requestMessage instanceof PublishRequest publishRequest + && intercepted.incrementAndGet() == 1) { + + var future = new CompletableFuture(); + + // Completed on the transport's executor, which is where AbstractUascClientTransport + // completes any response that is not a PublishResponse: only PublishResponses go through + // the serial PublishResponse queue. + getConfig() + .getExecutor() + .execute( + () -> { + if (answer == PublishAnswer.FAILURE) { + future.completeExceptionally(new UaException(StatusCodes.Bad_InternalError)); + } else { + future.complete(notAPublishResponse(publishRequest)); + } + }); + + return future; + } + + return super.sendRequestMessage(requestMessage); + } + + /** + * A well-formed response of the wrong type, as a Server that mixed up its handlers would send. + */ + private static UaResponseMessageType notAPublishResponse(PublishRequest request) { + var responseHeader = + new ResponseHeader( + DateTime.now(), + request.getRequestHeader().getRequestHandle(), + StatusCode.GOOD, + DiagnosticInfo.NULL_VALUE, + null, + null); + + return new ReadResponse(responseHeader, null, null); + } + } + + /** + * A running Server whose Publish requests are all parked by {@link + * ScriptableSubscriptionServiceSet} — nothing completes one unless the test says so — plus a + * client whose transport answers its first PublishRequest according to {@code answer}. + */ + private static final class Fixture implements AutoCloseable { + + private final OpcUaServer server; + private final OpcUaClient client; + private final ScriptableSubscriptionServiceSet scriptable; + private final MisansweringTransport transport; + + Fixture(PublishAnswer answer) throws Exception { + TestServer testServer = TestServer.create(); + server = testServer.getServer(); + + scriptable = new ScriptableSubscriptionServiceSet(server); + for (EndpointConfig endpoint : server.getConfig().getEndpoints()) { + server.addServiceSet(endpoint.getPath(), scriptable); + } + + server.startup().get(); + + var transportHolder = new MisansweringTransport[1]; + + client = + TestClient.createWithTransport( + server, + transportConfig -> { + transportHolder[0] = new MisansweringTransport(transportConfig, answer); + return transportHolder[0]; + }, + cfg -> + cfg + // Long request timeout so parked Publish requests do not time out. + .setRequestTimeout(uint(REQUEST_TIMEOUT_MILLIS)) + // No Session keep-alive traffic: the only requests in flight during a test + // are the ones it scripts. + .setKeepAliveInterval(uint(REQUEST_TIMEOUT_MILLIS)) + .setMaxPendingPublishRequests(uint(PIPELINE_TARGET))); + + transport = transportHolder[0]; + + client.connect(); + } + + OpcUaSubscription createSubscription() throws UaException { + var subscription = new OpcUaSubscription(client); + subscription.create(); + + return subscription; + } + + /** Wait until the transport has answered a PublishRequest on the Server's behalf. */ + boolean awaitInterceptedRequest() throws Exception { + return awaitTrue(() -> transport.interceptedCount() >= 1, AWAIT_TIMEOUT_MILLIS); + } + + /** + * Wait until the client has its whole pipeline outstanding, parked at the Server exactly as a + * Server holding queued Publish requests for a Session would hold it. + */ + boolean awaitFullPipeline() throws Exception { + return awaitTrue( + () -> scriptable.getParkedRequestCount() >= PIPELINE_TARGET, AWAIT_TIMEOUT_MILLIS); + } + + boolean awaitPublishRequestAtServer() throws Exception { + return awaitPublishRequestsAtServer(1); + } + + boolean awaitPublishRequestsAtServer(int count) throws Exception { + return awaitTrue(() -> scriptable.getPublishRequestCount() >= count, AWAIT_TIMEOUT_MILLIS); + } + + @Override + public void close() throws Exception { + scriptable.failParkedRequests(StatusCodes.Bad_NoSubscription); + try { + client.disconnectAsync().get(5, TimeUnit.SECONDS); + } finally { + server.shutdown().get(10, TimeUnit.SECONDS); + } + } + } + + /** Polls {@code condition} until it holds or {@code timeoutMillis} elapses. */ + private static boolean awaitTrue(ThrowingBooleanSupplier condition, long timeoutMillis) + throws Exception { + + long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(timeoutMillis); + + while (System.nanoTime() < deadline) { + if (condition.get()) { + return true; + } + Thread.sleep(25); + } + + return condition.get(); + } + + @FunctionalInterface + private interface ThrowingBooleanSupplier { + boolean get() throws Exception; + } + + // endregion +} diff --git a/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/PublishPipelineRefillTest.java b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/PublishPipelineRefillTest.java new file mode 100644 index 0000000000..b930a31e73 --- /dev/null +++ b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/PublishPipelineRefillTest.java @@ -0,0 +1,406 @@ +/* + * Copyright (c) 2026 the Eclipse Milo Authors + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.milo.opcua.sdk.client.subscriptions; + +import static org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.Unsigned.uint; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import org.eclipse.milo.opcua.sdk.client.OpcUaClient; +import org.eclipse.milo.opcua.sdk.client.OpcUaClientConfigBuilder; +import org.eclipse.milo.opcua.sdk.server.EndpointConfig; +import org.eclipse.milo.opcua.sdk.server.OpcUaServer; +import org.eclipse.milo.opcua.sdk.test.ScriptableSubscriptionServiceSet; +import org.eclipse.milo.opcua.sdk.test.TestClient; +import org.eclipse.milo.opcua.sdk.test.TestServer; +import org.eclipse.milo.opcua.stack.core.StatusCodes; +import org.eclipse.milo.opcua.stack.core.UaException; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +/** + * When the Publish pipeline is refilled after a fault. + * + *

{@code PublishingManager} keeps {@code min(subscriptionCount + 1, maxPendingPublishRequests)} + * Publish requests outstanding per Session and refills the pipeline whenever one of them is dealt + * with. Two of the fault paths get the refill wrong, and both are exercised here through {@link + * ScriptableSubscriptionServiceSet}, which parks every Publish request until the test decides how + * it ends — so the timing belongs to the test rather than to a Server timer. + */ +public class PublishPipelineRefillTest { + + /** One Subscription targets min(1 + 1, UInteger.MAX_VALUE) = 2 outstanding Publish requests. */ + private static final int PIPELINE_TARGET = 2; + + /** + * The target of a client configured for a single pending PublishRequest: min(1 + 1, 1) = 1, so + * exactly one Publish request is ever outstanding. + */ + private static final int SINGLE_REQUEST_PIPELINE_TARGET = 1; + + private static final long PIPELINE_FILL_TIMEOUT_MILLIS = 10_000; + + private static final long REFILL_TIMEOUT_MILLIS = 10_000; + + /** How long to watch for a Publish request that must not be sent. */ + private static final long QUIET_WINDOW_MILLIS = 2_000; + + private static final long DELIVERY_TIMEOUT_MILLIS = 10_000; + + /** + * Part 4 §5.14.8.1: when the last Subscription of a Session is deleted, "all Publish requests + * still queued for that Session are de-queued and shall be returned with Bad_NoSubscription". + * Milo's own Server does exactly that in {@code SubscriptionManager}. + * + *

An application that deletes its last Subscription and immediately creates another therefore + * takes that whole burst of Bad_NoSubscription failures after its Subscription set has + * become non-empty again. {@code addSubscription()} cannot start the new Subscription's Publish + * traffic, because the permits taken by the de-queued requests are still held when it runs, so + * the refill has to come from the failure handler — and the failure handler suppresses it + * unconditionally for Bad_NoSubscription. + */ + @Nested + class DeletingTheLastSubscription { + + /** + * Control: the same delete/create/fail sequence, but with a status code the failure handler + * does not suppress the refill for. It proves the fixture can observe a refill at all, so the + * two tests below fail because of what Bad_NoSubscription does and not because a refill is + * unobservable here. + */ + @Test + void publishRequestsResumeWhenTheDeQueuedRequestsFailWithAnotherStatusCode() throws Exception { + try (var fixture = new Fixture()) { + OpcUaSubscription first = fixture.createSubscription(); + assertTrue(fixture.awaitFullPipeline(), "the Publish pipeline never filled"); + + first.delete(); + fixture.createSubscription(); + + int before = fixture.scriptable.getPublishRequestCount(); + + fixture.scriptable.failParkedRequests(StatusCodes.Bad_InternalError); + + assertTrue( + awaitTrue( + () -> fixture.scriptable.getPublishRequestCount() > before, REFILL_TIMEOUT_MILLIS), + "no Publish request was sent for the newly created Subscription"); + } + } + + @Test + void publishRequestsResumeAfterTheLastSubscriptionIsDeletedAndAnotherIsCreated() + throws Exception { + + try (var fixture = new Fixture()) { + OpcUaSubscription first = fixture.createSubscription(); + assertTrue(fixture.awaitFullPipeline(), "the Publish pipeline never filled"); + + first.delete(); + fixture.createSubscription(); + + int before = fixture.scriptable.getPublishRequestCount(); + + fixture.scriptable.failParkedRequests(StatusCodes.Bad_NoSubscription); + + assertTrue( + awaitTrue( + () -> fixture.scriptable.getPublishRequestCount() > before, REFILL_TIMEOUT_MILLIS), + "no Publish request was sent for the newly created Subscription: the de-queued requests" + + " released their permits without refilling the pipeline, and nothing else will"); + } + } + + /** + * The same defect stated as what the application sees. A Subscription that was created + * successfully, and that the Server is happily publishing for, never delivers anything. + */ + @Test + void recreatedSubscriptionReceivesNotificationsAfterTheLastOneWasDeleted() throws Exception { + try (var fixture = new Fixture()) { + OpcUaSubscription first = fixture.createSubscription(); + assertTrue(fixture.awaitFullPipeline(), "the Publish pipeline never filled"); + + first.delete(); + + OpcUaSubscription second = fixture.createSubscription(); + + var keepAliveReceived = new CountDownLatch(1); + second.setSubscriptionListener( + new OpcUaSubscription.SubscriptionListener() { + @Override + public void onKeepAliveReceived(OpcUaSubscription subscription) { + keepAliveReceived.countDown(); + } + }); + + fixture.scriptable.failParkedRequests(StatusCodes.Bad_NoSubscription); + + // Held by the harness until a Publish request arrives to carry it. + fixture.scriptable.enqueueKeepAlive(second.getSubscriptionId().orElseThrow(), 1); + + assertTrue( + keepAliveReceived.await(DELIVERY_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), + "the re-created Subscription never received a NotificationMessage: no Publish request" + + " was ever sent for it"); + } + } + } + + /** + * Part 4 §5.14.5.1: a Client "shall not issue another Publish request before one of its + * outstanding Publish requests is returned" once the Server has answered + * Bad_TooManyPublishRequests — one returned request buys one replacement. + * + *

The immediate half of that is implemented: the failed request is not replaced on the spot. + * But the target is left at subscriptionCount + 1, so the deficit the fault opened is still + * there, and the next PublishResponse that is delivered refills all of it at once. One returned + * request causes two to be issued, restoring exactly the outstanding count that drew the fault. + * + *

The same clause also requires a Server to accept at least subscriptionCount + 1 queued + * Publish requests, which is precisely what the Client aims for, so a conformant Server never + * answers Bad_TooManyPublishRequests here. This is about what the Client does when a + * non-conformant one does. + */ + @Nested + class TooManyPublishRequests { + + /** + * Control: the immediate suppression, which already works. It also establishes that the quiet + * window below is long enough for the failure to have been accounted for. + */ + @Test + void noPublishRequestIsSentImmediatelyAfterBadTooManyPublishRequests() throws Exception { + try (var fixture = new Fixture()) { + fixture.createSubscription(); + assertTrue(fixture.awaitFullPipeline(), "the Publish pipeline never filled"); + + int before = fixture.scriptable.getPublishRequestCount(); + + fixture.scriptable.enqueueServiceFault(StatusCodes.Bad_TooManyPublishRequests); + + assertFalse( + awaitTrue( + () -> fixture.scriptable.getPublishRequestCount() > before, QUIET_WINDOW_MILLIS), + "a Publish request was sent straight back at a Server that had just refused one for" + + " holding too many"); + } + } + + @Test + void atMostOnePublishRequestReplacesTheOneThatReturnedAfterTheFault() throws Exception { + try (var fixture = new Fixture()) { + OpcUaSubscription subscription = fixture.createSubscription(); + assertTrue(fixture.awaitFullPipeline(), "the Publish pipeline never filled"); + + fixture.scriptable.enqueueServiceFault(StatusCodes.Bad_TooManyPublishRequests); + + // The suppression asserted by the control test, repeated here because it is also the + // barrier: it is what makes the failure accounted for before the response below is + // delivered, so the number of requests that follow is a property of the client and not of + // which of the two completions won. + assertFalse( + awaitTrue( + () -> fixture.scriptable.getPublishRequestCount() > PIPELINE_TARGET, + QUIET_WINDOW_MILLIS), + "a Publish request was sent straight back at a Server that had just refused one for" + + " holding too many"); + + int before = fixture.scriptable.getPublishRequestCount(); + + fixture.scriptable.enqueueKeepAlive(subscription.getSubscriptionId().orElseThrow(), 1); + + assertTrue( + awaitTrue( + () -> fixture.scriptable.getPublishRequestCount() > before, REFILL_TIMEOUT_MILLIS), + "the returned Publish request was not replaced at all"); + + assertFalse( + awaitTrue( + () -> fixture.scriptable.getPublishRequestCount() > before + 1, + QUIET_WINDOW_MILLIS), + "two Publish requests were issued in response to one returning, restoring the very" + + " outstanding count that had just drawn Bad_TooManyPublishRequests"); + } + } + } + + /** + * The other half of Part 4 §5.14.5.1: a Client "shall not issue another Publish request before + * one of its outstanding Publish requests is returned". The refused request is itself one + * being returned, so when it was the only one outstanding that condition is met the moment the + * fault has been accounted for — and nothing else is left in flight whose return could ever meet + * it again. + * + *

"Exactly one outstanding" is reached by configuring maxPendingPublishRequests = 1: the + * client targets min(subscriptionCount + 1, maxPendingPublishRequests), which with one + * Subscription is min(2, 1) = 1. The ceiling the fault installs is max(1, outstanding) = 1, i.e. + * the target the client already had, so a pipeline that stays empty here cannot be blamed on the + * clamp. + * + *

This does not contradict {@link TooManyPublishRequests}: there two requests are outstanding, + * the fault returns one of them, one is still in flight, and the deficit is filled by exactly one + * replacement when that one comes back. The rule is one replacement per returned request, and the + * refused request is a returned request. + */ + @Nested + class TooManyPublishRequestsReturningTheOnlyOutstandingRequest { + + /** + * Control: the identical one-deep pipeline faulted with a status code the failure handler + * refills for unconditionally. It proves the fixture can observe a one-deep pipeline being + * refilled at all, so the test below fails because of what Bad_TooManyPublishRequests does and + * not because a refill is unobservable here. + */ + @Test + void publishRequestsResumeWhenTheOnlyOutstandingRequestFailsWithAnotherStatusCode() + throws Exception { + + try (Fixture fixture = Fixture.withSinglePendingPublishRequest()) { + fixture.createSubscription(); + assertTrue(fixture.awaitFullPipeline(), "the Publish pipeline never filled"); + + int before = fixture.scriptable.getPublishRequestCount(); + + fixture.scriptable.enqueueServiceFault(StatusCodes.Bad_InternalError); + + assertTrue( + awaitTrue( + () -> fixture.scriptable.getPublishRequestCount() > before, REFILL_TIMEOUT_MILLIS), + "the only outstanding Publish request failed and was not replaced"); + } + } + + @Test + void publishRequestsResumeWhenBadTooManyPublishRequestsReturnsTheLastOne() throws Exception { + + try (Fixture fixture = Fixture.withSinglePendingPublishRequest()) { + fixture.createSubscription(); + assertTrue(fixture.awaitFullPipeline(), "the Publish pipeline never filled"); + + int before = fixture.scriptable.getPublishRequestCount(); + + fixture.scriptable.enqueueServiceFault(StatusCodes.Bad_TooManyPublishRequests); + + assertTrue( + awaitTrue( + () -> fixture.scriptable.getPublishRequestCount() > before, REFILL_TIMEOUT_MILLIS), + "the Publish pipeline was left empty: the request the Server refused was the only one" + + " outstanding, so returning it met Part 4 §5.14.5.1's condition for issuing" + + " another, and no other request is left in flight whose return could restart the" + + " pipeline. Publish traffic is halted until something unrelated — a Subscription" + + " added, a Session activated — happens to restart it"); + } + } + } + + private static boolean awaitTrue(BooleanSupplierThrowing condition, long timeoutMillis) + throws Exception { + + long deadline = System.currentTimeMillis() + timeoutMillis; + while (System.currentTimeMillis() < deadline) { + if (condition.get()) { + return true; + } + Thread.sleep(25); + } + return condition.get(); + } + + @FunctionalInterface + private interface BooleanSupplierThrowing { + boolean get() throws Exception; + } + + /** + * A running Server whose Publish requests are all parked by {@link + * ScriptableSubscriptionServiceSet} — nothing completes one unless the test says so — plus a + * connected client. + */ + private static final class Fixture implements AutoCloseable { + + private final OpcUaServer server; + private final OpcUaClient client; + private final ScriptableSubscriptionServiceSet scriptable; + + /** The number of Publish requests this fixture's client keeps outstanding. */ + private final int pipelineTarget; + + Fixture() throws Exception { + this(PIPELINE_TARGET, cfg -> {}); + } + + /** + * A fixture whose client keeps a single PublishRequest outstanding, so a fault that answers it + * leaves nothing in flight. + */ + static Fixture withSinglePendingPublishRequest() throws Exception { + return new Fixture( + SINGLE_REQUEST_PIPELINE_TARGET, + cfg -> cfg.setMaxPendingPublishRequests(uint(SINGLE_REQUEST_PIPELINE_TARGET))); + } + + private Fixture(int pipelineTarget, Consumer configCustomizer) + throws Exception { + + this.pipelineTarget = pipelineTarget; + + TestServer testServer = TestServer.create(); + server = testServer.getServer(); + + scriptable = new ScriptableSubscriptionServiceSet(server); + for (EndpointConfig endpoint : server.getConfig().getEndpoints()) { + server.addServiceSet(endpoint.getPath(), scriptable); + } + + server.startup().get(); + + // Long request timeout so parked Publish requests do not time out during the test. + client = + TestClient.create( + server, + cfg -> { + cfg.setRequestTimeout(uint(60_000)); + configCustomizer.accept(cfg); + }); + client.connect(); + } + + OpcUaSubscription createSubscription() throws UaException { + var subscription = new OpcUaSubscription(client); + subscription.create(); + return subscription; + } + + /** + * Wait until the client has its whole pipeline outstanding, all of it parked by the harness. + * The Server therefore holds the requests exactly as a Server holding queued Publish requests + * for a Session would. + */ + boolean awaitFullPipeline() throws Exception { + return awaitTrue( + () -> scriptable.getParkedRequestCount() >= pipelineTarget, PIPELINE_FILL_TIMEOUT_MILLIS); + } + + @Override + public void close() throws Exception { + scriptable.failParkedRequests(StatusCodes.Bad_NoSubscription); + try { + client.disconnectAsync().get(5, TimeUnit.SECONDS); + } finally { + server.shutdown().get(5, TimeUnit.SECONDS); + } + } + } +} diff --git a/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/PublishReconnectRecoveryTest.java b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/PublishReconnectRecoveryTest.java new file mode 100644 index 0000000000..e5afe91497 --- /dev/null +++ b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/PublishReconnectRecoveryTest.java @@ -0,0 +1,943 @@ +/* + * Copyright (c) 2026 the Eclipse Milo Authors + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.milo.opcua.sdk.client.subscriptions; + +import static org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.Unsigned.uint; +import static org.junit.jupiter.api.Assertions.assertAll; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Deque; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import org.eclipse.milo.opcua.sdk.client.OpcUaClient; +import org.eclipse.milo.opcua.sdk.client.SessionActivityListener; +import org.eclipse.milo.opcua.sdk.client.UaSession; +import org.eclipse.milo.opcua.sdk.server.EndpointConfig; +import org.eclipse.milo.opcua.sdk.server.OpcUaServer; +import org.eclipse.milo.opcua.sdk.test.DelegatingSessionServiceSet; +import org.eclipse.milo.opcua.sdk.test.ScriptableSubscriptionServiceSet; +import org.eclipse.milo.opcua.sdk.test.TestClient; +import org.eclipse.milo.opcua.sdk.test.TestServer; +import org.eclipse.milo.opcua.stack.core.NodeIds; +import org.eclipse.milo.opcua.stack.core.StatusCodes; +import org.eclipse.milo.opcua.stack.core.UaException; +import org.eclipse.milo.opcua.stack.core.types.builtin.DataValue; +import org.eclipse.milo.opcua.stack.core.types.builtin.DateTime; +import org.eclipse.milo.opcua.stack.core.types.builtin.ExtensionObject; +import org.eclipse.milo.opcua.stack.core.types.builtin.StatusCode; +import org.eclipse.milo.opcua.stack.core.types.builtin.Variant; +import org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.UInteger; +import org.eclipse.milo.opcua.stack.core.types.structured.ActivateSessionRequest; +import org.eclipse.milo.opcua.stack.core.types.structured.ActivateSessionResponse; +import org.eclipse.milo.opcua.stack.core.types.structured.DataChangeNotification; +import org.eclipse.milo.opcua.stack.core.types.structured.MonitoredItemNotification; +import org.eclipse.milo.opcua.stack.core.types.structured.PublishRequest; +import org.eclipse.milo.opcua.stack.core.types.structured.PublishResponse; +import org.eclipse.milo.opcua.stack.core.types.structured.RepublishRequest; +import org.eclipse.milo.opcua.stack.core.types.structured.RepublishResponse; +import org.eclipse.milo.opcua.stack.core.types.structured.ResponseHeader; +import org.eclipse.milo.opcua.stack.core.types.structured.TransferResult; +import org.eclipse.milo.opcua.stack.core.types.structured.TransferSubscriptionsRequest; +import org.eclipse.milo.opcua.stack.core.types.structured.TransferSubscriptionsResponse; +import org.eclipse.milo.opcua.stack.transport.server.ServiceRequestContext; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +/** + * What the client does with a Subscription the first time its Session becomes {@code Active} again + * after a connection or Session fault. + * + *

Part 4 §6.7 "Re-establishing connections": "After re-establishing the connection the Client + * shall call Republish in a loop, starting with the next expected sequence number and incrementing + * the sequence number until the Server returns the status Bad_MessageNotAvailable." The loop comes + * before normal Publish handling resumes, and the reason is mechanical: the Server's + * retransmission queue is finite, and Part 4 §5.14.1.1 says that "in the case of a retransmission + * queue overflow, the oldest sent NotificationMessage gets deleted". Every NotificationMessage the + * Server sends in answer to a resumed Publish request is therefore capable of evicting one the + * client has not collected yet. + * + *

Milo already performs the rest of §6.7 — it re-establishes the SecureChannel, re-activates or + * re-creates the Session, and calls TransferSubscriptions with {@code sendInitialValues} — and it + * repairs a sequence gap reactively, once a PublishResponse has revealed one. What it does + * not do is drain the retransmission queue before letting that first PublishResponse arrive: {@code + * PublishingManager}'s {@code SessionActivityListener} goes straight to {@code + * maybeSendPublishRequests()}. The tests below drive both reconnect paths the Session FSM has and + * assert on the order in which requests reach the Server, and on what the application ends up + * seeing when the retransmission queue overflows in the window the missing order opens. + * + *

They also cover the second half of the same clause set. A successful TransferSubscriptions + * returns, per Part 4 §5.14.7.1, the sequence numbers of the NotificationMessages the Server is + * still holding for the transferred Subscription — "The Client should acknowledge all Messages in + * this list for which it will not request retransmission" — which is exactly the input a Republish + * drain needs. {@code SessionFsmFactory} reads only the TransferResult's StatusCode. + */ +public class PublishReconnectRecoveryTest { + + /** Log entry written when a PublishRequest reaches the Server. */ + private static final String PUBLISH = "Publish"; + + /** Prefix of the log entry written when a RepublishRequest reaches the Server. */ + private static final String REPUBLISH = "Republish:"; + + /** + * The sequence number of the last NotificationMessage the client accounts for before the Session + * fault, and therefore the sequence number §6.7's Republish loop has to start from. + */ + private static final long LAST_SEQUENCE_NUMBER_BEFORE_FAULT = 2; + + private static final long NEXT_EXPECTED_SEQUENCE_NUMBER = LAST_SEQUENCE_NUMBER_BEFORE_FAULT + 1; + + /** + * The two NotificationMessages the Server generates while the client is not connected, i.e. the + * ones §6.7's Republish loop exists to collect. + */ + private static final long[] GENERATED_WHILE_DISCONNECTED = {3, 4}; + + /** The sequence number of the first NotificationMessage the Server sends after the reconnect. */ + private static final long FIRST_SEQUENCE_NUMBER_AFTER_RECONNECT = 5; + + /** + * A retransmission queue with room for exactly the two NotificationMessages generated while the + * client was disconnected. Sending one more overflows it and deletes the oldest, which is the + * loss §6.7's ordering requirement is there to prevent. Real queues are larger — Part 4 §5.14.5.1 + * makes the Server hold "at least two times the number of Publish requests per Session" — but + * every queue has a size, and the number here only decides how many new NotificationMessages it + * takes to lose an old one. + */ + private static final int OVERFLOWING_RETRANSMISSION_QUEUE = 2; + + /** A retransmission queue with room to spare, so nothing is evicted during the test. */ + private static final int ROOMY_RETRANSMISSION_QUEUE = 4; + + /** How long to wait for something that must happen. */ + private static final long AWAIT_TIMEOUT_MILLIS = 10_000; + + /** + * How long to wait for a reconnect. The Session FSM waits one second in {@code ReactivatingWait} + * before its first re-activation attempt and another second in {@code CreatingWait} before + * creating a replacement Session, and doubles each wait on every failure. + */ + private static final long RECONNECT_TIMEOUT_MILLIS = 30_000; + + /** + * Long enough that nothing times out on its own: no parked Publish request, no Republish, and no + * Session keep-alive. Every stall asserted against below is therefore the client's own doing. + */ + private static final long REQUEST_TIMEOUT_MILLIS = 60_000; + + /** + * The same-session reconnect path: {@code Active -> ReactivatingWait -> Reactivating -> + * Initializing -> Active}. No TransferSubscriptions happens on it, so the only thing that can + * tell the client what the Server still holds is the Republish loop itself. + */ + @Nested + class ReactivatedSession { + + /** + * Part 4 §6.7: "After re-establishing the connection the Client shall call Republish in a loop, + * starting with the next expected sequence number and incrementing the sequence number until + * the Server returns the status Bad_MessageNotAvailable." + * + *

The very first request the client sends for a Subscription once its Session is {@code + * Active} again must therefore be a Republish for the next expected sequence number, not a + * Publish. + */ + @Test + void firstRequestAfterReactivationIsRepublishFromTheNextExpectedSequenceNumber() + throws Exception { + + try (var fixture = new Fixture(OVERFLOWING_RETRANSMISSION_QUEUE)) { + fixture.deliverInitialNotifications(); + + fixture.armRequestLog(); + fixture.faultSession(); + fixture.awaitReactivation(); + + assertTrue( + fixture.awaitRequestLogEntries(1), + "no request at all reached the Server after the Session was re-activated"); + + assertEquals( + REPUBLISH + NEXT_EXPECTED_SEQUENCE_NUMBER, + fixture.requestLog().get(0), + "the first request sent after re-activation must be the Republish that §6.7 requires;" + + " resuming Publish first lets the Server answer with a NotificationMessage that" + + " overwrites one still sitting in its retransmission queue"); + } + } + + /** + * The control that keeps the assertion above from passing vacuously: in this fixture the + * re-activated Session really does resume Publish traffic and deliver notifications, so a + * failure above is about what the client sends first, not about the client sending + * nothing at all. + */ + @Test + void publishResumesAfterReactivation() throws Exception { + try (var fixture = new Fixture(OVERFLOWING_RETRANSMISSION_QUEUE)) { + fixture.deliverInitialNotifications(); + + fixture.armRequestLog(); + fixture.faultSession(); + fixture.enqueueDataChange(NEXT_EXPECTED_SEQUENCE_NUMBER); + fixture.awaitReactivation(); + + assertTrue( + fixture.awaitDeliveredValues(List.of(1, 2, (int) NEXT_EXPECTED_SEQUENCE_NUMBER)), + "control: the NotificationMessage scripted for the first Publish after re-activation" + + " was never delivered, actual deliveries: " + + fixture.deliveredValues()); + } + } + } + + /** + * What the ordering requirement is worth, measured at the application. The Server generates two + * NotificationMessages while the client is away and holds them in a retransmission queue that has + * no room for a third. Whether the client collects them depends entirely on whether it asks + * before or after it lets the Server publish something new. + */ + @Nested + class NotificationMessagesGeneratedWhileTheSessionWasDown { + + /** + * The two NotificationMessages the Server generated while the client was away are in its + * retransmission queue when the Session becomes {@code Active} again, so §6.7's Republish loop + * would collect both. Resuming Publish first spends the queue's last slot on a new + * NotificationMessage instead, and the oldest of the two is deleted before the client ever asks + * for it — reported to the application only as {@code onNotificationDataLost}. + */ + @Test + void areRecoveredEvenWhenTheNextPublishResponseOverflowsTheRetransmissionQueue() + throws Exception { + + try (var fixture = new Fixture(OVERFLOWING_RETRANSMISSION_QUEUE)) { + fixture.deliverInitialNotifications(); + fixture.generateWhileDisconnected(GENERATED_WHILE_DISCONNECTED); + + fixture.armRequestLog(); + fixture.faultSession(); + fixture.enqueueDataChange(FIRST_SEQUENCE_NUMBER_AFTER_RECONNECT); + fixture.awaitReactivation(); + + fixture.awaitDeliveredValues(List.of(1, 2, 3, 4, 5)); + + assertAll( + () -> + assertEquals( + List.of(1, 2, 3, 4, 5), + fixture.deliveredValues(), + "every NotificationMessage the Server generated while the client was away was" + + " still in its retransmission queue when the Session became Active" + + " again, so all of them must reach the application exactly once and in" + + " sequence order"), + () -> + assertEquals( + 0, + fixture.notificationDataLostCount(), + "no NotificationMessage was unrecoverable at the moment the Session became" + + " Active again; reporting data lost means the client let the Server" + + " overwrite one before asking for it")); + } + } + + /** + * The control that isolates the retransmission queue overflow as the cause: the identical + * script, against a Server whose retransmission queue has room for the new NotificationMessage + * as well, loses nothing today. The reactive gap repair recovers both missing messages because + * the Server still happens to hold them when the first PublishResponse reveals the gap — which + * is luck, not a guarantee, and is exactly what §6.7's ordering removes the need for. + */ + @Test + void areRecoveredWhenTheRetransmissionQueueDoesNotOverflow() throws Exception { + try (var fixture = new Fixture(ROOMY_RETRANSMISSION_QUEUE)) { + fixture.deliverInitialNotifications(); + fixture.generateWhileDisconnected(GENERATED_WHILE_DISCONNECTED); + + fixture.armRequestLog(); + fixture.faultSession(); + fixture.enqueueDataChange(FIRST_SEQUENCE_NUMBER_AFTER_RECONNECT); + fixture.awaitReactivation(); + + fixture.awaitDeliveredValues(List.of(1, 2, 3, 4, 5)); + + assertAll( + () -> + assertEquals( + List.of(1, 2, 3, 4, 5), + fixture.deliveredValues(), + "control: with a retransmission queue that does not overflow, every" + + " NotificationMessage must reach the application"), + () -> + assertEquals( + 0, + fixture.notificationDataLostCount(), + "control: nothing was ever evicted, so nothing can be reported lost")); + } + } + } + + /** + * The new-session reconnect path: re-activation is refused, so the client creates a replacement + * Session and runs {@code Active -> ... -> Transferring -> Initializing -> Active}. Here the + * Server does tell the client what it is holding — Part 4 §5.14.7.1 gives each successful + * TransferResult the "sequence numbers of the NotificationMessages available for retransmission" + * — and the client throws that list away. + */ + @Nested + class TransferredSubscription { + + /** + * §6.7's Republish loop applies to this path too, and the TransferResult has just named the + * sequence numbers it should ask for. Every one of them must be requested before the client + * lets the Server answer a Publish request, because answering one is what overflows the + * retransmission queue holding them. + */ + @Test + void everySequenceNumberTheTransferAdvertisedIsRepublishedBeforePublishResumes() + throws Exception { + + try (var fixture = new Fixture(OVERFLOWING_RETRANSMISSION_QUEUE)) { + fixture.deliverInitialNotifications(); + fixture.generateWhileDisconnected(GENERATED_WHILE_DISCONNECTED); + fixture.advertiseOnTransfer(GENERATED_WHILE_DISCONNECTED); + fixture.refuseNextReactivation(); + + fixture.armRequestLog(); + fixture.faultSession(); + fixture.enqueueDataChange(FIRST_SEQUENCE_NUMBER_AFTER_RECONNECT); + fixture.awaitReactivation(); + + assertTrue( + fixture.awaitRequestLogContains(PUBLISH), + "Publish never resumed after the Subscription was transferred to the new Session"); + assertTrue( + fixture.transferCount() >= 1, + "precondition: the reconnect did not go through TransferSubscriptions, so this test" + + " is not exercising the transfer path at all"); + + List log = fixture.requestLog(); + int firstPublish = log.indexOf(PUBLISH); + + assertAll( + () -> + assertEquals( + REPUBLISH + GENERATED_WHILE_DISCONNECTED[0], + log.get(0), + "the first request sent after the transfer must be the Republish §6.7" + + " requires, for the oldest NotificationMessage the TransferResult said" + + " the Server still holds; request log: " + + log), + () -> + assertTrue( + isBefore(log, REPUBLISH + GENERATED_WHILE_DISCONNECTED[1], firstPublish), + "the TransferResult advertised sequence number " + + GENERATED_WHILE_DISCONNECTED[1] + + " as available for retransmission, so it must be requested before the" + + " first Publish; request log: " + + log)); + } + } + + /** + * The control for the test above: it proves the fixture really does drive the client onto the + * new-session path, that TransferSubscriptions is reached, and that Publish traffic resumes + * afterwards — so a failure above is about ordering rather than about a reconnect that never + * happened. + */ + @Test + void isTransferredToAReplacementSessionAndPublishResumes() throws Exception { + try (var fixture = new Fixture(OVERFLOWING_RETRANSMISSION_QUEUE)) { + fixture.deliverInitialNotifications(); + fixture.generateWhileDisconnected(GENERATED_WHILE_DISCONNECTED); + fixture.advertiseOnTransfer(GENERATED_WHILE_DISCONNECTED); + fixture.refuseNextReactivation(); + + fixture.armRequestLog(); + fixture.faultSession(); + fixture.enqueueDataChange(FIRST_SEQUENCE_NUMBER_AFTER_RECONNECT); + fixture.awaitReactivation(); + + assertTrue( + fixture.awaitTrue( + () -> + fixture.deliveredValues().contains((int) FIRST_SEQUENCE_NUMBER_AFTER_RECONNECT), + AWAIT_TIMEOUT_MILLIS), + "control: the NotificationMessage scripted for the first Publish after the transfer" + + " was never delivered, actual deliveries: " + + fixture.deliveredValues()); + + assertTrue( + fixture.transferCount() >= 1, + "control: the reconnect did not go through TransferSubscriptions"); + } + } + } + + /** + * Gating Publish on a recovery step trades a loss window for a permanent stall if any branch of + * that step can fail without releasing the gate. These are the guards for that: whatever the + * Republish loop runs into, Publish traffic has to resume. + */ + @Nested + class RecoveryFailureMustNotStallPublish { + + /** A Republish that fails with a service error, rather than Bad_MessageNotAvailable. */ + @Test + void publishResumesWhenEveryRepublishFailsWithAServiceError() throws Exception { + try (var fixture = new Fixture(OVERFLOWING_RETRANSMISSION_QUEUE)) { + fixture.deliverInitialNotifications(); + fixture.generateWhileDisconnected(GENERATED_WHILE_DISCONNECTED); + fixture.failEveryRepublishWith(StatusCodes.Bad_UnexpectedError); + + fixture.armRequestLog(); + fixture.faultSession(); + fixture.enqueueDataChange(FIRST_SEQUENCE_NUMBER_AFTER_RECONNECT); + fixture.awaitReactivation(); + + assertTrue( + fixture.awaitTrue( + () -> + fixture.deliveredValues().contains((int) FIRST_SEQUENCE_NUMBER_AFTER_RECONNECT), + AWAIT_TIMEOUT_MILLIS), + "Publish did not resume after every Republish failed: recovery that cannot complete" + + " must still hand the pipeline back, actual deliveries: " + + fixture.deliveredValues()); + } + } + + /** + * The Subscription no longer exists on the Server. Part 4 §6.7: "If the Republish returns + * Bad_SubscriptionIdInvalid, then the Client needs to create a new Subscription" — which is a + * statement about the Subscription, not a reason for the client's Publish pipeline to stop. + */ + @Test + void publishResumesWhenRepublishReportsTheSubscriptionIsGone() throws Exception { + try (var fixture = new Fixture(OVERFLOWING_RETRANSMISSION_QUEUE)) { + fixture.deliverInitialNotifications(); + fixture.generateWhileDisconnected(GENERATED_WHILE_DISCONNECTED); + fixture.failEveryRepublishWith(StatusCodes.Bad_SubscriptionIdInvalid); + + fixture.armRequestLog(); + fixture.faultSession(); + fixture.enqueueDataChange(FIRST_SEQUENCE_NUMBER_AFTER_RECONNECT); + fixture.awaitReactivation(); + + assertTrue( + fixture.awaitTrue( + () -> + fixture.deliveredValues().contains((int) FIRST_SEQUENCE_NUMBER_AFTER_RECONNECT), + AWAIT_TIMEOUT_MILLIS), + "Publish did not resume after Republish reported Bad_SubscriptionIdInvalid, actual" + + " deliveries: " + + fixture.deliveredValues()); + } + } + } + + /** + * @return {@code true} if {@code entry} appears in {@code log} before index {@code limit}. + */ + private static boolean isBefore(List log, String entry, int limit) { + int index = log.indexOf(entry); + + return index >= 0 && index < limit; + } + + // region fixture + + /** + * The order in which Publish and Republish requests reach the Server. + * + *

Recording starts only when the log is {@linkplain #arm() armed}, which a test does once its + * Publish pipeline is quiescent — one request parked at the Server and no responder scripted for + * it. From that moment nothing can reach the Server until the scripted Session fault has been + * answered and the Session has become {@code Active} again, so the first entry recorded is + * exactly the first request the client sends after the reconnect. + */ + private static final class RequestLog { + + private final List entries = Collections.synchronizedList(new ArrayList<>()); + + private volatile boolean armed = false; + + void arm() { + entries.clear(); + armed = true; + } + + void record(String entry) { + if (armed) { + entries.add(entry); + } + } + + List entries() { + return List.copyOf(entries); + } + } + + /** + * A Server-side retransmission queue of bounded size. + * + *

Part 4 §5.14.1.1: "In the case of a retransmission queue overflow, the oldest sent + * NotificationMessage gets deleted." A NotificationMessage is available for Republish while it is + * in the queue, and the queue's contents are what a PublishResponse advertises in + * availableSequenceNumbers. + */ + private static final class RetransmissionQueue { + + private final Deque queue = new ArrayDeque<>(); + + private final int capacity; + + RetransmissionQueue(int capacity) { + this.capacity = capacity; + } + + /** Record that the Server has sent the NotificationMessage with this sequence number. */ + synchronized void send(long sequenceNumber) { + queue.addLast(sequenceNumber); + + while (queue.size() > capacity) { + queue.removeFirst(); + } + } + + synchronized boolean holds(long sequenceNumber) { + return queue.contains(sequenceNumber); + } + + synchronized UInteger[] available() { + return queue.stream().map(sequenceNumber -> uint(sequenceNumber)).toArray(UInteger[]::new); + } + } + + /** + * A {@link ScriptableSubscriptionServiceSet} that records the arrival of every PublishRequest and + * can answer TransferSubscriptions with a scripted availableSequenceNumbers list. + */ + private static final class LoggingSubscriptionServiceSet + extends ScriptableSubscriptionServiceSet { + + private final AtomicInteger transferCount = new AtomicInteger(0); + + /** + * {@code null} until a test scripts the transfer, at which point the Server delegate is no + * longer consulted. + */ + private volatile UInteger[] transferAvailableSequenceNumbers = null; + + private final RequestLog requestLog; + + LoggingSubscriptionServiceSet(OpcUaServer server, RequestLog requestLog) { + super(server); + + this.requestLog = requestLog; + } + + @Override + public CompletableFuture onPublish( + ServiceRequestContext context, PublishRequest request) { + + requestLog.record(PUBLISH); + + return super.onPublish(context, request); + } + + @Override + public TransferSubscriptionsResponse onTransferSubscriptions( + ServiceRequestContext context, TransferSubscriptionsRequest request) throws UaException { + + UInteger[] available = transferAvailableSequenceNumbers; + + if (available == null) { + return super.onTransferSubscriptions(context, request); + } + + transferCount.incrementAndGet(); + + UInteger[] subscriptionIds = request.getSubscriptionIds(); + int count = subscriptionIds != null ? subscriptionIds.length : 0; + + var results = new TransferResult[count]; + for (int i = 0; i < count; i++) { + results[i] = new TransferResult(StatusCode.GOOD, available); + } + + var responseHeader = + new ResponseHeader( + DateTime.now(), + request.getRequestHeader().getRequestHandle(), + StatusCode.GOOD, + null, + null, + null); + + return new TransferSubscriptionsResponse(responseHeader, results, null); + } + } + + /** + * A {@link DelegatingSessionServiceSet} that can refuse a single ActivateSession with a + * ServiceFault, which is what drives the Session FSM off the re-activation path and onto the + * create-a-new-Session-and-transfer path. + */ + private static final class RefusingSessionServiceSet extends DelegatingSessionServiceSet { + + private final AtomicBoolean refuseNext = new AtomicBoolean(false); + + RefusingSessionServiceSet(OpcUaServer server) { + super(server); + } + + @Override + public ActivateSessionResponse onActivateSession( + ServiceRequestContext context, ActivateSessionRequest request) throws UaException { + + if (refuseNext.compareAndSet(true, false)) { + throw new UaException(StatusCodes.Bad_SessionIdInvalid); + } + + return super.onActivateSession(context, request); + } + } + + /** + * A running Server whose Publish, Republish and TransferSubscriptions responses are scripted, and + * a connected client with one Subscription carrying one MonitoredItem. + * + *

The client is configured for a single pending PublishRequest. That makes the pipeline state + * unambiguous at the moment the Session fault is scripted: exactly one request is parked at the + * Server, the fault answers it, and nothing is left outstanding to muddle what the client sends + * once the Session is {@code Active} again. + */ + private static final class Fixture implements AutoCloseable { + + private final RequestLog requestLog = new RequestLog(); + + private final RetransmissionQueue retransmissionQueue; + + private final List deliveredValues = Collections.synchronizedList(new ArrayList<>()); + + private final AtomicInteger notificationDataLostCount = new AtomicInteger(0); + + private final CountDownLatch sessionInactive = new CountDownLatch(1); + private final CountDownLatch sessionReactivated = new CountDownLatch(1); + + /** Non-zero while every Republish is scripted to fail with that StatusCode. */ + private volatile long republishFailure = 0L; + + private final OpcUaServer server; + private final OpcUaClient client; + private final LoggingSubscriptionServiceSet scriptable; + private final RefusingSessionServiceSet sessionServiceSet; + + private final OpcUaSubscription subscription; + private final UInteger subscriptionId; + private final UInteger clientHandle; + + Fixture(int retransmissionQueueCapacity) throws Exception { + retransmissionQueue = new RetransmissionQueue(retransmissionQueueCapacity); + + TestServer testServer = TestServer.create(); + server = testServer.getServer(); + + scriptable = new LoggingSubscriptionServiceSet(server, requestLog); + sessionServiceSet = new RefusingSessionServiceSet(server); + + for (EndpointConfig endpoint : server.getConfig().getEndpoints()) { + server.addServiceSet(endpoint.getPath(), scriptable); + server.addServiceSet(endpoint.getPath(), sessionServiceSet); + } + + server.startup().get(); + + client = + TestClient.create( + server, + cfg -> + cfg.setRequestTimeout(uint(REQUEST_TIMEOUT_MILLIS)) + // No Session keep-alive traffic: the only requests in flight during a test + // are the ones it scripts. + .setKeepAliveInterval(uint(REQUEST_TIMEOUT_MILLIS)) + .setMaxPendingPublishRequests(uint(1))); + client.connect(); + + client.addSessionActivityListener( + new SessionActivityListener() { + @Override + public void onSessionInactive(UaSession session) { + sessionInactive.countDown(); + } + + @Override + public void onSessionActive(UaSession session) { + if (sessionInactive.getCount() == 0) { + sessionReactivated.countDown(); + } + } + }); + + scriptable.setRepublishResponder(this::respondToRepublish); + + subscription = new OpcUaSubscription(client); + subscription.setSubscriptionListener( + new OpcUaSubscription.SubscriptionListener() { + @Override + public void onDataReceived( + OpcUaSubscription s, List items, List values) { + + for (DataValue value : values) { + deliveredValues.add((Integer) value.getValue().getValue()); + } + } + + @Override + public void onNotificationDataLost(OpcUaSubscription s) { + notificationDataLostCount.incrementAndGet(); + } + }); + subscription.create(); + + subscriptionId = subscription.getSubscriptionId().orElseThrow(); + + // The MonitoredItem only has to exist on the client: addMonitoredItem assigns the + // ClientHandle the notification fan-out looks scripted notifications up by, and no + // Server-side item participates in delivering one. + OpcUaMonitoredItem item = + OpcUaMonitoredItem.newDataItem(NodeIds.Server_ServerStatus_CurrentTime); + subscription.addMonitoredItem(item); + + clientHandle = item.getClientHandle().orElseThrow(); + } + + /** + * Deliver NotificationMessages 1 and 2, leaving the client's last accounted-for sequence number + * at {@value #LAST_SEQUENCE_NUMBER_BEFORE_FAULT} and its Publish pipeline quiescent: one + * request parked at the Server with no responder scripted for it. + */ + void deliverInitialNotifications() throws Exception { + enqueueDataChange(1); + assertTrue( + awaitTrue(() -> deliveredValues().size() >= 1, AWAIT_TIMEOUT_MILLIS), + "the first NotificationMessage was never delivered"); + + enqueueDataChange(LAST_SEQUENCE_NUMBER_BEFORE_FAULT); + assertTrue( + awaitTrue(() -> deliveredValues().size() >= 2, AWAIT_TIMEOUT_MILLIS), + "the second NotificationMessage was never delivered"); + + assertTrue( + awaitTrue(() -> scriptable.getParkedRequestCount() == 1, AWAIT_TIMEOUT_MILLIS), + "the client did not refill its Publish pipeline"); + + assertEquals( + List.of(1, (int) LAST_SEQUENCE_NUMBER_BEFORE_FAULT), + deliveredValues(), + "precondition: the client must have accounted for NotificationMessages 1 and 2 before" + + " the Session fault"); + assertEquals( + 0, + notificationDataLostCount(), + "precondition: nothing may be lost before the Session fault"); + } + + /** + * Record that the Server generated these NotificationMessages, and is therefore holding them in + * its retransmission queue, while the client was not connected. + */ + void generateWhileDisconnected(long... sequenceNumbers) { + for (long sequenceNumber : sequenceNumbers) { + retransmissionQueue.send(sequenceNumber); + } + } + + /** Answer TransferSubscriptions with these availableSequenceNumbers (Part 4 §5.14.7.1). */ + void advertiseOnTransfer(long... sequenceNumbers) { + var available = new UInteger[sequenceNumbers.length]; + for (int i = 0; i < sequenceNumbers.length; i++) { + available[i] = uint(sequenceNumbers[i]); + } + + scriptable.transferAvailableSequenceNumbers = available; + } + + /** + * Refuse the next ActivateSession with a ServiceFault, which sends the Session FSM to {@code + * CreatingWait} and from there onto the create-a-new-Session-and-transfer path. + */ + void refuseNextReactivation() { + sessionServiceSet.refuseNext.set(true); + } + + /** Script every Republish to fail with {@code statusCode} rather than model the queue. */ + void failEveryRepublishWith(long statusCode) { + republishFailure = statusCode; + } + + void armRequestLog() { + requestLog.arm(); + } + + /** + * Answer the one parked PublishRequest with a Bad_SessionIdInvalid ServiceFault, which {@code + * SessionFsmFactory}'s SessionFaultListener classifies as a Session error and turns into a + * reconnect. The Server-side Session is untouched, so re-activation succeeds unless {@link + * #refuseNextReactivation()} was called. + */ + void faultSession() { + scriptable.enqueueServiceFault(StatusCodes.Bad_SessionIdInvalid); + } + + /** + * Script the next PublishResponse as a data message carrying {@code sequenceNumber}, whose + * value identifies the NotificationMessage it came from. + * + *

Sending it puts the sequence number in the Server's retransmission queue, evicting the + * oldest entry if the queue is full, and the queue's contents become the response's + * availableSequenceNumbers. That is the causal link the ordering requirement is about: the + * eviction happens when the Server answers a Publish request, so whether a Republish finds a + * NotificationMessage depends on whether it was sent before or after Publish resumed. + * + *

Called immediately after {@link #faultSession()}, when the Server has no parked + * PublishRequest left, so this responder waits in the script rather than being applied at once. + * The Session FSM waits a whole second in {@code ReactivatingWait} before it even attempts to + * reconnect, so the client cannot have sent its next PublishRequest by then. + */ + void enqueueDataChange(long sequenceNumber) { + scriptable.enqueue( + request -> { + retransmissionQueue.send(sequenceNumber); + + return CompletableFuture.completedFuture( + scriptable.buildPublishResponse( + request, + subscriptionId, + sequenceNumber, + notificationData(sequenceNumber), + retransmissionQueue.available(), + false)); + }); + } + + /** + * Answer a Republish the way a Server holding a bounded retransmission queue would: with the + * NotificationMessage if it is still in the queue, and Bad_MessageNotAvailable if it is not — + * which is also what terminates the Republish loop Part 4 §6.7 describes. + */ + private RepublishResponse respondToRepublish(RepublishRequest request) throws UaException { + long sequenceNumber = request.getRetransmitSequenceNumber().longValue(); + + requestLog.record(REPUBLISH + sequenceNumber); + + long failure = republishFailure; + if (failure != 0L) { + throw new UaException(failure); + } + + if (!retransmissionQueue.holds(sequenceNumber)) { + throw new UaException(StatusCodes.Bad_MessageNotAvailable); + } + + return scriptable.buildRepublishResponse( + request, sequenceNumber, notificationData(sequenceNumber)); + } + + private ExtensionObject[] notificationData(long sequenceNumber) { + var notification = + new MonitoredItemNotification( + clientHandle, new DataValue(Variant.ofInt32((int) sequenceNumber))); + + return new ExtensionObject[] { + scriptable.encode( + new DataChangeNotification(new MonitoredItemNotification[] {notification}, null)) + }; + } + + /** Wait for the Session to leave {@code Active} and come back to it. */ + void awaitReactivation() throws Exception { + assertTrue( + sessionInactive.await(RECONNECT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), + "the scripted Bad_SessionIdInvalid Publish fault did not take the Session out of Active"); + assertTrue( + sessionReactivated.await(RECONNECT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), + "the Session never became Active again"); + } + + List requestLog() { + return requestLog.entries(); + } + + boolean awaitRequestLogEntries(int count) throws Exception { + return awaitTrue(() -> requestLog.entries().size() >= count, AWAIT_TIMEOUT_MILLIS); + } + + boolean awaitRequestLogContains(String entry) throws Exception { + return awaitTrue(() -> requestLog.entries().contains(entry), AWAIT_TIMEOUT_MILLIS); + } + + List deliveredValues() { + return List.copyOf(deliveredValues); + } + + boolean awaitDeliveredValues(List expected) throws Exception { + return awaitTrue(() -> deliveredValues().equals(expected), AWAIT_TIMEOUT_MILLIS); + } + + int notificationDataLostCount() { + return notificationDataLostCount.get(); + } + + int transferCount() { + return scriptable.transferCount.get(); + } + + /** Polls {@code condition} until it holds or the timeout elapses. */ + boolean awaitTrue(ThrowingBooleanSupplier condition, long timeoutMillis) throws Exception { + long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(timeoutMillis); + + while (System.nanoTime() < deadline) { + if (condition.get()) { + return true; + } + Thread.sleep(10); + } + + return condition.get(); + } + + @Override + public void close() throws Exception { + scriptable.failParkedRequests(StatusCodes.Bad_NoSubscription); + try { + client.disconnectAsync().get(5, TimeUnit.SECONDS); + } finally { + server.shutdown().get(10, TimeUnit.SECONDS); + } + } + } + + @FunctionalInterface + private interface ThrowingBooleanSupplier { + boolean get() throws Exception; + } + + // endregion +} diff --git a/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/PublishResponseOrderingTest.java b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/PublishResponseOrderingTest.java new file mode 100644 index 0000000000..fa2be7ea2a --- /dev/null +++ b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/PublishResponseOrderingTest.java @@ -0,0 +1,557 @@ +/* + * Copyright (c) 2026 the Eclipse Milo Authors + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.milo.opcua.sdk.client.subscriptions; + +import static org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.Unsigned.uint; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import org.eclipse.milo.opcua.sdk.client.OpcUaClient; +import org.eclipse.milo.opcua.sdk.server.EndpointConfig; +import org.eclipse.milo.opcua.sdk.server.OpcUaServer; +import org.eclipse.milo.opcua.sdk.test.ScriptableSubscriptionServiceSet; +import org.eclipse.milo.opcua.sdk.test.TestClient; +import org.eclipse.milo.opcua.sdk.test.TestServer; +import org.eclipse.milo.opcua.stack.core.NodeIds; +import org.eclipse.milo.opcua.stack.core.StatusCodes; +import org.eclipse.milo.opcua.stack.core.types.builtin.DataValue; +import org.eclipse.milo.opcua.stack.core.types.builtin.ExtensionObject; +import org.eclipse.milo.opcua.stack.core.types.builtin.Variant; +import org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.UInteger; +import org.eclipse.milo.opcua.stack.core.types.structured.DataChangeNotification; +import org.eclipse.milo.opcua.stack.core.types.structured.MonitoredItemNotification; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +/** + * Ordering and idempotence of NotificationMessage accounting in {@code + * PublishingManager.processPublishResponse}. + * + *

Part 4 §5.14.1.1 numbers NotificationMessages so a Client can tell a gap from an ordinary + * delivery; {@code lastSequenceNumber} is the client's record of the last one accounted for and is + * the sole input to gap detection and therefore to Republish recovery. Two independent things have + * to hold for that record to stay meaningful: + * + *

    + *
  1. PublishResponses must be processed in the order the Server sent them. The transport + * guarantees ordered delivery: {@code AbstractUascClientTransport.handleResponse} + * completes PublishResponse futures on a serial {@code ExecutionQueue} for exactly this + * reason. {@code PublishingManager} then hands its completion handler to {@code + * whenCompleteAsync(..., executor)}, and that executor is a genuinely multi-threaded pool, so + * two handlers dispatched in order can run in either order. + *
  2. Processing must be monotonic. A NotificationMessage that has already been accounted for + * must not move {@code lastSequenceNumber} backwards or be delivered a second time, whatever + * the reason it arrived late. + *
+ * + *

Either failure produces the same corruption: {@code lastSequenceNumber} regresses, the next + * NotificationMessage looks like the far end of a gap, and the client issues a blocking Republish + * for a message it already has — repeatedly, once per message, for the rest of the Subscription's + * life. The nested classes below drive each cause and assert on the same two observable + * consequences: which Republish requests reach the Server, and how many times each + * NotificationMessage is delivered to the application. + */ +public class PublishResponseOrderingTest { + + /** How long to wait for a condition that must become true. */ + private static final long AWAIT_TIMEOUT_MILLIS = 10_000; + + /** How long to wait for the two Publish completions to be inverted before giving up on it. */ + private static final long REORDER_TIMEOUT_MILLIS = 3_000; + + /** + * The Server sent 2 and then 3, and both were delivered to the client in that order. Whichever + * order the client's own executor happens to run the two completion handlers in, nothing was + * lost, so nothing may be requested via Republish. + */ + @Nested + class ReorderedInsideTheClient { + + @Test + void reorderedPublishResponsesDoNotTriggerRepublish() throws Exception { + try (var fixture = new Fixture()) { + fixture.driveReorderedResponses(); + + assertEquals( + List.of(), + List.copyOf(fixture.republishRequests), + "no NotificationMessage was lost — the Server sent 2, 3, 4 and the transport delivered" + + " them in that order — so the client must not ask the Server to retransmit" + + " anything"); + } + } + + /** + * The other half of the same corruption. Processing the later response first republishes the + * earlier one and delivers it, and processing the earlier one afterwards delivers it again: the + * application sees the same NotificationMessage twice. + */ + @Test + void reorderedPublishResponsesAreEachDeliveredExactlyOnce() throws Exception { + try (var fixture = new Fixture()) { + fixture.driveReorderedResponses(); + + assertEquals( + List.of(1, 2, 3, 4), + List.copyOf(fixture.deliveredValues), + "each NotificationMessage must be delivered to the application exactly once, in the" + + " order the Server sent it"); + } + } + } + + /** + * The same stale input, produced without touching the client's threading: the Server is scripted + * to send NotificationMessage 3 before 2. Sequence 3 legitimately looks like a gap, so Republish( + * 2) is correct and expected; what must not happen is that the subsequent, already-accounted-for + * sequence 2 rolls {@code lastSequenceNumber} back to 2 and turns the perfectly ordinary sequence + * 4 into another gap. + * + *

This pins the monotonicity requirement independently of the reordering above: preserving the + * order of the completion handlers fixes the cause, but does not by itself make processing safe + * against a message that has already been accounted for. + */ + @Nested + class StaleNotificationMessage { + + @Test + void alreadyAccountedNotificationMessageDoesNotCauseAFurtherRepublish() throws Exception { + try (var fixture = new Fixture()) { + fixture.driveStaleResponse(); + + assertEquals( + List.of(2L), + List.copyOf(fixture.republishRequests), + "sequence 2 was missing when 3 arrived and is correctly recovered; sequence 3 was" + + " received and accounted for, so the arrival of the stale sequence 2 must not" + + " make 3 look missing again"); + } + } + + @Test + void alreadyAccountedNotificationMessageIsNotDeliveredAgain() throws Exception { + try (var fixture = new Fixture()) { + fixture.driveStaleResponse(); + + assertEquals( + List.of(1, 2, 3, 4), + List.copyOf(fixture.deliveredValues), + "sequence 2 was already delivered when it was recovered via Republish; delivering the" + + " stale copy as well hands the application a duplicate"); + } + } + } + + /** + * Positive control for every "was Republish called for X?" assertion above: a genuine gap is + * recorded by the fixture's Republish responder. If this fails, the fixture is broken rather than + * the sequence accounting. + */ + @Test + void genuineGapIsRecordedByTheRepublishResponder() throws Exception { + try (var fixture = new Fixture()) { + // Sequence 2 is lost; sequence 3 reveals the gap. + fixture.scriptable.enqueueDataChange( + fixture.subscriptionId, 3, fixture.notifications(3), uint(2), uint(3)); + + assertTrue( + fixture.awaitTrue(() -> fixture.deliveredValues.contains(3)), + "sequence 3 was never delivered"); + + assertEquals( + List.of(2L), + List.copyOf(fixture.republishRequests), + "the missing NotificationMessage 2 must be requested via Republish"); + } + } + + /** + * Control for the fixture's {@link ReorderingExecutor}: it must actually invert two tasks that a + * single ordered source dispatched in order. Without this, {@link ReorderedInsideTheClient} could + * pass simply because no reordering ever took place. + */ + @Test + void reorderingExecutorInvertsTwoDispatchesFromASingleOrderedSource() throws Exception { + var executor = new ReorderingExecutor(); + + try { + List order = Collections.synchronizedList(new ArrayList<>()); + var bothRan = new CountDownLatch(2); + + executor.armReorder(); + + // The shape of the transport's serial publish-response queue: one task dispatching, in order, + // the dependent stages of two futures it completes. + executor.execute( + () -> { + executor.execute( + () -> { + order.add("first"); + bothRan.countDown(); + }); + executor.execute( + () -> { + order.add("second"); + bothRan.countDown(); + }); + }); + + assertTrue( + executor.awaitReorder(REORDER_TIMEOUT_MILLIS), + "the executor did not invert the two dispatches"); + assertTrue(bothRan.await(5, TimeUnit.SECONDS), "not every dispatched task ran"); + + assertEquals( + List.of("second", "first"), + List.copyOf(order), + "the task dispatched first must run after the task dispatched second"); + } finally { + executor.shutdownNow(); + } + } + + /** + * A running Server whose Publish and Republish responses are scripted, plus a client driven by a + * {@link ReorderingExecutor}, a Subscription with one client-side MonitoredItem, and a listener + * recording every value delivered. + * + *

Construction leaves the client in a known state: NotificationMessage 1 has been received and + * delivered, so {@code lastSequenceNumber} is 1, and the Publish pipeline is full again. + */ + private static final class Fixture implements AutoCloseable { + + /** Every {@code retransmitSequenceNumber} the client has asked the Server to Republish. */ + private final List republishRequests = Collections.synchronizedList(new ArrayList<>()); + + /** Every value handed to {@code onDataReceived}, in delivery order. */ + private final List deliveredValues = Collections.synchronizedList(new ArrayList<>()); + + private final ReorderingExecutor executor = new ReorderingExecutor(); + + private final OpcUaServer server; + private final OpcUaClient client; + private final ScriptableSubscriptionServiceSet scriptable; + private final OpcUaSubscription subscription; + private final UInteger subscriptionId; + private final UInteger clientHandle; + + Fixture() throws Exception { + TestServer testServer = TestServer.create(); + server = testServer.getServer(); + + scriptable = new ScriptableSubscriptionServiceSet(server); + for (EndpointConfig endpoint : server.getConfig().getEndpoints()) { + server.addServiceSet(endpoint.getPath(), scriptable); + } + + server.startup().get(); + + client = + TestClient.create( + server, + transportConfig -> transportConfig.setExecutor(executor), + cfg -> + cfg + // Long request timeout so parked Publish requests do not time out. + .setRequestTimeout(uint(60_000)) + // No Session keep-alive traffic: the only responses in flight during the test + // are the scripted ones. + .setKeepAliveInterval(uint(60_000))); + client.connect(); + + subscription = new OpcUaSubscription(client); + subscription.setSubscriptionListener( + new OpcUaSubscription.SubscriptionListener() { + @Override + public void onDataReceived( + OpcUaSubscription subscription, + List items, + List values) { + + values.forEach(value -> deliveredValues.add((Integer) value.getValue().getValue())); + } + }); + subscription.create(); + + subscriptionId = subscription.getSubscriptionId().orElseThrow(); + + // Client-side only: addMonitoredItem() assigns the ClientHandle the notification fan-out + // looks values up by, which is all a scripted notification needs. + var item = OpcUaMonitoredItem.newDataItem(NodeIds.Server_ServerStatus_CurrentTime); + subscription.addMonitoredItem(item); + clientHandle = item.getClientHandle().orElseThrow(); + + scriptable.setRepublishResponder( + request -> { + long sequenceNumber = request.getRetransmitSequenceNumber().longValue(); + republishRequests.add(sequenceNumber); + + return scriptable.buildRepublishResponse( + request, sequenceNumber, encodedNotifications((int) sequenceNumber)); + }); + + establishFirstNotificationMessage(); + } + + /** + * Drive {@code lastSequenceNumber} to 1 and wait until the Publish pipeline has refilled, so + * that both of the responses the scenarios script below land on requests that are already + * parked at the Server. + */ + private void establishFirstNotificationMessage() throws Exception { + scriptable.enqueueDataChange(subscriptionId, 1, notifications(1), uint(1)); + + assertTrue( + awaitTrue(() -> deliveredValues.contains(1)), "the first NotificationMessage was lost"); + assertTrue( + awaitTrue(() -> scriptable.getParkedRequestCount() >= 2), + "the client did not refill its Publish pipeline"); + } + + /** + * NotificationMessages 2 and 3 are sent, in order, onto the two parked Publish requests; the + * client's executor is armed to run their completion handlers in the opposite order. Sequence 4 + * follows once both have been processed, and its delivery is the barrier that proves everything + * before it has been delivered too: the processing queue and the delivery queue are both FIFO. + */ + void driveReorderedResponses() throws Exception { + int publishRequestsBefore = scriptable.getPublishRequestCount(); + + executor.armReorder(); + + scriptable.enqueueDataChange(subscriptionId, 2, notifications(2), uint(2)); + scriptable.enqueueDataChange(subscriptionId, 3, notifications(3), uint(3)); + + // Bounded: an implementation that does not dispatch the two completions through this executor + // simply proceeds in order, which is the behavior being asserted anyway. + executor.awaitReorder(REORDER_TIMEOUT_MILLIS); + + awaitResponsesProcessed(publishRequestsBefore, 2); + + scriptable.enqueueDataChange(subscriptionId, 4, notifications(4), uint(4)); + + assertTrue(awaitTrue(() -> deliveredValues.contains(4)), "sequence 4 was never delivered"); + } + + /** + * The Server sends NotificationMessage 3 before 2, then 4. Sequence 3 is a genuine gap and 2 is + * recovered from it; the copy of 2 that arrives afterwards has already been accounted for. + */ + void driveStaleResponse() throws Exception { + int publishRequestsBefore = scriptable.getPublishRequestCount(); + + scriptable.enqueueDataChange(subscriptionId, 3, notifications(3), uint(2), uint(3)); + + assertTrue(awaitTrue(() -> deliveredValues.contains(3)), "sequence 3 was never delivered"); + + scriptable.enqueueDataChange(subscriptionId, 2, notifications(2), uint(2)); + + awaitResponsesProcessed(publishRequestsBefore, 2); + + scriptable.enqueueDataChange(subscriptionId, 4, notifications(4), uint(4)); + + assertTrue(awaitTrue(() -> deliveredValues.contains(4)), "sequence 4 was never delivered"); + } + + /** + * Wait until {@code count} PublishResponses have been fully processed and delivered. + * + *

The client holds a fixed number of Publish requests in flight and only replaces one once + * the notifications it carried have been delivered — that is the SDK's backpressure mechanism — + * so {@code count} further Publish requests arriving at the Server is exactly that signal. + * + * @param publishRequestsBefore the Publish request count before the responses were scripted. + * @param count the number of PublishResponses to wait for. + */ + private void awaitResponsesProcessed(int publishRequestsBefore, int count) throws Exception { + assertTrue( + awaitTrue(() -> scriptable.getPublishRequestCount() >= publishRequestsBefore + count), + "the client did not finish processing " + count + " PublishResponses"); + } + + /** A DataChangeNotification carrying {@code value} for this Fixture's MonitoredItem. */ + List notifications(int value) { + return List.of( + new MonitoredItemNotification(clientHandle, new DataValue(Variant.ofInt32(value)))); + } + + private ExtensionObject[] encodedNotifications(int value) { + return new ExtensionObject[] { + scriptable.encode( + new DataChangeNotification( + notifications(value).toArray(MonitoredItemNotification[]::new), null)) + }; + } + + /** Polls {@code condition} until it holds or the (generous) timeout elapses. */ + boolean awaitTrue(ThrowingBooleanSupplier condition) throws Exception { + long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(AWAIT_TIMEOUT_MILLIS); + + while (System.nanoTime() < deadline) { + if (condition.get()) { + return true; + } + Thread.sleep(25); + } + + return condition.get(); + } + + @Override + public void close() throws Exception { + executor.releaseHeld(); + scriptable.failParkedRequests(StatusCodes.Bad_NoSubscription); + try { + client.disconnectAsync().get(5, TimeUnit.SECONDS); + } finally { + try { + server.shutdown().get(5, TimeUnit.SECONDS); + } finally { + executor.shutdownNow(); + } + } + } + } + + @FunctionalInterface + private interface ThrowingBooleanSupplier { + boolean get() throws Exception; + } + + /** + * The client's {@link ExecutorService}, with one test affordance: once {@link #armReorder()} has + * been called, the next task submitted from inside a task that was itself submitted from + * outside this executor is held; the following such task is run to completion; only then is the + * held task released. + * + *

That "one level in" position is exactly where the PublishResponse completion handlers land. + * {@code AbstractUascClientTransport} completes PublishResponse futures from a serial {@code + * ExecutionQueue} whose drain task runs on this executor (submitted from a Netty I/O thread, so + * it is at the outer level), and {@code PublishingManager}'s {@code whenCompleteAsync(..., + * executor)} dispatches the handler back onto this executor from there. Inverting those two + * dispatches is a scheduling decision an unordered pool is entitled to make; the pool the client + * uses by default — {@code Stack.sharedExecutor()}, whose shape this class copies — makes it by + * luck rather than on request. + * + *

The hold is bounded by {@link #awaitReorder(long)} and by {@link #releaseHeld()}, so an + * implementation that never dispatches a second such task is not deadlocked by the fixture. + */ + private static final class ReorderingExecutor extends ThreadPoolExecutor { + + /** Nesting level of the task the current thread is running; absent when it is not one. */ + private static final ThreadLocal DEPTH = new ThreadLocal<>(); + + private enum State { + PASS_THROUGH, + ARMED, + HOLDING + } + + private final Object lock = new Object(); + private final CountDownLatch reordered = new CountDownLatch(1); + + private State state = State.PASS_THROUGH; + private Runnable held; + + ReorderingExecutor() { + // The shape of Stack.sharedExecutor(): an unbounded cached pool, so nothing but this class's + // own bookkeeping orders the tasks that run on it. + super(0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS, new SynchronousQueue<>()); + } + + void armReorder() { + synchronized (lock) { + state = State.ARMED; + } + } + + /** + * Wait for the inversion to complete, then release the held task if a second one never arrived. + * + * @param timeoutMillis how long to wait for the inversion. + * @return {@code true} if two tasks were actually inverted. + * @throws InterruptedException if interrupted while waiting. + */ + boolean awaitReorder(long timeoutMillis) throws InterruptedException { + boolean applied = reordered.await(timeoutMillis, TimeUnit.MILLISECONDS); + + releaseHeld(); + + return applied; + } + + /** Release the held task, if any, without waiting for a second one. */ + void releaseHeld() { + Runnable toRelease; + + synchronized (lock) { + toRelease = held; + held = null; + state = State.PASS_THROUGH; + } + + if (toRelease != null) { + super.execute(toRelease); + } + } + + @Override + public void execute(Runnable command) { + Integer parentDepth = DEPTH.get(); + int depth = parentDepth == null ? 0 : parentDepth + 1; + + Runnable task = + () -> { + DEPTH.set(depth); + try { + command.run(); + } finally { + DEPTH.remove(); + } + }; + + if (depth == 1) { + synchronized (lock) { + if (state == State.ARMED) { + held = task; + state = State.HOLDING; + return; + } else if (state == State.HOLDING) { + Runnable first = held; + held = null; + state = State.PASS_THROUGH; + + super.execute( + () -> { + try { + task.run(); + } finally { + super.execute(first); + reordered.countDown(); + } + }); + return; + } + } + } + + super.execute(task); + } + } +} diff --git a/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/PublishScriptHarnessTest.java b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/PublishScriptHarnessTest.java new file mode 100644 index 0000000000..e54cba82c8 --- /dev/null +++ b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/PublishScriptHarnessTest.java @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2026 the Eclipse Milo Authors + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.milo.opcua.sdk.client.subscriptions; + +import static org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.Unsigned.uint; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import org.eclipse.milo.opcua.sdk.client.OpcUaClient; +import org.eclipse.milo.opcua.sdk.server.EndpointConfig; +import org.eclipse.milo.opcua.sdk.server.OpcUaServer; +import org.eclipse.milo.opcua.sdk.test.ScriptableSubscriptionServiceSet; +import org.eclipse.milo.opcua.sdk.test.TestClient; +import org.eclipse.milo.opcua.sdk.test.TestServer; +import org.eclipse.milo.opcua.stack.core.StatusCodes; +import org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.UInteger; +import org.junit.jupiter.api.Test; + +/** + * Exercises {@link ScriptableSubscriptionServiceSet} end-to-end against the real client stack. + * + *

This is a "does the harness work" smoke test, not a bug reproduction. It proves the seam can + * (a) deliver a scripted keep-alive to the client's {@code SubscriptionListener} and (b) capture + * the acknowledgements the client sends back after processing a scripted notification. The + * deterministic bug-reproduction tests built on this harness are enumerated in {@code + * docs/plans/publishing-manager-reliability-repair.md}. + */ +public class PublishScriptHarnessTest { + + @Test + void scriptedKeepAliveIsDeliveredAndNotificationIsAcknowledged() throws Exception { + TestServer testServer = TestServer.create(); + OpcUaServer server = testServer.getServer(); + + var scriptable = new ScriptableSubscriptionServiceSet(server); + for (EndpointConfig endpoint : server.getConfig().getEndpoints()) { + server.addServiceSet(endpoint.getPath(), scriptable); + } + + server.startup().get(); + + // Long request timeout so parked Publish requests do not time out during the test. + OpcUaClient client = TestClient.create(server, cfg -> cfg.setRequestTimeout(uint(60_000))); + client.connect(); + + try { + var keepAliveLatch = new CountDownLatch(1); + + var subscription = new OpcUaSubscription(client); + subscription.setSubscriptionListener( + new OpcUaSubscription.SubscriptionListener() { + @Override + public void onKeepAliveReceived(OpcUaSubscription subscription) { + keepAliveLatch.countDown(); + } + }); + subscription.create(); + + UInteger subscriptionId = subscription.getSubscriptionId().orElseThrow(); + + // (a) A scripted keep-alive (sequence 1) reaches the listener. + scriptable.enqueueKeepAlive(subscriptionId, 1); + assertTrue( + keepAliveLatch.await(5, TimeUnit.SECONDS), + "keep-alive was not delivered to the listener"); + + // (b) A scripted notification (sequence 2) advertising available sequence number 2 causes the + // client to acknowledge sequence 2 on a subsequent Publish request. + scriptable.enqueueDataChange(subscriptionId, 2, List.of(), uint(2)); + + boolean acknowledged = + awaitTrue( + () -> + scriptable.getReceivedAcknowledgements().stream() + .anyMatch( + ack -> + ack.getSubscriptionId().equals(subscriptionId) + && ack.getSequenceNumber().longValue() == 2L), + 5000); + + assertTrue(acknowledged, "client did not acknowledge sequence 2"); + } finally { + scriptable.failParkedRequests(StatusCodes.Bad_NoSubscription); + try { + client.disconnectAsync().get(2, TimeUnit.SECONDS); + } finally { + server.shutdown().get(2, TimeUnit.SECONDS); + } + } + } + + private static boolean awaitTrue(BooleanSupplierThrowing condition, long timeoutMillis) + throws Exception { + long deadline = System.currentTimeMillis() + timeoutMillis; + while (System.currentTimeMillis() < deadline) { + if (condition.get()) { + return true; + } + Thread.sleep(25); + } + return condition.get(); + } + + @FunctionalInterface + private interface BooleanSupplierThrowing { + boolean get() throws Exception; + } +} diff --git a/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/PublishSequenceRecoveryTest.java b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/PublishSequenceRecoveryTest.java new file mode 100644 index 0000000000..46d9e08bcc --- /dev/null +++ b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/PublishSequenceRecoveryTest.java @@ -0,0 +1,477 @@ +/* + * Copyright (c) 2026 the Eclipse Milo Authors + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.milo.opcua.sdk.client.subscriptions; + +import static org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.Unsigned.uint; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.eclipse.milo.opcua.sdk.client.OpcUaClient; +import org.eclipse.milo.opcua.sdk.server.EndpointConfig; +import org.eclipse.milo.opcua.sdk.server.OpcUaServer; +import org.eclipse.milo.opcua.sdk.test.ScriptableSubscriptionServiceSet; +import org.eclipse.milo.opcua.sdk.test.ScriptableSubscriptionServiceSet.RepublishResponder; +import org.eclipse.milo.opcua.sdk.test.TestClient; +import org.eclipse.milo.opcua.sdk.test.TestServer; +import org.eclipse.milo.opcua.stack.core.StatusCodes; +import org.eclipse.milo.opcua.stack.core.UaException; +import org.eclipse.milo.opcua.stack.core.types.builtin.DataValue; +import org.eclipse.milo.opcua.stack.core.types.builtin.ExtensionObject; +import org.eclipse.milo.opcua.stack.core.types.builtin.Variant; +import org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.UInteger; +import org.eclipse.milo.opcua.stack.core.types.structured.DataChangeNotification; +import org.eclipse.milo.opcua.stack.core.types.structured.MonitoredItemNotification; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +/** + * Sequence-number accounting in {@code PublishingManager.processPublishResponse}. + * + *

Part 4 §5.14.1.1 defines the NotificationMessage sequence number: "The value 0 is never used + * for the sequence number. The first NotificationMessage sent on a Subscription has a sequence + * number of 1. If the sequence number rolls over, it rolls over to 1." The same clause defines a + * keep-alive as a message that "does not contain any Notifications and ... contains the sequence + * number of the next NotificationMessage that is to be sent" — so a keep-alive carrying sequence + * {@code n} is positive evidence that {@code n} has not been sent yet, and is not + * evidence that {@code n - 1} was received. + * + *

{@code lastSequenceNumber} is the client's record of the last NotificationMessage actually + * accounted for; it is the sole input to gap detection and therefore to Republish recovery. Every + * test here drives real PublishResponses through the real client stack via {@link + * ScriptableSubscriptionServiceSet} and observes the Republish requests, {@code + * onNotificationDataLost} callbacks, and SubscriptionAcknowledgements that result. + */ +public class PublishSequenceRecoveryTest { + + /** The largest legal sequence number; the successor of this value is 1, never 0 (§5.14.1.1). */ + private static final long MAX_SEQUENCE_NUMBER = 0xFFFF_FFFFL; + + private TestServer testServer; + private OpcUaServer server; + private OpcUaClient client; + private ScriptableSubscriptionServiceSet scriptable; + private OpcUaSubscription subscription; + private UInteger subscriptionId; + + /** Every {@code retransmitSequenceNumber} the client has asked the Server to Republish. */ + private final List republishRequests = Collections.synchronizedList(new ArrayList<>()); + + private final AtomicInteger keepAliveCount = new AtomicInteger(); + private final AtomicInteger dataReceivedCount = new AtomicInteger(); + private final AtomicInteger notificationDataLostCount = new AtomicInteger(); + + @BeforeEach + void startClientAndServerAndCreateSubscription() throws Exception { + testServer = TestServer.create(); + server = testServer.getServer(); + + scriptable = new ScriptableSubscriptionServiceSet(server); + for (EndpointConfig endpoint : server.getConfig().getEndpoints()) { + server.addServiceSet(endpoint.getPath(), scriptable); + } + + server.startup().get(); + + // Long request timeout so parked Publish requests do not time out during the test. + client = TestClient.create(server, cfg -> cfg.setRequestTimeout(uint(60_000))); + client.connect(); + + subscription = new OpcUaSubscription(client); + subscription.setSubscriptionListener( + new OpcUaSubscription.SubscriptionListener() { + @Override + public void onKeepAliveReceived(OpcUaSubscription subscription) { + keepAliveCount.incrementAndGet(); + } + + @Override + public void onDataReceived( + OpcUaSubscription subscription, + List items, + List values) { + dataReceivedCount.incrementAndGet(); + } + + @Override + public void onNotificationDataLost(OpcUaSubscription subscription) { + notificationDataLostCount.incrementAndGet(); + } + }); + subscription.create(); + + subscriptionId = subscription.getSubscriptionId().orElseThrow(); + } + + @AfterEach + void stopClientAndServer() throws Exception { + scriptable.failParkedRequests(StatusCodes.Bad_NoSubscription); + try { + client.disconnectAsync().get(5, TimeUnit.SECONDS); + } finally { + server.shutdown().get(5, TimeUnit.SECONDS); + } + } + + /** + * Positive control for every "was Republish called for X?" assertion below: an ordinary gap + * between two data messages is detected today, proving the recording Republish responder + * and the Republish observation are wired up. If this test fails, the fixture is broken rather + * than the sequence accounting. + */ + @Test + void gapBetweenDataMessagesIsRepublished() throws Exception { + scriptable.setRepublishResponder(recordingRepublishResponder()); + + // Sequence 1 is received normally, sequence 2 is lost, sequence 3 arrives. + scriptable.enqueueDataChange(subscriptionId, 1, dataNotifications(), uint(1)); + assertTrue(awaitAcknowledgement(1), "sequence 1 was never acknowledged"); + + scriptable.enqueueDataChange(subscriptionId, 3, dataNotifications(), uint(1), uint(3)); + assertTrue(awaitAtLeast(dataReceivedCount, 2), "sequence 3 was never delivered"); + + assertEquals( + List.of(2L), + List.copyOf(republishRequests), + "the missing NotificationMessage 2 must be requested via Republish"); + } + + /** + * Finding 1: an initial keep-alive carrying sequence 1 is recorded as if NotificationMessage 1 + * had been received. + * + *

Part 4 §5.14.1.1: a keep-alive carries "the sequence number of the next NotificationMessage + * that is to be sent", so an initial keep-alive with sequence 1 means the first + * NotificationMessage has not yet been sent. Recording it as received means the loss of + * the real NotificationMessage 1 is never detected, and the client goes on to acknowledge it — + * which purges it from the Server's retransmission queue and makes it unrecoverable. + * + *

These tests deliberately re-create the interop scenario from #1401 (keep-alive 1 followed by + * first data 2). That commit added the {@code receivedSequenceNumber == 1} disjunct to suppress a + * Republish(1) it considered doomed; per §5.14.1.1 NotificationMessage 1 is genuinely missing in + * that trace and Republish(1) is the correct behavior. + */ + @Nested + class InitialKeepAlive { + + /** + * Keep-alive 1, then data 2: NotificationMessage 1 was sent and lost, so the client must try to + * recover it. + */ + @Test + void notificationMessageLostAfterAnInitialKeepAliveIsRepublished() throws Exception { + scriptable.setRepublishResponder(recordingRepublishResponder()); + + scriptable.enqueueKeepAlive(subscriptionId, 1); + assertTrue(awaitAtLeast(keepAliveCount, 1), "the initial keep-alive was never delivered"); + + // Sequence 1 was sent by the Server (it is still available for Republish) but never reached + // the client; sequence 2 is the next message to arrive. + scriptable.enqueueDataChange(subscriptionId, 2, dataNotifications(), uint(1), uint(2)); + assertTrue(awaitAtLeast(dataReceivedCount, 1), "sequence 2 was never delivered"); + + assertEquals( + List.of(1L), + List.copyOf(republishRequests), + "NotificationMessage 1 was lost and must be requested via Republish; an initial" + + " keep-alive carrying sequence 1 is not evidence that message 1 was received"); + } + + /** + * The aggravating half of finding 1: because the acknowledgement set is refilled verbatim from + * {@code availableSequenceNumbers}, the client acknowledges sequence 1 — a NotificationMessage + * it never received and never recovered. The Server then purges it from the retransmission + * queue and the data is gone for good. + * + *

The Republish attempt is scripted to fail with {@code Bad_MessageNotAvailable}, so nothing + * in this trace ever recovers sequence 1: acknowledging it is unconditionally wrong. + */ + @Test + void unreceivedNotificationMessageIsNotAcknowledged() throws Exception { + scriptable.setRepublishResponder(recordingRepublishResponder()); + + scriptable.enqueueKeepAlive(subscriptionId, 1); + assertTrue(awaitAtLeast(keepAliveCount, 1), "the initial keep-alive was never delivered"); + + scriptable.enqueueDataChange(subscriptionId, 2, dataNotifications(), uint(1), uint(2)); + + // Sequences 1 and 2 are both advertised as available, so both are acknowledged in the same + // PublishRequest: observing the acknowledgement of 2 is enough to decide about 1. + assertTrue(awaitAcknowledgement(2), "sequence 2 was never acknowledged"); + + assertFalse( + acknowledged(1), + "the client acknowledged NotificationMessage 1, which it never received and never" + + " recovered; the Server will purge it from the retransmission queue"); + } + } + + /** + * Finding 2: a gap that ends at a keep-alive advances {@code lastSequenceNumber} to the first + * missing sequence instead of to the last one, so the next keep-alive re-detects the tail of the + * same gap. + * + *

The gap loop bounds are correct, but the recovered range is not recorded: {@code + * lastSequenceNumber = expectedSequenceNumber} sets it to the first missing sequence, and + * the trailing {@code == 1 || !isKeepAlive} guard declines to correct it because this is a + * keep-alive. Each subsequent keep-alive therefore repeats the recovery for the remaining tail. + * Meanwhile the client has already acknowledged the whole range, so the Server has purged it and + * the repeat cycles fail — producing spurious {@code onNotificationDataLost} callbacks (and, if + * the acknowledgements have not landed yet, duplicate delivery to the application). + */ + @Nested + class GapEndingInAKeepAlive { + + /** The number of keep-alives carrying sequence 10 that the Server sends after the gap. */ + private static final int KEEP_ALIVE_ROUNDS = 4; + + private final List expectedAttempts = List.of(6L, 7L, 8L, 9L); + + /** + * Every missing NotificationMessage must be requested exactly once. Repeating the request is + * not merely wasteful: the client has already acknowledged 6-9, so the Server no longer holds + * them and every repeat is a blocking round-trip that can only fail. + */ + @Test + void eachMissingNotificationMessageIsRepublishedExactlyOnce() throws Exception { + driveGapEndingInKeepAlives(); + + assertEquals( + expectedAttempts, + List.copyOf(republishRequests), + "each NotificationMessage missing before the keep-alive must be requested exactly once"); + } + + /** + * {@code onNotificationDataLost} is the SDK's "your data is gone" signal. Recovering 6-9 + * successfully and then re-requesting them from a Server that has (correctly) purged them + * raises that alarm for data the application already received. + */ + @Test + void fullyRecoveredGapDoesNotReportNotificationDataLost() throws Exception { + driveGapEndingInKeepAlives(); + + assertEquals( + 0, + notificationDataLostCount.get(), + "every missing NotificationMessage was recovered, so no data was lost"); + } + + /** + * Drives {@code lastSequenceNumber} to 5, then delivers {@link #KEEP_ALIVE_ROUNDS} keep-alives + * carrying sequence 10 with 6-9 advertised as available. + * + *

The Republish responder models the Server's retransmission queue: each sequence can be + * republished once, after which it is gone (the client's own acknowledgements purge it). + */ + private void driveGapEndingInKeepAlives() throws Exception { + // Establish lastSequenceNumber = 5 with five in-order data messages, staying in lockstep with + // the responder queue by waiting for each acknowledgement before enqueueing the next. + for (long sequenceNumber = 1; sequenceNumber <= 5; sequenceNumber++) { + scriptable.enqueueDataChange( + subscriptionId, sequenceNumber, dataNotifications(), uint(sequenceNumber)); + + assertTrue( + awaitAcknowledgement(sequenceNumber), + "sequence " + sequenceNumber + " was never acknowledged"); + } + + assertEquals( + List.of(), + List.copyOf(republishRequests), + "control: five in-order data messages must not trigger any Republish"); + + scriptable.setRepublishResponder(retransmissionQueueResponder(Set.of(6L, 7L, 8L, 9L))); + + // Sequences 6-9 were sent and lost; the keep-alive announces that 10 is next. + for (int round = 1; round <= KEEP_ALIVE_ROUNDS; round++) { + scriptable.enqueueKeepAlive(subscriptionId, 10, uint(6), uint(7), uint(8), uint(9)); + + assertTrue( + awaitAtLeast(keepAliveCount, round), + "keep-alive round " + round + " was not delivered"); + } + } + } + + /** + * Finding 12: the sequence arithmetic is plain unwrapped {@code long} math, so it breaks at the + * UInt32 rollover boundary defined by Part 4 §5.14.1.1 ("If the sequence number rolls over, it + * rolls over to 1"). + * + *

White-box seeding. There is no wire path to a near-maximum {@code + * lastSequenceNumber}: it only ever advances one message at a time, and any forward jump large + * enough to reach the boundary would spin the (unbounded) Republish loop billions of times. These + * tests therefore seed {@code lastSequenceNumber} reflectively and then drive real + * PublishResponses through the real client stack, so everything after the seed — gap detection, + * Republish recovery, delivery — is the production code path. + */ + @Nested + class SequenceNumberRollover { + + /** + * The message before the rollover is lost. {@code expected = last + 1} is the last legal + * sequence number, the wrapped sequence 1 compares as "behind" it, and the gap is silently + * swallowed. + */ + @Test + void notificationMessageLostAtTheRolloverBoundaryIsRepublished() throws Exception { + scriptable.setRepublishResponder(recordingRepublishResponder()); + seedLastSequenceNumber(MAX_SEQUENCE_NUMBER - 1); + + // MAX_SEQUENCE_NUMBER was sent and lost; the sequence rolls over to 1. + scriptable.enqueueDataChange( + subscriptionId, 1, dataNotifications(), uint(MAX_SEQUENCE_NUMBER), uint(1)); + assertTrue(awaitAtLeast(dataReceivedCount, 1), "the wrapped sequence 1 was never delivered"); + + assertEquals( + List.of(MAX_SEQUENCE_NUMBER), + List.copyOf(republishRequests), + "the NotificationMessage lost at the rollover boundary must be requested via Republish"); + } + + /** + * A keep-alive arriving immediately after the rollover. {@code expected = last + 1} is + * 4294967296 — a value no NotificationMessage can carry — so no received sequence number can + * ever again compare as "ahead" and gap detection is dead from here on. + */ + @Test + void keepAliveAfterRolloverDetectsTheMissingNotificationMessage() throws Exception { + scriptable.setRepublishResponder(recordingRepublishResponder()); + seedLastSequenceNumber(MAX_SEQUENCE_NUMBER); + + // The wrapped sequence 1 was sent and lost; the keep-alive announces that 2 is next. + scriptable.enqueueKeepAlive(subscriptionId, 2, uint(1)); + assertTrue(awaitAtLeast(keepAliveCount, 1), "the keep-alive was never delivered"); + + assertEquals( + List.of(1L), + List.copyOf(republishRequests), + "the wrapped NotificationMessage 1 was lost and must be requested via Republish"); + } + } + + // region fixture helpers + + /** Records the requested sequence number and reports that the Server no longer holds it. */ + private RepublishResponder recordingRepublishResponder() { + return request -> { + republishRequests.add(request.getRetransmitSequenceNumber().longValue()); + + throw new UaException(StatusCodes.Bad_MessageNotAvailable); + }; + } + + /** + * Models a Server retransmission queue: each sequence in {@code held} can be republished once, + * after which it is gone — the same effect the client's own acknowledgements have. + */ + private RepublishResponder retransmissionQueueResponder(Set held) { + Set remaining = Collections.synchronizedSet(new HashSet<>(held)); + + return request -> { + long sequenceNumber = request.getRetransmitSequenceNumber().longValue(); + republishRequests.add(sequenceNumber); + + if (remaining.remove(sequenceNumber)) { + return scriptable.buildRepublishResponse( + request, sequenceNumber, encodedDataNotifications()); + } else { + throw new UaException(StatusCodes.Bad_MessageNotAvailable); + } + }; + } + + /** + * A single MonitoredItemNotification. The ClientHandle is not registered with the Subscription, + * which is irrelevant here: what matters is that the NotificationMessage carries notification + * data, making it a data message rather than a keep-alive, and that delivery reaches {@code + * onDataReceived}. + */ + private static List dataNotifications() { + return List.of(new MonitoredItemNotification(uint(1), new DataValue(new Variant(0)))); + } + + private ExtensionObject[] encodedDataNotifications() { + return new ExtensionObject[] { + scriptable.encode( + new DataChangeNotification( + dataNotifications().toArray(MonitoredItemNotification[]::new), null)) + }; + } + + /** + * Seeds the {@code lastSequenceNumber} the {@code PublishingManager} tracks for this + * Subscription. Called before any PublishResponse has been processed, so nothing races with it. + */ + private void seedLastSequenceNumber(long sequenceNumber) throws Exception { + PublishingManager publishingManager = client.getPublishingManager(); + + Field subscriptionDetailsField = + PublishingManager.class.getDeclaredField("subscriptionDetails"); + subscriptionDetailsField.setAccessible(true); + + Map subscriptionDetails = (Map) subscriptionDetailsField.get(publishingManager); + Object details = subscriptionDetails.get(subscriptionId); + assertNotNull(details, "the Subscription is not registered with the PublishingManager"); + + Field lastSequenceNumberField = details.getClass().getDeclaredField("lastSequenceNumber"); + lastSequenceNumberField.setAccessible(true); + lastSequenceNumberField.setLong(details, sequenceNumber); + } + + private boolean acknowledged(long sequenceNumber) { + return scriptable.getReceivedAcknowledgements().stream() + .anyMatch( + ack -> + ack.getSubscriptionId().equals(subscriptionId) + && ack.getSequenceNumber().longValue() == sequenceNumber); + } + + private boolean awaitAcknowledgement(long sequenceNumber) throws Exception { + return awaitTrue(() -> acknowledged(sequenceNumber)); + } + + private static boolean awaitAtLeast(AtomicInteger counter, int count) throws Exception { + return awaitTrue(() -> counter.get() >= count); + } + + /** Polls {@code condition} until it holds or the (generous) timeout elapses. */ + private static boolean awaitTrue(ThrowingBooleanSupplier condition) throws Exception { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (System.nanoTime() < deadline) { + if (condition.get()) { + return true; + } + Thread.sleep(25); + } + return condition.get(); + } + + @FunctionalInterface + private interface ThrowingBooleanSupplier { + boolean get() throws Exception; + } + + // endregion +} diff --git a/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/PublishSequenceRegressionTest.java b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/PublishSequenceRegressionTest.java new file mode 100644 index 0000000000..2cc221f820 --- /dev/null +++ b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/PublishSequenceRegressionTest.java @@ -0,0 +1,380 @@ +/* + * Copyright (c) 2026 the Eclipse Milo Authors + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.milo.opcua.sdk.client.subscriptions; + +import static org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.Unsigned.uint; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.eclipse.milo.opcua.sdk.client.OpcUaClient; +import org.eclipse.milo.opcua.sdk.server.EndpointConfig; +import org.eclipse.milo.opcua.sdk.server.OpcUaServer; +import org.eclipse.milo.opcua.sdk.test.ScriptableSubscriptionServiceSet; +import org.eclipse.milo.opcua.sdk.test.TestClient; +import org.eclipse.milo.opcua.sdk.test.TestServer; +import org.eclipse.milo.opcua.stack.core.NodeIds; +import org.eclipse.milo.opcua.stack.core.StatusCodes; +import org.eclipse.milo.opcua.stack.core.UaException; +import org.eclipse.milo.opcua.stack.core.types.builtin.DataValue; +import org.eclipse.milo.opcua.stack.core.types.builtin.Variant; +import org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.UInteger; +import org.eclipse.milo.opcua.stack.core.types.structured.MonitoredItemNotification; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * What {@code PublishingManager.processPublishResponse} does with a NotificationMessage whose + * sequence number is behind the one it expects next. + * + *

Two different things can produce such a message, and they need opposite handling: + * + *

+ * + *

The two are told apart by distance. A message no further behind than the larger of the + * retransmission queue the Server is advertising and {@code + * PublishingManager.DEFAULT_MAX_RECOVERABLE_GAP} ({@value #DUPLICATE_WINDOW}) is within + * retransmission range and is treated as a duplicate; anything further behind than that cannot be a + * duplicate of anything the Server still holds, and is delivered and resynchronized to. + * + *

This is the boundary 93bedb32c introduced. Before it, everything behind the expected + * sequence number was discarded, which for a genuine renumbering meant silently and indefinitely. + * The tests below drive real PublishResponses through the real client stack via {@link + * ScriptableSubscriptionServiceSet} and observe which NotificationMessages reach the application. + */ +public class PublishSequenceRegressionTest { + + /** + * {@code PublishingManager.DEFAULT_MAX_RECOVERABLE_GAP}: the floor on how far behind the expected + * sequence number a NotificationMessage may be and still be taken for a duplicate. + */ + private static final int DUPLICATE_WINDOW = 64; + + /** How long to wait for a NotificationMessage that must be delivered. */ + private static final long DELIVERY_WINDOW_MILLIS = 5_000; + + /** + * Long enough that nothing times out on its own, so a parked Publish request stays parked and any + * failure observed below is scripted rather than incidental. + */ + private static final long REQUEST_TIMEOUT_MILLIS = 60_000; + + private TestServer testServer; + private OpcUaServer server; + private OpcUaClient client; + private ScriptableSubscriptionServiceSet scriptable; + private OpcUaSubscription subscription; + private UInteger subscriptionId; + private UInteger clientHandle; + + /** + * The Int32 payload of every DataChangeNotification the application has been handed, in delivery + * order. Each scripted NotificationMessage carries its own sequence number as its value, so this + * says exactly which messages were delivered and in which order — a count alone could not + * distinguish "the regressed message was delivered" from "a later one was". + */ + private final List deliveredValues = Collections.synchronizedList(new ArrayList<>()); + + /** Every {@code retransmitSequenceNumber} the client has asked the Server to Republish. */ + private final List republishRequests = Collections.synchronizedList(new ArrayList<>()); + + private final AtomicInteger notificationDataLostCount = new AtomicInteger(); + + @BeforeEach + void startClientAndServerAndCreateSubscription() throws Exception { + testServer = TestServer.create(); + server = testServer.getServer(); + + scriptable = new ScriptableSubscriptionServiceSet(server); + for (EndpointConfig endpoint : server.getConfig().getEndpoints()) { + server.addServiceSet(endpoint.getPath(), scriptable); + } + + // Records the requested sequence number and reports that the Server no longer holds it: no test + // here expects a Republish, so any request one of them provokes is recorded and asserted about. + scriptable.setRepublishResponder( + request -> { + republishRequests.add(request.getRetransmitSequenceNumber().longValue()); + + throw new UaException(StatusCodes.Bad_MessageNotAvailable); + }); + + server.startup().get(); + + client = TestClient.create(server, cfg -> cfg.setRequestTimeout(uint(REQUEST_TIMEOUT_MILLIS))); + client.connect(); + + subscription = new OpcUaSubscription(client); + subscription.setSubscriptionListener( + new OpcUaSubscription.SubscriptionListener() { + @Override + public void onDataReceived( + OpcUaSubscription subscription, + List items, + List values) { + + values.forEach(value -> deliveredValues.add((Integer) value.value().value())); + } + + @Override + public void onNotificationDataLost(OpcUaSubscription subscription) { + notificationDataLostCount.incrementAndGet(); + } + }); + subscription.create(); + + subscriptionId = subscription.getSubscriptionId().orElseThrow(); + + // Client-side only: addMonitoredItem assigns the ClientHandle the notification fan-out looks + // notifications up by, and no Server-side item takes part in delivering a scripted one. + var item = OpcUaMonitoredItem.newDataItem(NodeIds.Server_ServerStatus_CurrentTime); + subscription.addMonitoredItem(item); + clientHandle = item.getClientHandle().orElseThrow(); + } + + @AfterEach + void stopClientAndServer() throws Exception { + scriptable.failParkedRequests(StatusCodes.Bad_NoSubscription); + try { + client.disconnectAsync().get(5, TimeUnit.SECONDS); + } finally { + server.shutdown().get(5, TimeUnit.SECONDS); + } + } + + /** + * The defect. A NotificationMessage far behind the expected sequence number is the Server's + * numbering having regressed, so it must be delivered and resynchronized to — and the messages + * that follow it in the new numbering must be delivered as well, which is the part that + * makes the difference between one dropped message and a Subscription that never delivers + * anything again. + * + *

Nothing here should provoke a Republish: the received sequence number is not ahead of the + * expected one, so there is no gap to repair, and a resynchronization that reported a gap would + * ask the Server to retransmit billions of NotificationMessages. + */ + @Test + void regressedNumberingIsDeliveredAndTheNewNumberingContinues() throws Exception { + seedLastSequenceNumber(1000); + + // 1000 behind the expected 1001, so far outside the duplicate window that no retransmission + // queue could hold the messages in between: the Server has started numbering again from 1. + sendDataChange(1, uint(1)); + + assertTrue( + awaitDelivered(1), + "the NotificationMessage carrying sequence 1 was never delivered. It is 1000 behind the" + + " expected 1001 — far beyond the " + + DUPLICATE_WINDOW + + " that could be a duplicate — so it is the Server's numbering having regressed, and" + + " discarding it drops every NotificationMessage until the numbering catches back up," + + " which for a renumbered Subscription is never"); + + sendDataChange(2, uint(1), uint(2)); + sendDataChange(3, uint(1), uint(2), uint(3)); + + assertTrue( + awaitTrue(() -> deliveredValues.size() >= 3), + "the NotificationMessages that followed the regressed one were not delivered: the" + + " accounting was not resynchronized to it, so they are all behind" + + " lastSequenceNumber too"); + + assertEquals( + List.of(1, 2, 3), + List.copyOf(deliveredValues), + "every NotificationMessage of the Server's new numbering must be delivered exactly once and" + + " in order"); + + assertEquals( + List.of(), + List.copyOf(republishRequests), + "resynchronizing to a regressed sequence number must not report a gap: the received" + + " sequence number is behind the expected one, not ahead of it"); + + assertEquals( + 0, + notificationDataLostCount.get(), + "no NotificationMessage was lost: the Server renumbered, it did not drop anything"); + } + + /** + * Control for the test above, at the exact boundary: a NotificationMessage {@value + * #DUPLICATE_WINDOW} behind the expected one is still within retransmission range, so it is still + * a duplicate and must still be discarded. Without this the fix could have turned every duplicate + * into a resynchronization, which rolls {@code lastSequenceNumber} backwards and makes every + * message already received after it look missing. + */ + @Test + void duplicateAtTheEdgeOfTheDuplicateWindowIsStillDiscarded() throws Exception { + seedLastSequenceNumber(100); + + // Exactly DUPLICATE_WINDOW behind the expected 101. + long duplicate = 101 - DUPLICATE_WINDOW; + sendDataChange(duplicate, uint(duplicate)); + + // The next message in the established numbering, which is delivered: the serial processing + // queue means observing this one is proof the duplicate ahead of it has been dealt with. + sendDataChange(101, uint(duplicate), uint(101)); + + assertTrue(awaitDelivered(101), "the expected NotificationMessage 101 was never delivered"); + + assertEquals( + List.of(101), + List.copyOf(deliveredValues), + "the NotificationMessage " + + DUPLICATE_WINDOW + + " behind the expected one is within the Server's retransmission range, so it is a" + + " duplicate and must not be delivered a second time"); + } + + /** + * The other side of the same boundary: one step further behind than a duplicate can be, so it is + * the Server's numbering having regressed and is delivered. Together with the test above this + * pins where the boundary is, not merely that there is one. + */ + @Test + void numberingRegressedJustBeyondTheDuplicateWindowIsDelivered() throws Exception { + seedLastSequenceNumber(100); + + long regressed = 101 - (DUPLICATE_WINDOW + 1); + sendDataChange(regressed, uint(regressed)); + + assertTrue( + awaitDelivered((int) regressed), + "a NotificationMessage " + + (DUPLICATE_WINDOW + 1) + + " behind the expected 101 is further behind than any duplicate can be — beyond both" + + " the advertised retransmission queue and DEFAULT_MAX_RECOVERABLE_GAP — so it is a" + + " regression in the Server's numbering and must be delivered"); + + assertEquals( + List.of((int) regressed), + List.copyOf(deliveredValues), + "exactly the regressed NotificationMessage must be delivered"); + } + + /** + * The same control as {@link #duplicateAtTheEdgeOfTheDuplicateWindowIsStillDiscarded} on an + * ordinary trace with no seeded state: a retransmitted copy of a message received a moment ago is + * one step behind, and is discarded. This is also the positive control for the fixture — it + * proves the scripted notifications, the ClientHandle and the delivery observation are wired up + * without any reflection involved. + */ + @Test + void duplicateInAnOrdinaryTraceIsStillDiscarded() throws Exception { + for (long sequenceNumber = 1; sequenceNumber <= 3; sequenceNumber++) { + sendDataChange(sequenceNumber, uint(sequenceNumber)); + + assertTrue( + awaitDelivered((int) sequenceNumber), + "sequence " + sequenceNumber + " was never delivered"); + } + + // A retransmitted copy of NotificationMessage 2, which the client has already accounted for. + sendDataChange(2, uint(1), uint(2), uint(3)); + sendDataChange(4, uint(1), uint(2), uint(3), uint(4)); + + assertTrue(awaitDelivered(4), "sequence 4 was never delivered"); + + assertEquals( + List.of(1, 2, 3, 4), + List.copyOf(deliveredValues), + "the retransmitted copy of NotificationMessage 2 must not be delivered a second time"); + } + + // region fixture helpers + + /** + * Enqueue a PublishResponse carrying a DataChangeNotification whose value is {@code + * sequenceNumber}, so the delivered value identifies the NotificationMessage it came from. + */ + private void sendDataChange(long sequenceNumber, UInteger... available) { + scriptable.enqueueDataChange( + subscriptionId, + sequenceNumber, + List.of( + new MonitoredItemNotification( + clientHandle, new DataValue(Variant.ofInt32((int) sequenceNumber)))), + available); + } + + /** + * Seeds the {@code lastSequenceNumber} the {@code PublishingManager} tracks for this + * Subscription. + * + *

White-box seeding. {@code lastSequenceNumber} only ever advances one + * NotificationMessage at a time, so there is no wire path to a value hundreds ahead of where the + * Server's numbering will restart: reaching it legitimately would mean scripting that many + * responses, and reaching it by a forward jump would spin the Republish loop instead. The same + * technique {@code PublishSequenceRecoveryTest} uses for the rollover boundary. Everything after + * the seed is the production code path: real PublishResponses over a real connection, and the + * real accounting, gap detection and delivery. Called before any PublishResponse has been + * processed, so nothing races with it. + */ + private void seedLastSequenceNumber(long sequenceNumber) throws Exception { + PublishingManager publishingManager = client.getPublishingManager(); + + Field subscriptionDetailsField = + PublishingManager.class.getDeclaredField("subscriptionDetails"); + subscriptionDetailsField.setAccessible(true); + + Map subscriptionDetails = (Map) subscriptionDetailsField.get(publishingManager); + Object details = subscriptionDetails.get(subscriptionId); + assertNotNull(details, "the Subscription is not registered with the PublishingManager"); + + Field lastSequenceNumberField = details.getClass().getDeclaredField("lastSequenceNumber"); + lastSequenceNumberField.setAccessible(true); + lastSequenceNumberField.setLong(details, sequenceNumber); + } + + private boolean awaitDelivered(int value) throws Exception { + return awaitTrue(() -> deliveredValues.contains(value)); + } + + /** Polls {@code condition} until it holds or {@link #DELIVERY_WINDOW_MILLIS} elapses. */ + private static boolean awaitTrue(ThrowingBooleanSupplier condition) throws Exception { + long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(DELIVERY_WINDOW_MILLIS); + + while (System.nanoTime() < deadline) { + if (condition.get()) { + return true; + } + Thread.sleep(25); + } + + return condition.get(); + } + + @FunctionalInterface + private interface ThrowingBooleanSupplier { + boolean get() throws Exception; + } + + // endregion +} diff --git a/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/PublishSuspensionGateRaceTest.java b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/PublishSuspensionGateRaceTest.java new file mode 100644 index 0000000000..42bd598740 --- /dev/null +++ b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/PublishSuspensionGateRaceTest.java @@ -0,0 +1,731 @@ +/* + * Copyright (c) 2026 the Eclipse Milo Authors + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.milo.opcua.sdk.client.subscriptions; + +import static org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.Unsigned.uint; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import org.eclipse.milo.opcua.sdk.client.OpcUaClient; +import org.eclipse.milo.opcua.sdk.client.SessionActivityListener; +import org.eclipse.milo.opcua.sdk.client.UaSession; +import org.eclipse.milo.opcua.sdk.server.EndpointConfig; +import org.eclipse.milo.opcua.sdk.server.OpcUaServer; +import org.eclipse.milo.opcua.sdk.test.DelegatingSessionServiceSet; +import org.eclipse.milo.opcua.sdk.test.ScriptableSubscriptionServiceSet; +import org.eclipse.milo.opcua.sdk.test.TestClient; +import org.eclipse.milo.opcua.sdk.test.TestServer; +import org.eclipse.milo.opcua.stack.core.NodeIds; +import org.eclipse.milo.opcua.stack.core.StatusCodes; +import org.eclipse.milo.opcua.stack.core.UaException; +import org.eclipse.milo.opcua.stack.core.types.builtin.DataValue; +import org.eclipse.milo.opcua.stack.core.types.builtin.DateTime; +import org.eclipse.milo.opcua.stack.core.types.builtin.ExtensionObject; +import org.eclipse.milo.opcua.stack.core.types.builtin.StatusCode; +import org.eclipse.milo.opcua.stack.core.types.builtin.Variant; +import org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.UInteger; +import org.eclipse.milo.opcua.stack.core.types.structured.ActivateSessionRequest; +import org.eclipse.milo.opcua.stack.core.types.structured.ActivateSessionResponse; +import org.eclipse.milo.opcua.stack.core.types.structured.DataChangeNotification; +import org.eclipse.milo.opcua.stack.core.types.structured.MonitoredItemNotification; +import org.eclipse.milo.opcua.stack.core.types.structured.PublishRequest; +import org.eclipse.milo.opcua.stack.core.types.structured.PublishResponse; +import org.eclipse.milo.opcua.stack.core.types.structured.RepublishRequest; +import org.eclipse.milo.opcua.stack.core.types.structured.RepublishResponse; +import org.eclipse.milo.opcua.stack.core.types.structured.ResponseHeader; +import org.eclipse.milo.opcua.stack.core.types.structured.TransferResult; +import org.eclipse.milo.opcua.stack.core.types.structured.TransferSubscriptionsRequest; +import org.eclipse.milo.opcua.stack.core.types.structured.TransferSubscriptionsResponse; +import org.eclipse.milo.opcua.stack.transport.server.ServiceRequestContext; +import org.junit.jupiter.api.Test; + +/** + * The window between the Session future completing and the {@code onSessionActive} callbacks + * running, measured against the Part 4 §6.7 ordering the Publish suspension gate exists to impose. + * + *

Part 4 §6.7: "After re-establishing the connection the Client shall call Republish in a + * loop... After the Republish returns Bad_MessageNotAvailable the Client shall start sending + * Publish requests with the normal Publish handling." {@link PublishReconnectRecoveryTest} asserts + * that order for the ordinary caller — one that asks for a Session after the reconnect is over. + * This class covers the caller that was already there. + * + *

{@code SessionFsmFactory} completes the Session future in one executor task and fans the + * {@code onSessionActive} callbacks out in another, submitted after it. Every continuation parked + * on {@code getSessionAsync()} therefore runs before the {@code PublishingManager} has been + * told the Session is Active. A suspension gate that asks only "has every Session activation + * counted so far had its recovery run?" is open in that window, because the new activation has not + * been counted yet and the answer still describes the previous one — and a PublishRequest goes out + * on the new Session ahead of the Republish loop, which is precisely what the ordering forbids. + * Counting the activation earlier does not close it either: a re-activation hands back the same + * {@link UaSession} object, so the gate has to know both which activation was recovered and which + * Session that recovery ran on, and it has to stop trusting the latter the moment that Session goes + * away. + * + *

The interleaving is forced rather than raced. The client's transport executor has a single + * thread, so the task that completes the Session future runs to completion — the continuations + * parked on it included — before the task that delivers the activation callbacks starts. The parked + * caller is a real one: a PublishRequest returned with a failure while the Session is down makes + * {@code PublishingManager} try to replace it, find no Session, and wait for the one being + * established. + */ +public class PublishSuspensionGateRaceTest { + + /** Log entry written when a PublishRequest reaches the Server. */ + private static final String PUBLISH = "Publish"; + + /** Prefix of the log entry written when a RepublishRequest reaches the Server. */ + private static final String REPUBLISH = "Republish:"; + + /** Recorded when the Session future completes, by a caller parked on it. */ + private static final String SESSION_FUTURE = "SessionFuture"; + + /** Recorded when the {@code onSessionActive} callbacks run. */ + private static final String SESSION_ACTIVE = "SessionActive"; + + /** + * The sequence number of the last NotificationMessage the client accounts for before the Session + * fault. + */ + private static final long LAST_SEQUENCE_NUMBER_BEFORE_FAULT = 2; + + /** The sequence number the Republish loop of Part 4 §6.7 has to start from. */ + private static final long NEXT_EXPECTED_SEQUENCE_NUMBER = LAST_SEQUENCE_NUMBER_BEFORE_FAULT + 1; + + /** How long to wait for something that must happen. */ + private static final long AWAIT_TIMEOUT_MILLIS = 10_000; + + /** + * How long to wait for a reconnect. The Session FSM waits one second in {@code ReactivatingWait} + * before its first re-activation attempt, another second in {@code CreatingWait} before creating + * a replacement Session, and doubles each wait on every failure. + */ + private static final long RECONNECT_TIMEOUT_MILLIS = 30_000; + + /** + * Long enough that nothing times out on its own: no parked Publish request, no Republish, and no + * Session keep-alive. Every ordering asserted against below is therefore the client's own doing. + */ + private static final long REQUEST_TIMEOUT_MILLIS = 60_000; + + private static final long DISCONNECT_TIMEOUT_MILLIS = 5_000; + + /** Upper bound on how long a held ActivateSession is held, so nothing hangs indefinitely. */ + private static final long GATE_TIMEOUT_MILLIS = 30_000; + + /** + * The precondition the two tests below rest on, asserted on its own so that a failure there is + * unambiguous: a caller that parks on {@code getSessionAsync()} while the Session is down is + * released before any {@code onSessionActive} callback runs, and therefore before {@code + * PublishingManager} has counted the activation or started its Republish loop. + */ + @Test + void aCallerParkedOnTheSessionFutureRunsBeforeTheActivationCallbacks() throws Exception { + try (var fixture = new Fixture()) { + fixture.deliverInitialNotifications(); + + fixture.faultSession(); + fixture.awaitSessionInactive(); + + fixture.parkAnObserverOnTheSessionFuture(); + + fixture.awaitReactivation(); + + assertEquals( + List.of(SESSION_FUTURE, SESSION_ACTIVE), + fixture.activationOrder(), + "the Session future must complete before the activation callbacks run; if it does not," + + " there is no window for a parked caller to send a PublishRequest in and the" + + " ordering assertions in this class prove nothing"); + } + } + + /** + * The same-Session reconnect path: {@code Active -> ReactivatingWait -> Reactivating -> + * Initializing -> Active}. The FSM hands back the very same {@link UaSession} object, so a gate + * that remembers which Session the last finished recovery ran on still matches it — unless that + * record is revoked when the Session becomes inactive. + * + *

Whatever the gate is keyed on, the first request the client sends for this Subscription once + * the Session is Active again must be the Republish that Part 4 §6.7 requires, even though a + * caller was already parked on the Session future when it completed. + */ + @Test + void noPublishPrecedesTheRepublishDrainWhenACallerIsParkedOnAReactivatedSession() + throws Exception { + + try (var fixture = new Fixture()) { + fixture.deliverInitialNotifications(); + + fixture.armRequestLog(); + fixture.faultSession(); + fixture.awaitSessionInactive(); + fixture.parkAPublishRefillOnTheSessionFuture(); + fixture.awaitReactivation(); + + assertTrue( + fixture.awaitRequestLogEntries(1), + "no request at all reached the Server after the Session was re-activated"); + + assertEquals( + REPUBLISH + NEXT_EXPECTED_SEQUENCE_NUMBER, + fixture.requestLog().get(0), + "the first request sent after re-activation must be the Republish Part 4 §6.7 requires," + + " but a PublishRequest overtook it: the caller parked on the Session future was" + + " released before the activation was counted, and the suspension gate answered it" + + " with the previous activation's recovery; request log: " + + fixture.requestLog()); + } + } + + /** + * The replacement-Session reconnect path: re-activation is refused, so the client creates a new + * Session and transfers the Subscription to it. Here the {@link UaSession} handed to the parked + * caller is a different object from the one the last finished recovery ran on, so this is the + * half of the gate that the Session identity answers rather than the revocation. + * + *

The caller is parked while the replacement Session's ActivateSession is held at the Server, + * because the Session future a caller parks on during {@code ReactivatingWait} is discarded — + * never completed — when re-activation is refused, and a caller parked on that one is never + * released at all. + */ + @Test + void noPublishPrecedesTheRepublishDrainWhenACallerIsParkedOnAReplacementSession() + throws Exception { + + try (var fixture = new Fixture()) { + fixture.deliverInitialNotifications(); + fixture.advertiseNothingOnTransfer(); + fixture.refuseNextReactivationAndHoldTheReplacement(); + + fixture.armRequestLog(); + fixture.faultSession(); + fixture.awaitReplacementActivateSessionHeld(); + fixture.parkAPublishRefillOnTheSessionFuture(); + fixture.releaseReplacementActivateSession(); + fixture.awaitReactivation(); + + assertTrue( + fixture.awaitRequestLogEntries(1), + "no request at all reached the Server after the Subscription was transferred"); + assertTrue( + fixture.transferCount() >= 1, + "precondition: the reconnect did not go through TransferSubscriptions, so this test is" + + " not exercising the replacement-Session path at all"); + + assertEquals( + REPUBLISH + NEXT_EXPECTED_SEQUENCE_NUMBER, + fixture.requestLog().get(0), + "the first request sent after the transfer must be the Republish Part 4 §6.7 requires," + + " but a PublishRequest overtook it: the caller parked on the Session future was" + + " released before the activation was counted, and the suspension gate answered it" + + " with the previous Session's recovery; request log: " + + fixture.requestLog()); + } + } + + // region fixture + + /** + * The order in which Publish and Republish requests reach the Server. + * + *

Recording starts only when the log is {@linkplain #arm() armed}, which a test does once its + * Publish pipeline is quiescent — every request parked at the Server and no responder scripted + * for any of them. From that moment the only requests that can reach the Server are the ones the + * client sends after the reconnect. + */ + private static final class RequestLog { + + private final List entries = Collections.synchronizedList(new ArrayList<>()); + + private volatile boolean armed = false; + + void arm() { + entries.clear(); + armed = true; + } + + void record(String entry) { + if (armed) { + entries.add(entry); + } + } + + List entries() { + return List.copyOf(entries); + } + } + + /** + * A {@link ScriptableSubscriptionServiceSet} that records the arrival of every PublishRequest and + * can answer TransferSubscriptions with a scripted availableSequenceNumbers list. + */ + private static final class LoggingSubscriptionServiceSet + extends ScriptableSubscriptionServiceSet { + + private final AtomicBoolean transferScripted = new AtomicBoolean(false); + private volatile int transferCount = 0; + + private final RequestLog requestLog; + + LoggingSubscriptionServiceSet(OpcUaServer server, RequestLog requestLog) { + super(server); + + this.requestLog = requestLog; + } + + @Override + public CompletableFuture onPublish( + ServiceRequestContext context, PublishRequest request) { + + requestLog.record(PUBLISH); + + return super.onPublish(context, request); + } + + @Override + public TransferSubscriptionsResponse onTransferSubscriptions( + ServiceRequestContext context, TransferSubscriptionsRequest request) throws UaException { + + if (!transferScripted.get()) { + return super.onTransferSubscriptions(context, request); + } + + transferCount++; + + UInteger[] subscriptionIds = request.getSubscriptionIds(); + int count = subscriptionIds != null ? subscriptionIds.length : 0; + + var results = new TransferResult[count]; + for (int i = 0; i < count; i++) { + // Part 4 §5.14.7.1: the Server is holding nothing for retransmission, so the Republish loop + // of §6.7 is the increment-until-Bad_MessageNotAvailable form. + results[i] = new TransferResult(StatusCode.GOOD, new UInteger[0]); + } + + var responseHeader = + new ResponseHeader( + DateTime.now(), + request.getRequestHeader().getRequestHandle(), + StatusCode.GOOD, + null, + null, + null); + + return new TransferSubscriptionsResponse(responseHeader, results, null); + } + } + + /** + * A {@link DelegatingSessionServiceSet} that can refuse a single ActivateSession with a + * ServiceFault — which is what drives the Session FSM off the re-activation path and onto the + * create-a-new-Session-and-transfer path — and then hold the next one until the test releases it. + */ + private static final class ScriptedSessionServiceSet extends DelegatingSessionServiceSet { + + private final AtomicBoolean refuseNext = new AtomicBoolean(false); + private final AtomicBoolean holdNext = new AtomicBoolean(false); + + private final CountDownLatch heldActivateSession = new CountDownLatch(1); + private final CountDownLatch activateSessionGate = new CountDownLatch(1); + + ScriptedSessionServiceSet(OpcUaServer server) { + super(server); + } + + @Override + public ActivateSessionResponse onActivateSession( + ServiceRequestContext context, ActivateSessionRequest request) throws UaException { + + if (refuseNext.compareAndSet(true, false)) { + throw new UaException(StatusCodes.Bad_SessionIdInvalid); + } + + if (holdNext.compareAndSet(true, false)) { + heldActivateSession.countDown(); + + try { + if (!activateSessionGate.await(GATE_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { + throw new UaException(StatusCodes.Bad_Timeout); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new UaException(StatusCodes.Bad_UnexpectedError, e); + } + } + + return super.onActivateSession(context, request); + } + } + + /** + * A running Server whose Publish, Republish and TransferSubscriptions responses are scripted, and + * a connected client with one Subscription carrying one MonitoredItem. + * + *

The client's transport executor has a single thread. That is what makes the window this + * class is about deterministic: the Session FSM submits the task that completes the Session + * future before the task that fans the activation callbacks out, so on one thread the first runs + * to completion — parked continuations included — before the second begins. + * + *

The client is configured for two pending PublishRequests, which is one per role: one to be + * answered with the Session fault, and one to be returned, once the Session is down, with a + * failure the client answers by trying to send a replacement. + */ + private static final class Fixture implements AutoCloseable { + + private static final long MAX_PENDING_PUBLISH_REQUESTS = 2; + + private final RequestLog requestLog = new RequestLog(); + + private final List deliveredValues = Collections.synchronizedList(new ArrayList<>()); + + /** The order in which the Session future completed and the activation callbacks ran. */ + private final List activationOrder = Collections.synchronizedList(new ArrayList<>()); + + private final CountDownLatch sessionInactive = new CountDownLatch(1); + private final CountDownLatch sessionReactivated = new CountDownLatch(1); + + /** Counted down when the client observes the ServiceFault that provokes the parked refill. */ + private final CountDownLatch refillFaultObserved = new CountDownLatch(1); + + private final ExecutorService executor = + Executors.newSingleThreadExecutor(daemonThreadFactory("publish-suspension-gate-race")); + + private final OpcUaServer server; + private final OpcUaClient client; + private final LoggingSubscriptionServiceSet scriptable; + private final ScriptedSessionServiceSet sessionServiceSet; + + private final UInteger subscriptionId; + private final UInteger clientHandle; + + Fixture() throws Exception { + TestServer testServer = TestServer.create(); + server = testServer.getServer(); + + scriptable = new LoggingSubscriptionServiceSet(server, requestLog); + sessionServiceSet = new ScriptedSessionServiceSet(server); + + for (EndpointConfig endpoint : server.getConfig().getEndpoints()) { + server.addServiceSet(endpoint.getPath(), scriptable); + server.addServiceSet(endpoint.getPath(), sessionServiceSet); + } + + server.startup().get(); + + client = + TestClient.create( + server, + transportConfig -> transportConfig.setExecutor(executor), + cfg -> + cfg.setRequestTimeout(uint(REQUEST_TIMEOUT_MILLIS)) + // No Session keep-alive traffic: the only requests in flight during a test + // are the ones it scripts. + .setKeepAliveInterval(uint(REQUEST_TIMEOUT_MILLIS)) + .setMaxPendingPublishRequests(uint(MAX_PENDING_PUBLISH_REQUESTS))); + client.connect(); + + client.addFaultListener( + serviceFault -> { + if (serviceFault.getResponseHeader().getServiceResult().value() + == StatusCodes.Bad_UnexpectedError) { + + refillFaultObserved.countDown(); + } + }); + + client.addSessionActivityListener( + new SessionActivityListener() { + @Override + public void onSessionInactive(UaSession session) { + sessionInactive.countDown(); + } + + @Override + public void onSessionActive(UaSession session) { + if (sessionInactive.getCount() == 0) { + activationOrder.add(SESSION_ACTIVE); + sessionReactivated.countDown(); + } + } + }); + + scriptable.setRepublishResponder(this::respondToRepublish); + + var subscription = new OpcUaSubscription(client); + subscription.setSubscriptionListener( + new OpcUaSubscription.SubscriptionListener() { + @Override + public void onDataReceived( + OpcUaSubscription s, List items, List values) { + + for (DataValue value : values) { + deliveredValues.add((Integer) value.getValue().getValue()); + } + } + }); + subscription.create(); + + subscriptionId = subscription.getSubscriptionId().orElseThrow(); + + // The MonitoredItem only has to exist on the client: addMonitoredItem assigns the + // ClientHandle + // the notification fan-out looks scripted notifications up by, and no Server-side item + // participates in delivering one. + OpcUaMonitoredItem item = + OpcUaMonitoredItem.newDataItem(NodeIds.Server_ServerStatus_CurrentTime); + subscription.addMonitoredItem(item); + + clientHandle = item.getClientHandle().orElseThrow(); + } + + /** + * Deliver NotificationMessages 1 and 2, leaving the client's last accounted-for sequence number + * at {@value #LAST_SEQUENCE_NUMBER_BEFORE_FAULT} and its Publish pipeline quiescent: every + * request parked at the Server with no responder scripted for any of them. + */ + void deliverInitialNotifications() throws Exception { + enqueueDataChange(1); + assertTrue( + awaitTrue(() -> deliveredValues().size() >= 1, AWAIT_TIMEOUT_MILLIS), + "the first NotificationMessage was never delivered"); + + enqueueDataChange(LAST_SEQUENCE_NUMBER_BEFORE_FAULT); + assertTrue( + awaitTrue(() -> deliveredValues().size() >= 2, AWAIT_TIMEOUT_MILLIS), + "the second NotificationMessage was never delivered"); + + assertTrue( + awaitTrue( + () -> scriptable.getParkedRequestCount() == MAX_PENDING_PUBLISH_REQUESTS, + AWAIT_TIMEOUT_MILLIS), + "the client did not refill its Publish pipeline"); + + assertEquals( + List.of(1, (int) LAST_SEQUENCE_NUMBER_BEFORE_FAULT), + deliveredValues(), + "precondition: the client must have accounted for NotificationMessages 1 and 2 before the" + + " Session fault"); + } + + /** Answer TransferSubscriptions with an empty availableSequenceNumbers (Part 4 §5.14.7.1). */ + void advertiseNothingOnTransfer() { + scriptable.transferScripted.set(true); + } + + /** + * Refuse the next ActivateSession with a ServiceFault, which sends the Session FSM to {@code + * CreatingWait} and from there onto the create-a-new-Session-and-transfer path, and hold the + * ActivateSession of the replacement Session so that a caller can be parked on the Session + * future the FSM will complete for it. + */ + void refuseNextReactivationAndHoldTheReplacement() { + sessionServiceSet.refuseNext.set(true); + sessionServiceSet.holdNext.set(true); + } + + void awaitReplacementActivateSessionHeld() throws Exception { + assertTrue( + sessionServiceSet.heldActivateSession.await( + RECONNECT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), + "the client never got as far as activating a replacement Session"); + } + + void releaseReplacementActivateSession() { + sessionServiceSet.activateSessionGate.countDown(); + } + + void armRequestLog() { + requestLog.arm(); + } + + /** + * Answer one parked PublishRequest with a Bad_SessionIdInvalid ServiceFault, which {@code + * SessionFsmFactory}'s SessionFaultListener classifies as a Session error and turns into a + * reconnect. The Server-side Session is untouched, so re-activation succeeds unless {@link + * #refuseNextReactivation()} was called. + */ + void faultSession() { + scriptable.enqueueServiceFault(StatusCodes.Bad_SessionIdInvalid); + } + + /** + * Return the remaining outstanding PublishRequest with a failure that is a statement about that + * one request rather than about the Session or the Subscription set, so {@code + * PublishingManager} answers it by trying to send a replacement. The Session is gone by now, so + * that replacement's caller finds no Session and parks on the one being established: exactly + * the caller the suspension gate has to hold back when it arrives. + * + *

Returns only once the client has run the failure handler, so the caller is provably parked + * before the Session future can complete. The fault listener fires from a task the failing + * request's completion handler submits, and the failure handler is another such task, so a + * barrier queued once the listener has fired can still be ahead of it — but the second barrier + * cannot, because the first only runs after that completion handler has returned, by which time + * the failure handler is queued. + */ + void parkAPublishRefillOnTheSessionFuture() throws Exception { + scriptable.enqueueServiceFault(StatusCodes.Bad_UnexpectedError); + + assertTrue( + refillFaultObserved.await(AWAIT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), + "the client never observed the ServiceFault that provokes the parked Publish refill"); + + awaitTransportExecutorDrained(); + awaitTransportExecutorDrained(); + } + + /** Wait until the transport executor has run everything queued before this call. */ + private void awaitTransportExecutorDrained() throws Exception { + var drained = new CompletableFuture(); + executor.execute(() -> drained.complete(null)); + + drained.get(AWAIT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); + } + + /** Park a caller on the Session future that records when it is released. */ + void parkAnObserverOnTheSessionFuture() { + client.getSessionAsync().whenComplete((session, ex) -> activationOrder.add(SESSION_FUTURE)); + } + + /** + * Script the next PublishResponse as a data message carrying {@code sequenceNumber}, whose + * value identifies the NotificationMessage it came from. + */ + void enqueueDataChange(long sequenceNumber) { + scriptable.enqueue( + request -> + CompletableFuture.completedFuture( + scriptable.buildPublishResponse( + request, + subscriptionId, + sequenceNumber, + notificationData(sequenceNumber), + new UInteger[] {uint(sequenceNumber)}, + false))); + } + + /** + * The Server holds nothing for retransmission, so every Republish is answered + * Bad_MessageNotAvailable — which is also what terminates the Republish loop Part 4 §6.7 + * describes. + */ + private RepublishResponse respondToRepublish(RepublishRequest request) throws UaException { + requestLog.record(REPUBLISH + request.getRetransmitSequenceNumber().longValue()); + + throw new UaException(StatusCodes.Bad_MessageNotAvailable); + } + + private ExtensionObject[] notificationData(long sequenceNumber) { + var notification = + new MonitoredItemNotification( + clientHandle, new DataValue(Variant.ofInt32((int) sequenceNumber))); + + return new ExtensionObject[] { + scriptable.encode( + new DataChangeNotification(new MonitoredItemNotification[] {notification}, null)) + }; + } + + void awaitSessionInactive() throws Exception { + assertTrue( + sessionInactive.await(RECONNECT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), + "the scripted Bad_SessionIdInvalid Publish fault did not take the Session out of Active"); + } + + void awaitReactivation() throws Exception { + awaitSessionInactive(); + + assertTrue( + sessionReactivated.await(RECONNECT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), + "the Session never became Active again"); + } + + List requestLog() { + return requestLog.entries(); + } + + boolean awaitRequestLogEntries(int count) throws Exception { + return awaitTrue(() -> requestLog.entries().size() >= count, AWAIT_TIMEOUT_MILLIS); + } + + List activationOrder() { + return List.copyOf(activationOrder); + } + + List deliveredValues() { + return List.copyOf(deliveredValues); + } + + int transferCount() { + return scriptable.transferCount; + } + + /** Polls {@code condition} until it holds or the timeout elapses. */ + boolean awaitTrue(ThrowingBooleanSupplier condition, long timeoutMillis) throws Exception { + long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(timeoutMillis); + + while (System.nanoTime() < deadline) { + if (condition.get()) { + return true; + } + Thread.sleep(10); + } + + return condition.get(); + } + + @Override + public void close() throws Exception { + sessionServiceSet.activateSessionGate.countDown(); + scriptable.failParkedRequests(StatusCodes.Bad_NoSubscription); + try { + client.disconnectAsync().get(DISCONNECT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); + } catch (TimeoutException ignored) { + // A client whose single transport thread is occupied cannot run the disconnect it is asked + // for; shutting the Server and the executor down below is what releases it. Tolerated here + // so teardown does not mask the assertion that detected the stall. + } finally { + try { + server.shutdown().get(10, TimeUnit.SECONDS); + } finally { + executor.shutdownNow(); + } + } + } + } + + /** Daemon threads, so a test that leaves work behind cannot keep the JVM alive. */ + private static ThreadFactory daemonThreadFactory(String name) { + return runnable -> { + var thread = new Thread(runnable, name); + thread.setDaemon(true); + + return thread; + }; + } + + @FunctionalInterface + private interface ThrowingBooleanSupplier { + boolean get() throws Exception; + } + + // endregion +} diff --git a/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/PublishTimeoutHintOverflowTest.java b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/PublishTimeoutHintOverflowTest.java new file mode 100644 index 0000000000..4b0fa62a51 --- /dev/null +++ b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/PublishTimeoutHintOverflowTest.java @@ -0,0 +1,192 @@ +/* + * Copyright (c) 2026 the Eclipse Milo Authors + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.milo.opcua.sdk.client.subscriptions; + +import static org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.Unsigned.uint; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import org.eclipse.milo.opcua.sdk.client.OpcUaClient; +import org.eclipse.milo.opcua.sdk.client.OpcUaSession; +import org.eclipse.milo.opcua.sdk.server.EndpointConfig; +import org.eclipse.milo.opcua.sdk.server.OpcUaServer; +import org.eclipse.milo.opcua.sdk.test.ScriptableSubscriptionServiceSet; +import org.eclipse.milo.opcua.sdk.test.TestClient; +import org.eclipse.milo.opcua.sdk.test.TestServer; +import org.eclipse.milo.opcua.stack.core.StatusCodes; +import org.eclipse.milo.opcua.stack.core.UaException; +import org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.UInteger; +import org.eclipse.milo.opcua.stack.core.types.structured.CreateSubscriptionRequest; +import org.eclipse.milo.opcua.stack.core.types.structured.CreateSubscriptionResponse; +import org.eclipse.milo.opcua.stack.core.types.structured.PublishRequest; +import org.eclipse.milo.opcua.stack.core.types.structured.PublishResponse; +import org.eclipse.milo.opcua.stack.transport.server.ServiceRequestContext; +import org.junit.jupiter.api.Test; + +/** + * A Server is free to revise the publishing interval and max keep-alive count it returns from + * CreateSubscription (Part 4 §5.13.2); nothing bounds their product. + * + *

{@code PublishingManager} derives the Publish request's {@code timeoutHint} from {@code + * revisedPublishingInterval * revisedMaxKeepAliveCount * maxPendingPublishes * 1.5}, and {@code + * timeoutHint} is encoded as a UInt32. When the product exceeds {@code UInteger.MAX_VALUE} the + * client must clamp it. If it does not, building the request throws — and because {@code + * maybeSendPublishRequests()} takes a pending-publish permit before calling {@code + * sendPublishRequest()}, the permit is never returned. The permits ratchet up to the maximum and + * the client stops sending Publish requests for the rest of the Session. + */ +public class PublishTimeoutHintOverflowTest { + + /** Revised parameters a Server would plausibly return; their product is far below UInt32. */ + private static final double ORDINARY_PUBLISHING_INTERVAL = 1_000.0; + + private static final UInteger ORDINARY_MAX_KEEP_ALIVE_COUNT = uint(10); + + /** + * A one-hour publishing interval with a max keep-alive count of 1000: both legal, and their + * product (3.6e9 ms) multiplied by the two in-flight Publish requests and the 1.5 safety factor + * yields ~1.08e10, well beyond the UInt32 range {@code timeoutHint} is carried in. + */ + private static final double LARGE_PUBLISHING_INTERVAL = 3_600_000.0; + + private static final UInteger LARGE_MAX_KEEP_ALIVE_COUNT = uint(1_000); + + /** + * Control for {@link #publishRequestIsSentWhenRevisedParametersOverflowTheTimeoutHint()}: with + * ordinary revised parameters the client pipelines a Publish request as soon as the Subscription + * is created. If this one fails, the fixture is broken rather than the timeout hint. + */ + @Test + void publishRequestIsSentWithOrdinaryRevisedParameters() throws Exception { + try (var fixture = new Fixture(ORDINARY_PUBLISHING_INTERVAL, ORDINARY_MAX_KEEP_ALIVE_COUNT)) { + fixture.createSubscription(); + + assertTrue( + fixture.awaitPublishRequest(), + "no Publish request was sent after the Subscription was created"); + } + } + + /** + * Creating a Subscription whose revised parameters overflow the computed timeout hint must still + * start Publish traffic. Under the defect the overflow throws out of {@code sendPublishRequest()} + * after the pending-publish permit was taken, so no Publish request is ever sent. + */ + @Test + void publishRequestIsSentWhenRevisedParametersOverflowTheTimeoutHint() throws Exception { + try (var fixture = new Fixture(LARGE_PUBLISHING_INTERVAL, LARGE_MAX_KEEP_ALIVE_COUNT)) { + fixture.createSubscription(); + + assertTrue( + fixture.awaitPublishRequest(), + "no Publish request was sent: the timeout hint overflowed UInt32 and the" + + " pending-publish permit was leaked"); + } + } + + /** + * The same defect observed at its source. Anything thrown while building or sending the Publish + * request escapes with the caller's pending-publish permit already taken and no completion + * handler registered to return it. + */ + @Test + void sendPublishRequestDoesNotThrowWhenRevisedParametersOverflowTheTimeoutHint() + throws Exception { + try (var fixture = new Fixture(LARGE_PUBLISHING_INTERVAL, LARGE_MAX_KEEP_ALIVE_COUNT)) { + fixture.createSubscription(); + + PublishingManager publishingManager = fixture.client.getPublishingManager(); + OpcUaSession session = fixture.client.getSession(); + + assertDoesNotThrow(() -> publishingManager.sendPublishRequest(session, new AtomicLong(1))); + } + } + + /** + * A running Server whose CreateSubscription response is rewritten to advertise the given revised + * publishing interval and max keep-alive count, plus a connected client. Publish requests are + * counted and then parked by {@link ScriptableSubscriptionServiceSet}. + */ + private static final class Fixture implements AutoCloseable { + + private final CountDownLatch publishRequestReceived = new CountDownLatch(1); + + private final OpcUaServer server; + private final OpcUaClient client; + private final ScriptableSubscriptionServiceSet scriptable; + + Fixture(double revisedPublishingInterval, UInteger revisedMaxKeepAliveCount) throws Exception { + TestServer testServer = TestServer.create(); + server = testServer.getServer(); + + scriptable = + new ScriptableSubscriptionServiceSet(server) { + @Override + public CreateSubscriptionResponse onCreateSubscription( + ServiceRequestContext context, CreateSubscriptionRequest request) + throws UaException { + + CreateSubscriptionResponse response = super.onCreateSubscription(context, request); + + return new CreateSubscriptionResponse( + response.getResponseHeader(), + response.getSubscriptionId(), + revisedPublishingInterval, + response.getRevisedLifetimeCount(), + revisedMaxKeepAliveCount); + } + + @Override + public CompletableFuture onPublish( + ServiceRequestContext context, PublishRequest request) { + + publishRequestReceived.countDown(); + + return super.onPublish(context, request); + } + }; + + for (EndpointConfig endpoint : server.getConfig().getEndpoints()) { + server.addServiceSet(endpoint.getPath(), scriptable); + } + + server.startup().get(); + + // Long request timeout so parked Publish requests do not time out during the test. + client = TestClient.create(server, cfg -> cfg.setRequestTimeout(uint(60_000))); + client.connect(); + } + + OpcUaSubscription createSubscription() throws UaException { + var subscription = new OpcUaSubscription(client); + subscription.create(); + return subscription; + } + + boolean awaitPublishRequest() throws InterruptedException { + return publishRequestReceived.await(5, TimeUnit.SECONDS); + } + + @Override + public void close() throws Exception { + scriptable.failParkedRequests(StatusCodes.Bad_NoSubscription); + try { + client.disconnectAsync().get(5, TimeUnit.SECONDS); + } finally { + server.shutdown().get(5, TimeUnit.SECONDS); + } + } + } +} diff --git a/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/PublishingManagerSessionIsolationTest.java b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/PublishingManagerSessionIsolationTest.java new file mode 100644 index 0000000000..92755ba1f5 --- /dev/null +++ b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/PublishingManagerSessionIsolationTest.java @@ -0,0 +1,340 @@ +/* + * Copyright (c) 2026 the Eclipse Milo Authors + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.milo.opcua.sdk.client.subscriptions; + +import static org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.Unsigned.uint; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; +import java.util.stream.LongStream; +import org.eclipse.milo.opcua.sdk.client.OpcUaClient; +import org.eclipse.milo.opcua.sdk.client.OpcUaClientConfig; +import org.eclipse.milo.opcua.sdk.client.UaSession; +import org.eclipse.milo.opcua.stack.core.StatusCodes; +import org.eclipse.milo.opcua.stack.core.UaException; +import org.eclipse.milo.opcua.stack.core.types.UaResponseMessageType; +import org.eclipse.milo.opcua.stack.core.types.builtin.DateTime; +import org.eclipse.milo.opcua.stack.core.types.builtin.ExtensionObject; +import org.eclipse.milo.opcua.stack.core.types.builtin.NodeId; +import org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.UInteger; +import org.eclipse.milo.opcua.stack.core.types.structured.NotificationMessage; +import org.eclipse.milo.opcua.stack.core.types.structured.RepublishRequest; +import org.eclipse.milo.opcua.stack.core.types.structured.RepublishResponse; +import org.eclipse.milo.opcua.stack.core.types.structured.RequestHeader; +import org.eclipse.milo.opcua.stack.core.types.structured.ResponseHeader; +import org.eclipse.milo.opcua.stack.core.util.TaskQueue; +import org.junit.jupiter.api.Test; + +/** Session-activation ownership of reconnect recovery state in {@link PublishingManager}. */ +public class PublishingManagerSessionIsolationTest { + + private static final long AWAIT_TIMEOUT_SECONDS = 5L; + + /** + * A recovery from activation A can remain queued when replacement Session B transfers the same + * Subscription. A must not consume B's advertised sequence numbers; otherwise B falls back to a + * blind drain and can lose messages the TransferResult proved were recoverable. + */ + @Test + void staleRecoveryDoesNotConsumeReplacementSessionsTransferResult() throws Exception { + Fixture fixture = new Fixture(1, 1); + UaSession staleSession = fixture.newSession(); + UaSession replacementSession = fixture.newSession(); + + fixture.setLastSequenceNumber(6L); + fixture.setActivation(1L); + fixture.setResponder(request -> response(request.getRetransmitSequenceNumber().longValue())); + + fixture.manager.notifySubscriptionTransferred( + replacementSession, fixture.subscriptionId, new UInteger[] {uint(7)}); + + // Models another activation callback being counted before this Session's callback runs. The + // TransferResult belongs to the Session itself; predicting "current + 1" at notification time + // would stamp it with activation 2 and make activation 3 ignore it. + fixture.setActivation(3L); + fixture.republishUntilUnavailable(staleSession, 1L); + fixture.republishUntilUnavailable(replacementSession, 3L); + + assertEquals( + List.of(7L), + fixture.republishRequests, + "the stale recovery consumed the replacement Session's TransferResult, so the replacement" + + " performed a blind extra Republish instead of following its advertised set"); + } + + /** + * Milo can permit more than 64 outstanding PublishRequests. Part 4 §6.7 recovery must cover every + * response that pipeline could have lost and still make the extra request that receives + * Bad_MessageNotAvailable; otherwise Publish resumes before the retransmission queue is drained. + */ + @Test + void blindRecoveryCoversConfiguredPipelineDepthBeyond64AndTerminationProbe() throws Exception { + int pipelineDepth = 65; + Fixture fixture = new Fixture(pipelineDepth - 1, pipelineDepth); + UaSession session = fixture.newSession(); + + fixture.capturePermittedPipelineDepth(); + + // The recovery bound belongs to the pre-disconnect pipeline. Removing all but the Subscription + // under test after the outage must not shrink it to the new two-request natural target. + fixture.retainOnlyPrimarySubscription(); + fixture.setActivation(1L); + fixture.setResponder( + request -> { + long sequenceNumber = request.getRetransmitSequenceNumber().longValue(); + + return sequenceNumber <= pipelineDepth + ? response(sequenceNumber) + : CompletableFuture.failedFuture( + new UaException(StatusCodes.Bad_MessageNotAvailable)); + }); + + fixture.republishUntilUnavailable(session, 1L); + + List expected = LongStream.rangeClosed(1L, pipelineDepth + 1L).boxed().toList(); + + assertEquals( + expected, + fixture.republishRequests, + "a blind reconnect drain must request all 65 NotificationMessages a configured 65-deep" + + " Publish pipeline could have lost, then request sequence 66 to receive the" + + " Bad_MessageNotAvailable termination response"); + } + + /** + * Bad_TooManyPublishRequests describes the Session that rejected the request. A delayed failure + * from activation A must not install a ceiling after activation B has reset its independent + * Publish pipeline state. + */ + @Test + void oldActivationCannotClampReplacementSessionsPublishCeiling() throws Exception { + Fixture fixture = new Fixture(0, 1); + fixture.setActivation(2L); + + fixture.handlePublishFailure( + new UaException(StatusCodes.Bad_TooManyPublishRequests), new AtomicLong(1L), 1L); + + assertEquals( + Long.MAX_VALUE, + fixture.pendingPublishCeiling(), + "Bad_TooManyPublishRequests from activation 1 clamped activation 2 even though pending" + + " Publish limits are Session-scoped"); + } + + /** A delayed successful response cannot pay down a replacement Session's learned ceiling. */ + @Test + void oldActivationCannotAdvanceReplacementSessionsCeilingCooldown() throws Exception { + Fixture fixture = new Fixture(0, 1); + fixture.setActivation(2L); + + fixture.handlePublishFailure( + new UaException(StatusCodes.Bad_TooManyPublishRequests), new AtomicLong(2L), 2L); + + fixture.releasePendingPublish(new AtomicLong(1L), 1L); + + assertEquals( + 0L, + fixture.pendingPublishCeilingSuccesses(), + "a successful PublishResponse from activation 1 advanced activation 2's cooldown"); + } + + private static CompletableFuture response(long sequenceNumber) { + var notificationMessage = + new NotificationMessage(uint(sequenceNumber), DateTime.now(), new ExtensionObject[0]); + + return CompletableFuture.completedFuture( + new RepublishResponse(mock(ResponseHeader.class), notificationMessage)); + } + + /** A minimal, synchronously executed PublishingManager fixture for activation-boundary tests. */ + private static final class Fixture { + + private final List republishRequests = new ArrayList<>(); + + private final AtomicReference< + Function>> + responder = new AtomicReference<>(); + + private final OpcUaClient client = mock(OpcUaClient.class); + private final PublishingManager manager; + private final Class subscriptionDetailsClass; + private final Object primaryDetails; + private final UInteger subscriptionId = uint(1); + + Fixture(int subscriptionCount, int maxPendingPublishRequests) throws Exception { + OpcUaClientConfig config = mock(OpcUaClientConfig.class); + when(config.getMaxPendingPublishRequests()).thenReturn(uint(maxPendingPublishRequests)); + when(client.getConfig()).thenReturn(config); + when(client.newRequestHeader(any(NodeId.class))).thenReturn(mock(RequestHeader.class)); + when(client.getSessionAsync()).thenReturn(new CompletableFuture<>()); + when(client.sendRequestAsync(any())) + .thenAnswer( + invocation -> { + RepublishRequest request = invocation.getArgument(0); + republishRequests.add(request.getRetransmitSequenceNumber().longValue()); + + return responder.get().apply(request); + }); + + manager = new PublishingManager(client); + + subscriptionDetailsClass = + Class.forName(PublishingManager.class.getName() + "$SubscriptionDetails"); + + Map details = subscriptionDetails(manager); + + primaryDetails = newSubscriptionDetails(subscriptionId); + + for (int i = 0; i < subscriptionCount; i++) { + UInteger id = uint(i + 1L); + details.put(id, i == 0 ? primaryDetails : newSubscriptionDetails(id)); + } + } + + UaSession newSession() { + UaSession session = mock(UaSession.class); + when(session.getAuthenticationToken()).thenReturn(NodeId.NULL_VALUE); + when(session.getSessionId()).thenReturn(new NodeId(1, System.identityHashCode(session))); + + return session; + } + + void capturePermittedPipelineDepth() throws Exception { + Method method = PublishingManager.class.getDeclaredMethod("maybeSendPublishRequests"); + method.setAccessible(true); + method.invoke(manager); + } + + void retainOnlyPrimarySubscription() throws Exception { + Map details = subscriptionDetails(manager); + details.keySet().removeIf(id -> !id.equals(subscriptionId)); + } + + void setResponder( + Function> responder) { + + this.responder.set(responder); + } + + @SuppressWarnings("unchecked") + void setActivation(long activation) throws Exception { + Field field = PublishingManager.class.getDeclaredField("sessionActivations"); + field.setAccessible(true); + + ((AtomicLong) field.get(manager)).set(activation); + + Field ceilingField = PublishingManager.class.getDeclaredField("pendingPublishCeiling"); + ceilingField.setAccessible(true); + + AtomicReference ceiling = (AtomicReference) ceilingField.get(manager); + Method none = ceiling.get().getClass().getDeclaredMethod("none", long.class); + none.setAccessible(true); + ceiling.set(none.invoke(null, activation)); + } + + void setLastSequenceNumber(long sequenceNumber) throws Exception { + Field field = subscriptionDetailsClass.getDeclaredField("lastSequenceNumber"); + field.setAccessible(true); + field.setLong(primaryDetails, sequenceNumber); + } + + void republishUntilUnavailable(UaSession session, long activation) throws Exception { + Method method = + PublishingManager.class.getDeclaredMethod( + "republishUntilUnavailable", UaSession.class, subscriptionDetailsClass, long.class); + method.setAccessible(true); + + CompletableFuture future = + (CompletableFuture) method.invoke(manager, session, primaryDetails, activation); + future.get(AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + } + + void handlePublishFailure(Throwable failure, AtomicLong pendingCount, long activation) + throws Exception { + + Method method = + PublishingManager.class.getDeclaredMethod( + "handlePublishFailure", + Throwable.class, + UInteger.class, + AtomicLong.class, + List.class, + long.class, + long.class); + method.setAccessible(true); + method.invoke(manager, failure, uint(1), pendingCount, List.of(), 0L, activation); + } + + long pendingPublishCeiling() throws Exception { + return pendingPublishCeilingField("ceiling"); + } + + long pendingPublishCeilingSuccesses() throws Exception { + return pendingPublishCeilingField("successes"); + } + + void releasePendingPublish(AtomicLong pendingCount, long activation) throws Exception { + Method method = + PublishingManager.class.getDeclaredMethod( + "releasePendingPublish", AtomicLong.class, long.class); + method.setAccessible(true); + method.invoke(manager, pendingCount, activation); + } + + private long pendingPublishCeilingField(String name) throws Exception { + Field field = PublishingManager.class.getDeclaredField("pendingPublishCeiling"); + field.setAccessible(true); + + Object state = ((AtomicReference) field.get(manager)).get(); + Method accessor = state.getClass().getDeclaredMethod(name); + accessor.setAccessible(true); + + return (long) accessor.invoke(state); + } + + private Object newSubscriptionDetails(UInteger id) throws Exception { + OpcUaSubscription subscription = mock(OpcUaSubscription.class); + when(subscription.getIncarnation()).thenReturn(1L); + when(subscription.getDeliveryQueue()).thenReturn(new TaskQueue(Runnable::run)); + + Constructor constructor = + subscriptionDetailsClass.getDeclaredConstructor( + OpcUaSubscription.class, UInteger.class, java.util.concurrent.Executor.class); + constructor.setAccessible(true); + + return constructor.newInstance( + subscription, id, (java.util.concurrent.Executor) Runnable::run); + } + + @SuppressWarnings("unchecked") + private static Map subscriptionDetails(PublishingManager manager) + throws Exception { + + Field field = PublishingManager.class.getDeclaredField("subscriptionDetails"); + field.setAccessible(true); + + return (Map) field.get(manager); + } + } +} diff --git a/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/RepublishRecoveryTest.java b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/RepublishRecoveryTest.java new file mode 100644 index 0000000000..3305fbc697 --- /dev/null +++ b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/RepublishRecoveryTest.java @@ -0,0 +1,609 @@ +/* + * Copyright (c) 2026 the Eclipse Milo Authors + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.milo.opcua.sdk.client.subscriptions; + +import static org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.Unsigned.uint; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; +import org.eclipse.milo.opcua.sdk.client.OpcUaClient; +import org.eclipse.milo.opcua.sdk.server.EndpointConfig; +import org.eclipse.milo.opcua.sdk.server.OpcUaServer; +import org.eclipse.milo.opcua.sdk.test.ScriptableSubscriptionServiceSet; +import org.eclipse.milo.opcua.sdk.test.ScriptableSubscriptionServiceSet.RepublishResponder; +import org.eclipse.milo.opcua.sdk.test.TestClient; +import org.eclipse.milo.opcua.sdk.test.TestServer; +import org.eclipse.milo.opcua.stack.core.StatusCodes; +import org.eclipse.milo.opcua.stack.core.UaException; +import org.eclipse.milo.opcua.stack.core.types.builtin.DataValue; +import org.eclipse.milo.opcua.stack.core.types.builtin.ExtensionObject; +import org.eclipse.milo.opcua.stack.core.types.builtin.Variant; +import org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.UInteger; +import org.eclipse.milo.opcua.stack.core.types.structured.DataChangeNotification; +import org.eclipse.milo.opcua.stack.core.types.structured.MonitoredItemNotification; +import org.eclipse.milo.opcua.stack.core.util.TaskQueue; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +/** + * Republish recovery in {@code PublishingManager.recoverMissingNotificationMessages}. + * + *

When a PublishResponse reveals a gap in the NotificationMessage sequence, the client recovers + * the missing messages with the Republish service (Part 4 §5.14.6). Recovery runs on {@code + * PublishingManager}'s processing queue — one {@link TaskQueue} with a concurrency limit of 1, + * shared by every Subscription on the client — and calls the synchronous {@code + * OpcUaClient.republish}, which is {@code republishAsync(...).get()}: an unbounded blocking wait + * for a network round trip, taken on the queue that is processing PublishResponses. + * + *

Two consequences follow, and the first two nested classes assert against them: + * + *

    + *
  1. The thread that blocks belongs to the transport executor, and the RepublishResponse's + * completion is dispatched onto that same executor: {@code + * AbstractUascClientTransport.handleResponse} completes everything that is not a + * PublishResponse via {@code config.getExecutor().execute(...)}. A client whose executor has + * one thread therefore self-deadlocks — the only thread that could complete the Republish + * response is the one waiting for it — and the deadlock is permanent, not merely slow: {@code + * handleResponse} cancels the request timeout the moment the response arrives, so the wheel + * timer that rescues a Server that never answers cannot rescue a Server that does. A + * single-threaded executor is an ordinary application choice; {@code + * OpcTcpClientTransportConfigBuilder.setExecutor} exists for it. + *
  2. Even with threads to spare, recovery holds the single processing queue for a network round + * trip per missing message, so PublishResponses for every other Subscription on that + * client wait behind it. + *
+ * + *

The third nested class covers what recovery asks for rather than how it waits. Part 4 + * §5.14.5.2 defines the Publish response's availableSequenceNumbers as "a list of sequence number + * ranges that identify unacknowledged NotificationMessages that are available for retransmission + * from the Subscription's retransmission queue". Recovery walks the numeric gap instead of that + * list, so it spends blocking round trips asking for messages the Server has just said it no longer + * holds. + */ +public class RepublishRecoveryTest { + + /** How long to wait for something that must happen. */ + private static final long AWAIT_TIMEOUT_MILLIS = 10_000; + + /** + * How long to wait for work that a blocked processing queue is holding up. A client that does not + * block on the processing queue gets through a loopback Republish round trip in single-digit + * milliseconds, so this is generous by three orders of magnitude — but the stalls below do not + * end at all, so no larger value would change an outcome. + */ + private static final long STALL_WINDOW_MILLIS = 5_000; + + /** + * Long enough that nothing times out on its own: no parked Publish request, and no Republish. The + * stalls asserted against below are therefore the client's own doing and not a request timeout. + */ + private static final long REQUEST_TIMEOUT_MILLIS = 60_000; + + /** NotificationMessages 2, 3 and 4 are missing when 5 arrives. */ + private static final int MISSING_COUNT = 3; + + private static final long GAP_REVEALED_BY = 5; + + /** Upper bound on how long a gated Republish response is held, so nothing hangs indefinitely. */ + private static final long GATE_TIMEOUT_MILLIS = 30_000; + + /** How long teardown waits for a client that may be deadlocked to disconnect. */ + private static final long DISCONNECT_TIMEOUT_MILLIS = 5_000; + + /** + * Recovery must not require a second thread to make progress. The blocking {@code republish()} + * call is made from a transport-executor thread and the response it waits for is completed by the + * transport executor, so a client whose executor has a single thread cannot complete its own + * Republish. + * + *

The Server here is entirely healthy: it answers every Republish request immediately. + */ + @Nested + class BlockedByItsOwnRecovery { + + @Test + void everyMissingNotificationMessageIsRequestedWhenTheExecutorHasOneThread() throws Exception { + try (var fixture = new Fixture(Executors.newSingleThreadExecutor())) { + Sub sub = fixture.createSubscription(1); + fixture.revealGap(sub); + + assertTrue( + fixture.awaitRepublishRequests(1, AWAIT_TIMEOUT_MILLIS), + "the gap was never detected: no Republish request reached the Server"); + + assertTrue( + fixture.awaitRepublishRequests(MISSING_COUNT, STALL_WINDOW_MILLIS), + () -> + "only " + + fixture.republishRequests.size() + + " of " + + MISSING_COUNT + + " missing NotificationMessages were requested; the Server answered the" + + " first Republish immediately, but completing that response is a task for" + + " the executor thread that is blocked waiting for it"); + } + } + + /** + * The same stall as the application sees it. Until recovery finishes, the NotificationMessage + * that revealed the gap has not been delivered either — it is delivered after the recovered + * ones — so the Subscription simply stops producing data. + */ + @Test + void notificationMessageThatRevealedTheGapIsDeliveredWhenTheExecutorHasOneThread() + throws Exception { + + try (var fixture = new Fixture(Executors.newSingleThreadExecutor())) { + Sub sub = fixture.createSubscription(1); + int deliveriesBefore = sub.dataReceivedCount().get(); + + fixture.revealGap(sub); + + assertTrue( + fixture.awaitRepublishRequests(1, AWAIT_TIMEOUT_MILLIS), + "the gap was never detected: no Republish request reached the Server"); + + assertTrue( + fixture.awaitTrue( + () -> sub.dataReceivedCount().get() > deliveriesBefore, STALL_WINDOW_MILLIS), + "no further NotificationMessage was delivered: the recovery of the " + + MISSING_COUNT + + " missing messages is blocking the queue that processes PublishResponses"); + } + } + + /** + * The control that proves the two tests above are about the executor and not about the fixture: + * the identical script, driven by a client whose executor has threads to spare, recovers the + * whole gap and delivers. + */ + @Test + void everyMissingNotificationMessageIsRequestedWhenTheExecutorHasThreadsToSpare() + throws Exception { + + try (var fixture = new Fixture(cachedThreadPool())) { + Sub sub = fixture.createSubscription(1); + int deliveriesBefore = sub.dataReceivedCount().get(); + + fixture.revealGap(sub); + + assertTrue( + fixture.awaitRepublishRequests(MISSING_COUNT, STALL_WINDOW_MILLIS), + () -> + "control: with a multi-threaded executor all " + + MISSING_COUNT + + " missing NotificationMessages must be requested, but only " + + fixture.republishRequests.size() + + " were"); + + assertTrue( + fixture.awaitTrue( + () -> sub.dataReceivedCount().get() > deliveriesBefore, STALL_WINDOW_MILLIS), + "control: with a multi-threaded executor the recovered NotificationMessages and the" + + " one that revealed the gap must be delivered"); + } + } + } + + /** + * There is one {@code PublishingManager} per client and one processing queue for all of its + * Subscriptions, so a Subscription recovering a gap spends that queue on a synchronous network + * round trip per missing NotificationMessage. Nothing else the client received is processed + * meanwhile: an unrelated, healthy Subscription stops receiving its data because a different one + * lost a message. + * + *

The Republish response is gated by the test rather than delayed by a sleep, so the stall is + * a property of the code under test and not of the fixture's timing. + */ + @Nested + class RecoveryOnTheSharedProcessingQueue { + + @Test + void publishResponseForAnotherSubscriptionIsProcessedWhileRecoveryIsInFlight() + throws Exception { + try (var fixture = new Fixture(cachedThreadPool())) { + Sub recovering = fixture.createSubscription(2); + Sub healthy = fixture.createSubscription(3); + + fixture.gateRepublishResponses(); + fixture.revealGap(recovering); + + assertTrue( + fixture.republishStarted.await(AWAIT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), + "the gap was never detected: no Republish request reached the Server"); + + int deliveriesBefore = healthy.dataReceivedCount().get(); + fixture.scriptable.enqueueDataChange( + healthy.subscriptionId(), 2, dataNotifications(), uint(2)); + + assertTrue( + fixture.awaitTrue( + () -> healthy.dataReceivedCount().get() > deliveriesBefore, STALL_WINDOW_MILLIS), + "a PublishResponse for a Subscription with nothing missing was not processed while" + + " another Subscription's Republish recovery was in flight: recovery is blocking" + + " the processing queue that every Subscription on this client shares"); + } + } + + /** + * The control: the same two Subscriptions and the same gap, with the Republish answered + * immediately. It isolates the blocking wait as the cause — the PublishResponse for the second + * Subscription reaches the client identically in both cases. + */ + @Test + void publishResponseForAnotherSubscriptionIsProcessedWhenRecoveryDoesNotBlock() + throws Exception { + + try (var fixture = new Fixture(cachedThreadPool())) { + Sub recovering = fixture.createSubscription(2); + Sub healthy = fixture.createSubscription(3); + + fixture.revealGap(recovering); + + assertTrue( + fixture.awaitRepublishRequests(MISSING_COUNT, AWAIT_TIMEOUT_MILLIS), + "control: the gap was never recovered"); + + int deliveriesBefore = healthy.dataReceivedCount().get(); + fixture.scriptable.enqueueDataChange( + healthy.subscriptionId(), 2, dataNotifications(), uint(2)); + + assertTrue( + fixture.awaitTrue( + () -> healthy.dataReceivedCount().get() > deliveriesBefore, STALL_WINDOW_MILLIS), + "control: a PublishResponse for a second Subscription must be processed once no" + + " recovery is in flight"); + } + } + } + + /** + * Part 4 §5.14.5.2: availableSequenceNumbers is "a list of sequence number ranges that identify + * unacknowledged NotificationMessages that are available for retransmission from the + * Subscription's retransmission queue". A sequence number missing from that list is one the + * Server has just told the client it cannot retransmit — the retransmission queue overflowed and + * dropped it, or it was acknowledged — so Republishing it can only be answered + * Bad_MessageNotAvailable. + * + *

Recovery instead walks the numeric gap and asks for every sequence number in it. Each of + * those requests is a blocking round trip on the shared processing queue (see the nested classes + * above), spent on an answer the client was given before it asked. + */ + @Nested + class AvailableSequenceNumbers { + + @Test + void republishIsNotRequestedForNotificationMessagesTheServerIsNotHolding() throws Exception { + try (var fixture = new Fixture(cachedThreadPool())) { + Sub sub = fixture.createSubscription(1); + fixture.useRetransmissionQueueResponder(Set.of(3L, 4L, 5L)); + + int publishRequestsBefore = fixture.scriptable.getPublishRequestCount(); + + // The Server is holding 3, 4 and 5: NotificationMessage 2 is no longer in the + // retransmission queue. + fixture.scriptable.enqueueDataChange( + sub.subscriptionId(), GAP_REVEALED_BY, dataNotifications(), uint(3), uint(4), uint(5)); + + assertTrue( + fixture.awaitResponsesProcessed(publishRequestsBefore, 1, AWAIT_TIMEOUT_MILLIS), + "the PublishResponse that revealed the gap was never fully processed"); + + assertEquals( + List.of(3L, 4L), + List.copyOf(fixture.republishRequests), + "the Server advertised only 3, 4 and 5 as available for retransmission, so" + + " NotificationMessage 2 is gone; Republishing it is a blocking round trip that" + + " can only be answered Bad_MessageNotAvailable"); + } + } + + /** + * The control for the assertion above: when the Server is still holding every missing + * NotificationMessage, every one of them is requested. Without it, restricting recovery to the + * advertised set could pass by never Republishing anything at all. + */ + @Test + void republishIsRequestedForEveryNotificationMessageTheServerIsHolding() throws Exception { + try (var fixture = new Fixture(cachedThreadPool())) { + Sub sub = fixture.createSubscription(1); + fixture.useRetransmissionQueueResponder(Set.of(2L, 3L, 4L, 5L)); + + int publishRequestsBefore = fixture.scriptable.getPublishRequestCount(); + + fixture.scriptable.enqueueDataChange( + sub.subscriptionId(), + GAP_REVEALED_BY, + dataNotifications(), + uint(2), + uint(3), + uint(4), + uint(5)); + + assertTrue( + fixture.awaitResponsesProcessed(publishRequestsBefore, 1, AWAIT_TIMEOUT_MILLIS), + "the PublishResponse that revealed the gap was never fully processed"); + + assertEquals( + List.of(2L, 3L, 4L), + List.copyOf(fixture.republishRequests), + "control: every missing NotificationMessage the Server is still holding must be" + + " requested via Republish"); + } + } + } + + // region fixture + + /** A Subscription's Server-assigned id and the number of deliveries its listener has observed. */ + private record Sub(UInteger subscriptionId, AtomicInteger dataReceivedCount) {} + + /** + * A running Server whose Publish and Republish responses are scripted, plus a client driven by a + * caller-supplied {@link ExecutorService}. + */ + private static final class Fixture implements AutoCloseable { + + /** Every {@code retransmitSequenceNumber} the client has asked the Server to Republish. */ + private final List republishRequests = Collections.synchronizedList(new ArrayList<>()); + + /** Counted down when the Server receives a gated Republish request. */ + private final CountDownLatch republishStarted = new CountDownLatch(1); + + /** Releases a gated Republish response; always counted down by {@link #close()}. */ + private final CountDownLatch republishGate = new CountDownLatch(1); + + private final ExecutorService executor; + private final OpcUaServer server; + private final OpcUaClient client; + private final ScriptableSubscriptionServiceSet scriptable; + + Fixture(ExecutorService executor) throws Exception { + this.executor = executor; + + TestServer testServer = TestServer.create(); + server = testServer.getServer(); + + scriptable = new ScriptableSubscriptionServiceSet(server); + for (EndpointConfig endpoint : server.getConfig().getEndpoints()) { + server.addServiceSet(endpoint.getPath(), scriptable); + } + + server.startup().get(); + + client = + TestClient.create( + server, + transportConfig -> transportConfig.setExecutor(executor), + cfg -> + cfg.setRequestTimeout(uint(REQUEST_TIMEOUT_MILLIS)) + // No Session keep-alive traffic: the only requests in flight during a test + // are + // the ones it scripts. + .setKeepAliveInterval(uint(REQUEST_TIMEOUT_MILLIS))); + client.connect(); + + scriptable.setRepublishResponder(recordingRepublishResponder()); + } + + /** + * Create a Subscription, drive its {@code lastSequenceNumber} to 1, and wait for the client's + * Publish pipeline to refill so that responses scripted afterwards land on requests already + * parked at the Server. + * + * @param parkedRequests the number of parked Publish requests to wait for; the client keeps one + * more in flight than it has Subscriptions. + */ + Sub createSubscription(int parkedRequests) throws Exception { + var dataReceivedCount = new AtomicInteger(); + + var subscription = new OpcUaSubscription(client); + subscription.setSubscriptionListener( + new OpcUaSubscription.SubscriptionListener() { + @Override + public void onDataReceived( + OpcUaSubscription subscription, + List items, + List values) { + + dataReceivedCount.incrementAndGet(); + } + }); + subscription.create(); + + var sub = new Sub(subscription.getSubscriptionId().orElseThrow(), dataReceivedCount); + + scriptable.enqueueDataChange(sub.subscriptionId(), 1, dataNotifications(), uint(1)); + + assertTrue( + awaitTrue(() -> sub.dataReceivedCount().get() >= 1, AWAIT_TIMEOUT_MILLIS), + "the first NotificationMessage was never delivered"); + assertTrue( + awaitTrue( + () -> scriptable.getParkedRequestCount() >= parkedRequests, AWAIT_TIMEOUT_MILLIS), + "the client did not refill its Publish pipeline"); + + return sub; + } + + /** + * Send a NotificationMessage carrying sequence {@value #GAP_REVEALED_BY} to a Subscription + * whose last accounted-for sequence number is 1, leaving {@value #MISSING_COUNT} missing. Every + * missing message is advertised as available for retransmission, so the whole gap is + * recoverable. + */ + void revealGap(Sub sub) { + scriptable.enqueueDataChange( + sub.subscriptionId(), + GAP_REVEALED_BY, + dataNotifications(), + uint(2), + uint(3), + uint(4), + uint(5)); + } + + /** Records the requested sequence number and answers with a NotificationMessage. */ + private RepublishResponder recordingRepublishResponder() { + return request -> { + long sequenceNumber = request.getRetransmitSequenceNumber().longValue(); + republishRequests.add(sequenceNumber); + + return scriptable.buildRepublishResponse( + request, sequenceNumber, encodedDataNotifications()); + }; + } + + /** + * Install a Republish responder that suspends the caller until {@link #close()} releases it. + * The Server dispatches every service request on its own executor, so only the thread handling + * this one request waits; the Server keeps answering everything else. + */ + void gateRepublishResponses() { + scriptable.setRepublishResponder( + request -> { + long sequenceNumber = request.getRetransmitSequenceNumber().longValue(); + republishRequests.add(sequenceNumber); + republishStarted.countDown(); + + try { + if (!republishGate.await(GATE_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { + throw new UaException(StatusCodes.Bad_Timeout); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new UaException(StatusCodes.Bad_UnexpectedError, e); + } + + return scriptable.buildRepublishResponse( + request, sequenceNumber, encodedDataNotifications()); + }); + } + + /** + * Install a Republish responder that behaves like a real retransmission queue: it answers for + * the sequence numbers it is holding and reports Bad_MessageNotAvailable for anything else. + * + * @param held the sequence numbers still in the Server's retransmission queue, i.e. the ones + * its PublishResponse advertises in availableSequenceNumbers. + */ + void useRetransmissionQueueResponder(Set held) { + scriptable.setRepublishResponder( + request -> { + long sequenceNumber = request.getRetransmitSequenceNumber().longValue(); + republishRequests.add(sequenceNumber); + + if (held.contains(sequenceNumber)) { + return scriptable.buildRepublishResponse( + request, sequenceNumber, encodedDataNotifications()); + } else { + throw new UaException(StatusCodes.Bad_MessageNotAvailable); + } + }); + } + + boolean awaitRepublishRequests(int count, long timeoutMillis) throws Exception { + return awaitTrue(() -> republishRequests.size() >= count, timeoutMillis); + } + + /** + * Wait until the client has finished processing and delivering {@code count} more + * PublishResponses. + * + *

The client replaces a Publish request only once the notifications it carried have been + * delivered — that is the SDK's backpressure mechanism — so {@code count} further Publish + * requests arriving at the Server is exactly that signal. + */ + boolean awaitResponsesProcessed(int publishRequestsBefore, int count, long timeoutMillis) + throws Exception { + + return awaitTrue( + () -> scriptable.getPublishRequestCount() >= publishRequestsBefore + count, + timeoutMillis); + } + + private ExtensionObject[] encodedDataNotifications() { + return new ExtensionObject[] { + scriptable.encode( + new DataChangeNotification( + dataNotifications().toArray(MonitoredItemNotification[]::new), null)) + }; + } + + /** Polls {@code condition} until it holds or the timeout elapses. */ + boolean awaitTrue(ThrowingBooleanSupplier condition, long timeoutMillis) throws Exception { + long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(timeoutMillis); + + while (System.nanoTime() < deadline) { + if (condition.get()) { + return true; + } + Thread.sleep(10); + } + + return condition.get(); + } + + @Override + public void close() throws Exception { + republishGate.countDown(); + scriptable.failParkedRequests(StatusCodes.Bad_NoSubscription); + try { + client.disconnectAsync().get(DISCONNECT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); + } catch (TimeoutException ignored) { + // A client deadlocked inside a blocking Republish recovery cannot run the disconnect it is + // asked for; shutting the Server and the executor down below is what releases it. Tolerated + // here so teardown does not mask the assertion that detected the deadlock. + } finally { + try { + server.shutdown().get(10, TimeUnit.SECONDS); + } finally { + executor.shutdownNow(); + } + } + } + } + + /** + * A single MonitoredItemNotification. The ClientHandle is not registered with the Subscription, + * which is irrelevant here: what matters is that the NotificationMessage carries notification + * data, making it a data message rather than a keep-alive, and that delivery reaches {@code + * onDataReceived}. + */ + private static List dataNotifications() { + return List.of(new MonitoredItemNotification(uint(1), new DataValue(new Variant(0)))); + } + + /** The shape of {@code Stack.sharedExecutor()}: an unbounded cached pool. */ + private static ExecutorService cachedThreadPool() { + return new ThreadPoolExecutor( + 0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS, new SynchronousQueue<>()); + } + + @FunctionalInterface + private interface ThrowingBooleanSupplier { + boolean get() throws Exception; + } + + // endregion +} diff --git a/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SubscriptionAsyncLifecycleTest.java b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SubscriptionAsyncLifecycleTest.java new file mode 100644 index 0000000000..985054f838 --- /dev/null +++ b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SubscriptionAsyncLifecycleTest.java @@ -0,0 +1,541 @@ +/* + * Copyright (c) 2026 the Eclipse Milo Authors + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.milo.opcua.sdk.client.subscriptions; + +import static org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.Unsigned.ubyte; +import static org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.Unsigned.uint; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import java.util.Optional; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; +import org.eclipse.milo.opcua.sdk.client.OpcUaClient; +import org.eclipse.milo.opcua.sdk.server.EndpointConfig; +import org.eclipse.milo.opcua.sdk.server.OpcUaServer; +import org.eclipse.milo.opcua.sdk.test.ScriptableSubscriptionServiceSet; +import org.eclipse.milo.opcua.sdk.test.TestClient; +import org.eclipse.milo.opcua.sdk.test.TestServer; +import org.eclipse.milo.opcua.stack.core.StatusCodes; +import org.eclipse.milo.opcua.stack.core.UaException; +import org.eclipse.milo.opcua.stack.core.types.structured.CreateSubscriptionResponse; +import org.eclipse.milo.opcua.stack.core.util.Unit; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +/** + * What thread an {@link OpcUaSubscription}'s advertised asynchronous lifecycle calls need in order + * to finish. + * + *

{@code createAsync()}, {@code modifyAsync()}, {@code deleteAsync()} and {@code + * setPublishingModeAsync(boolean)} are not composed from the client's asynchronous services. Each + * one hands its blocking counterpart to {@code FutureUtils.supplyAsyncCompose} with the + * transport executor, i.e. {@code CompletableFuture.supplyAsync(supplier, transportExecutor)}, so + * the blocking call runs on the transport executor and waits there for a service response. + * + *

That response is completed by the transport executor: {@code + * AbstractUascClientTransport.handleResponse} dispatches everything that is not a PublishResponse + * via {@code config.getExecutor().execute(...)}. So the call waits on a thread pool for work only + * that same thread pool can do, and it consumes one of its threads while it waits. Two + * consequences, one per nested class below: + * + *

    + *
  1. Self-deadlock. With a single-threaded executor — an ordinary application choice; + * {@code OpcTcpClientTransportConfigBuilder.setExecutor} exists for it — the only thread that + * could complete the response is the one blocked waiting for it. The wedge is permanent + * rather than merely slow: {@code handleResponse} cancels the request timeout the moment the + * response arrives, so the wheel timer that rescues a Server which never answers cannot + * rescue a Server which does. + *
  2. Pool exhaustion. The same arithmetic on any bounded executor: n outstanding + * asynchronous lifecycle calls occupy n threads doing nothing, and once they occupy all of + * them none of their responses can be completed. Two concurrent {@code createAsync()} calls + * are enough on a two-thread pool. + *
+ * + *

The Server is entirely healthy throughout: it answers every request immediately. + * + *

A failure here manifests as a timeout, and that is the correct signal: the defect is + * precisely that a {@link CompletionStage} the SDK handed out never completes and never fails, so + * there is nothing else to observe. Every wait is bounded, both executors use daemon threads, and + * teardown tolerates a client too wedged to disconnect — a RED run leaves a permanently blocked + * thread, and it must not hang the build or leak into another test in the same fork. + */ +public class SubscriptionAsyncLifecycleTest { + + /** + * How long an asynchronous lifecycle call is given to complete. Against a loopback Server every + * one of them is a single round trip that completes in single-digit milliseconds, so this is + * generous by three orders of magnitude — and a self-deadlocked call does not complete at all, so + * no larger value would change an outcome. + */ + private static final long ASYNC_TIMEOUT_MILLIS = 5_000; + + /** + * Long enough that nothing times out on its own: neither a parked Publish request nor a lifecycle + * request in flight. A stage that fails to complete below is therefore the client's own doing and + * not a request timeout. + */ + private static final long REQUEST_TIMEOUT_MILLIS = 60_000; + + /** How long teardown waits for a client that may be wedged to disconnect. */ + private static final long DISCONNECT_TIMEOUT_MILLIS = 5_000; + + /** The PublishingInterval {@code modifyAsync()} is asked to install; the default is 1000ms. */ + private static final double MODIFIED_PUBLISHING_INTERVAL = 2_000.0; + + /** + * One thread, and the response every one of these calls waits for needs that thread to be free. + * + *

Where a test needs a Subscription that already exists, it is created with the + * blocking {@code create()}, called from the test thread: a blocking call made from a + * thread that is not an executor thread completes on any executor, which is what leaves the + * asynchronous call that follows as the only thing under test. + */ + @Nested + class SelfDeadlockOnASingleThreadedExecutor { + + /** Failure manifests as {@code createAsync()} never completing. */ + @Test + void createAsyncCompletesWhenTheExecutorHasOneThread() throws Exception { + try (var fixture = new Fixture(singleThreadExecutor())) { + var subscription = new OpcUaSubscription(fixture.client); + + assertCompletes(subscription.createAsync(), "createAsync()"); + + assertEquals( + OpcUaSubscription.SyncState.SYNCHRONIZED, + subscription.getSyncState(), + "createAsync() completed without creating the Subscription"); + assertTrue( + subscription.getSubscriptionId().isPresent(), + "createAsync() completed without installing the Server-assigned SubscriptionId"); + } + } + + /** Failure manifests as {@code modifyAsync()} never completing. */ + @Test + void modifyAsyncCompletesWhenTheExecutorHasOneThread() throws Exception { + try (var fixture = new Fixture(singleThreadExecutor())) { + OpcUaSubscription subscription = fixture.createdSubscription(); + + subscription.setPublishingInterval(MODIFIED_PUBLISHING_INTERVAL); + assertEquals( + OpcUaSubscription.SyncState.UNSYNCHRONIZED, + subscription.getSyncState(), + "there is nothing pending for modifyAsync() to send, so it would return without" + + " calling the ModifySubscription service and the scenario under test would not" + + " happen"); + + assertCompletes(subscription.modifyAsync(), "modifyAsync()"); + + assertEquals( + OpcUaSubscription.SyncState.SYNCHRONIZED, + subscription.getSyncState(), + "modifyAsync() completed without applying the revised parameters"); + assertEquals( + Optional.of(MODIFIED_PUBLISHING_INTERVAL), + subscription.getRevisedPublishingInterval(), + "modifyAsync() completed without installing the revised PublishingInterval"); + } + } + + /** Failure manifests as {@code setPublishingModeAsync()} never completing. */ + @Test + void setPublishingModeAsyncCompletesWhenTheExecutorHasOneThread() throws Exception { + try (var fixture = new Fixture(singleThreadExecutor())) { + OpcUaSubscription subscription = fixture.createdSubscription(); + + assertCompletes( + subscription.setPublishingModeAsync(false), "setPublishingModeAsync(false)"); + + assertEquals( + Optional.of(false), + subscription.isPublishingEnabled(), + "setPublishingModeAsync(false) completed without recording the new publishing mode"); + } + } + + /** Failure manifests as {@code deleteAsync()} never completing. */ + @Test + void deleteAsyncCompletesWhenTheExecutorHasOneThread() throws Exception { + try (var fixture = new Fixture(singleThreadExecutor())) { + OpcUaSubscription subscription = fixture.createdSubscription(); + + assertCompletes(subscription.deleteAsync(), "deleteAsync()"); + + assertEquals( + OpcUaSubscription.SyncState.INITIAL, + subscription.getSyncState(), + "deleteAsync() completed without discarding the Subscription"); + assertEquals( + 0, + fixture.server.getSubscriptions().size(), + "deleteAsync() completed without deleting the Subscription from the Server"); + } + } + + /** + * The control that isolates the wrapper, rather than the executor, as the cause: the client's + * own asynchronous CreateSubscription service completes on a single-threaded transport + * executor, because nothing occupies that thread while the response is in flight. + * + *

One thread is therefore enough for the round trip {@code createAsync()} makes. What is not + * enough is spending that thread on a blocking wait for it. + */ + @Test + void createSubscriptionAsyncCompletesWhenTheExecutorHasOneThread() throws Exception { + try (var fixture = new Fixture(singleThreadExecutor())) { + CreateSubscriptionResponse response = + fixture + .client + .createSubscriptionAsync(1_000.0, uint(30), uint(10), uint(0), true, ubyte(0)) + .get(ASYNC_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); + + assertTrue( + response.getResponseHeader().getServiceResult().isGood(), + "control: the client's asynchronous CreateSubscription service must complete on a" + + " single-threaded transport executor"); + } + } + } + + /** + * The same arithmetic without the degenerate single-threaded case: a bounded executor loses one + * thread per outstanding asynchronous lifecycle call, and once they are all outstanding at once + * none of them can be answered. + */ + @Nested + class PoolExhaustionOnABoundedExecutor { + + /** + * Two Subscriptions, two threads. Nothing serializes {@code createAsync()} on one Subscription + * against {@code createAsync()} on another — they are separate objects with separate lifecycle + * locks — so both are legitimately outstanding at once, and between them they hold every thread + * that could complete either response. + * + *

Failure manifests as the first of the two never completing. + */ + @Test + void concurrentCreateAsyncCallsCompleteWhenTheExecutorHasTwoThreads() throws Exception { + try (var fixture = new Fixture(fixedThreadPool(2))) { + var first = new OpcUaSubscription(fixture.client); + var second = new OpcUaSubscription(fixture.client); + + // Both submitted before either is awaited: the point is that they are outstanding together. + CompletionStage firstCreate = first.createAsync(); + CompletionStage secondCreate = second.createAsync(); + + assertCompletes(firstCreate, "the first of two concurrent createAsync() calls"); + assertCompletes(secondCreate, "the second of two concurrent createAsync() calls"); + + assertEquals( + 2, + fixture.server.getSubscriptions().size(), + "two concurrent createAsync() calls must create two Subscriptions on the Server"); + } + } + } + + /** + * The controls that fix every assertion above in place: the identical calls against an unbounded + * cached pool, which is the shape of the executor the client uses by default ({@code + * Stack.sharedExecutor()}). Each of them has a thread to spare for the response it is waiting + * for, so each completes today. + * + *

Without these, the assertions above could be measuring a broken fixture — a Server that + * never answers, a Subscription that was never created — rather than executor starvation. + */ + @Nested + class ControlsOnAnUnboundedExecutor { + + @Test + void createAsyncCompletesWhenTheExecutorHasThreadsToSpare() throws Exception { + try (var fixture = new Fixture(cachedThreadPool())) { + var subscription = new OpcUaSubscription(fixture.client); + + assertCompletes(subscription.createAsync(), "control: createAsync()"); + + assertEquals( + OpcUaSubscription.SyncState.SYNCHRONIZED, + subscription.getSyncState(), + "control: createAsync() must create the Subscription"); + assertTrue( + subscription.getSubscriptionId().isPresent(), + "control: createAsync() must install the Server-assigned SubscriptionId"); + } + } + + @Test + void modifyAsyncCompletesWhenTheExecutorHasThreadsToSpare() throws Exception { + try (var fixture = new Fixture(cachedThreadPool())) { + OpcUaSubscription subscription = fixture.createdSubscription(); + + subscription.setPublishingInterval(MODIFIED_PUBLISHING_INTERVAL); + + assertCompletes(subscription.modifyAsync(), "control: modifyAsync()"); + + assertEquals( + OpcUaSubscription.SyncState.SYNCHRONIZED, + subscription.getSyncState(), + "control: modifyAsync() must apply the revised parameters"); + assertEquals( + Optional.of(MODIFIED_PUBLISHING_INTERVAL), + subscription.getRevisedPublishingInterval(), + "control: modifyAsync() must install the revised PublishingInterval"); + } + } + + @Test + void setPublishingModeAsyncCompletesWhenTheExecutorHasThreadsToSpare() throws Exception { + try (var fixture = new Fixture(cachedThreadPool())) { + OpcUaSubscription subscription = fixture.createdSubscription(); + + assertCompletes( + subscription.setPublishingModeAsync(false), "control: setPublishingModeAsync(false)"); + + assertEquals( + Optional.of(false), + subscription.isPublishingEnabled(), + "control: setPublishingModeAsync(false) must record the new publishing mode"); + } + } + + @Test + void deleteAsyncCompletesWhenTheExecutorHasThreadsToSpare() throws Exception { + try (var fixture = new Fixture(cachedThreadPool())) { + OpcUaSubscription subscription = fixture.createdSubscription(); + + assertCompletes(subscription.deleteAsync(), "control: deleteAsync()"); + + assertEquals( + OpcUaSubscription.SyncState.INITIAL, + subscription.getSyncState(), + "control: deleteAsync() must discard the Subscription"); + assertEquals( + 0, + fixture.server.getSubscriptions().size(), + "control: deleteAsync() must delete the Subscription from the Server"); + } + } + + /** + * The control for the contract each of these stages has to keep on the way to completing: an + * asynchronous lifecycle call must complete exceptionally with exactly the {@link UaException} + * its blocking counterpart throws. Here that is {@code Bad_InvalidState} for a {@code + * createAsync()} on a Subscription that already exists (Part 4 §5.13.2 has no notion of + * creating one twice). + * + *

This passes today. It is here because composing these calls from the client's asynchronous + * services rewrites the path an error takes out of them, and an existing caller must not start + * seeing a different failure — or a success. + */ + @Test + void createAsyncFailsWithBadInvalidStateWhenTheSubscriptionAlreadyExists() throws Exception { + try (var fixture = new Fixture(cachedThreadPool())) { + OpcUaSubscription subscription = fixture.createdSubscription(); + + ExecutionException failure = + assertThrows( + ExecutionException.class, + () -> + subscription + .createAsync() + .toCompletableFuture() + .get(ASYNC_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), + "control: createAsync() on a Subscription that already exists must complete" + + " exceptionally"); + + UaException cause = + assertInstanceOf( + UaException.class, + failure.getCause(), + "control: createAsync() must report the same UaException create() throws"); + assertEquals( + StatusCodes.Bad_InvalidState, + cause.getStatusCode().value(), + "control: createAsync() on a Subscription that already exists must fail with" + + " Bad_InvalidState"); + } + } + + @Test + void concurrentCreateAsyncCallsCompleteWhenTheExecutorHasThreadsToSpare() throws Exception { + try (var fixture = new Fixture(cachedThreadPool())) { + var first = new OpcUaSubscription(fixture.client); + var second = new OpcUaSubscription(fixture.client); + + CompletionStage firstCreate = first.createAsync(); + CompletionStage secondCreate = second.createAsync(); + + assertCompletes(firstCreate, "control: the first of two concurrent createAsync() calls"); + assertCompletes(secondCreate, "control: the second of two concurrent createAsync() calls"); + + assertEquals( + 2, + fixture.server.getSubscriptions().size(), + "control: two concurrent createAsync() calls must create two Subscriptions on the" + + " Server"); + } + } + } + + // region helpers + + /** + * Wait for an asynchronous lifecycle call to finish, and fail the test if it does not. + * + *

A call that has wedged the transport executor neither completes nor completes exceptionally, + * so {@link #ASYNC_TIMEOUT_MILLIS} elapsing is this assertion failing rather than the test + * being impatient. The wait is bounded so that a RED run reports a failure instead of hanging. + * + * @param stage the {@link CompletionStage} returned by the call under test. + * @param call how to name that call in the failure message. + */ + private static void assertCompletes(CompletionStage stage, String call) throws Exception { + try { + stage.toCompletableFuture().get(ASYNC_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + fail( + call + + " did not complete within " + + ASYNC_TIMEOUT_MILLIS + + " ms against a Server that answers immediately: the blocking call it wraps was" + + " dispatched onto the transport executor and is waiting there for a response that" + + " only the transport executor can complete"); + } + } + + /** + * A running Server whose Publish responses are parked — so the only requests that matter during a + * test are the lifecycle ones it makes — and a connected client driven by a caller-supplied + * {@link ExecutorService}. + * + *

CreateSubscription, ModifySubscription, DeleteSubscriptions and SetPublishingMode are + * handled by the real Server implementation, so every assertion about what the client installed + * is an assertion about a real service round trip. + */ + private static final class Fixture implements AutoCloseable { + + private final ExecutorService executor; + private final OpcUaServer server; + private final OpcUaClient client; + private final ScriptableSubscriptionServiceSet scriptable; + + Fixture(ExecutorService executor) throws Exception { + this.executor = executor; + + TestServer testServer = TestServer.create(); + server = testServer.getServer(); + + scriptable = new ScriptableSubscriptionServiceSet(server); + for (EndpointConfig endpoint : server.getConfig().getEndpoints()) { + server.addServiceSet(endpoint.getPath(), scriptable); + } + + server.startup().get(); + + client = + TestClient.create( + server, + transportConfig -> transportConfig.setExecutor(executor), + cfg -> + cfg + // Long request timeout so a parked Publish request does not time out, and so + // a stalled lifecycle call is not rescued by one. + .setRequestTimeout(uint(REQUEST_TIMEOUT_MILLIS)) + // No Session keep-alive traffic competing for the executor. + .setKeepAliveInterval(uint(REQUEST_TIMEOUT_MILLIS))); + client.connect(); + } + + /** + * Create a Subscription with the blocking {@code create()}, called from the test thread. + * + * @return the created {@link OpcUaSubscription}. + */ + OpcUaSubscription createdSubscription() throws UaException { + var subscription = new OpcUaSubscription(client); + subscription.create(); + + return subscription; + } + + @Override + public void close() throws Exception { + scriptable.failParkedRequests(StatusCodes.Bad_NoSubscription); + try { + client.disconnectAsync().get(DISCONNECT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); + } catch (TimeoutException ignored) { + // A client whose executor is wedged by the call under test cannot run the disconnect it is + // asked for. Tolerated so teardown does not mask the assertion that detected the wedge; + // shutting the Server down and then interrupting the executor below releases the blocked + // thread, and every thread involved is a daemon. + } finally { + try { + server.shutdown().get(10, TimeUnit.SECONDS); + } finally { + executor.shutdownNow(); + } + } + } + } + + /** + * The shape of {@code Stack.sharedExecutor()}: an unbounded cached pool. Daemon threads, so a + * task left blocked by a failing test cannot keep the JVM alive. + */ + private static ExecutorService cachedThreadPool() { + return new ThreadPoolExecutor( + 0, + Integer.MAX_VALUE, + 60L, + TimeUnit.SECONDS, + new SynchronousQueue<>(), + daemonThreadFactory("async-lifecycle-cached")); + } + + /** + * A single-threaded transport executor, which {@code + * OpcTcpClientTransportConfigBuilder.setExecutor} exists to allow. + */ + private static ExecutorService singleThreadExecutor() { + return Executors.newSingleThreadExecutor(daemonThreadFactory("async-lifecycle-single")); + } + + /** A transport executor with a hard upper bound on the number of threads. */ + private static ExecutorService fixedThreadPool(int threads) { + return Executors.newFixedThreadPool(threads, daemonThreadFactory("async-lifecycle-bounded")); + } + + private static ThreadFactory daemonThreadFactory(String namePrefix) { + var counter = new AtomicInteger(0); + + return runnable -> { + var thread = new Thread(runnable, namePrefix + "-" + counter.incrementAndGet()); + thread.setDaemon(true); + + return thread; + }; + } + + // endregion +} diff --git a/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SubscriptionIdentityTest.java b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SubscriptionIdentityTest.java new file mode 100644 index 0000000000..11fa8a555a --- /dev/null +++ b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SubscriptionIdentityTest.java @@ -0,0 +1,756 @@ +/* + * Copyright (c) 2026 the Eclipse Milo Authors + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.milo.opcua.sdk.client.subscriptions; + +import static org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.Unsigned.uint; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.eclipse.milo.opcua.sdk.client.OpcUaClient; +import org.eclipse.milo.opcua.sdk.server.EndpointConfig; +import org.eclipse.milo.opcua.sdk.server.OpcUaServer; +import org.eclipse.milo.opcua.sdk.test.ScriptableSubscriptionServiceSet; +import org.eclipse.milo.opcua.sdk.test.TestClient; +import org.eclipse.milo.opcua.sdk.test.TestServer; +import org.eclipse.milo.opcua.stack.core.NodeIds; +import org.eclipse.milo.opcua.stack.core.StatusCodes; +import org.eclipse.milo.opcua.stack.core.UaException; +import org.eclipse.milo.opcua.stack.core.types.builtin.DataValue; +import org.eclipse.milo.opcua.stack.core.types.builtin.DiagnosticInfo; +import org.eclipse.milo.opcua.stack.core.types.builtin.ExtensionObject; +import org.eclipse.milo.opcua.stack.core.types.builtin.StatusCode; +import org.eclipse.milo.opcua.stack.core.types.builtin.Variant; +import org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.UInteger; +import org.eclipse.milo.opcua.stack.core.types.structured.CreateSubscriptionRequest; +import org.eclipse.milo.opcua.stack.core.types.structured.CreateSubscriptionResponse; +import org.eclipse.milo.opcua.stack.core.types.structured.MonitoredItemNotification; +import org.eclipse.milo.opcua.stack.core.types.structured.StatusChangeNotification; +import org.eclipse.milo.opcua.stack.core.types.structured.SubscriptionAcknowledgement; +import org.eclipse.milo.opcua.stack.transport.server.ServiceRequestContext; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +/** + * Which Subscription a piece of {@code PublishingManager} work belongs to. + * + *

A {@code SubscriptionDetails} entry is registered under the SubscriptionId the {@link + * OpcUaSubscription} had at registration time, but it carries only the mutable Subscription object. + * Everything that later needs an id — removing the entry when the Server reports Bad_Timeout, + * building SubscriptionAcknowledgements, delivering a NotificationMessage — asks that object for + * its current id. The object's id is not stable: {@link OpcUaSubscription#reset()} clears it + * and {@link OpcUaSubscription#create()} installs a new one, and neither is serialized against work + * already queued for the previous incarnation. + * + *

The tests below drive the two ways an entry's id and its Subscription's current id can + * disagree, and assert the invariant each case breaks: work belonging to one incarnation of a + * Subscription must never be applied to another. + */ +public class SubscriptionIdentityTest { + + /** How long to wait for something that must happen. */ + private static final long AWAIT_TIMEOUT_MILLIS = 10_000; + + /** + * How long to watch for stale work that must not take effect. Every task involved is already + * queued when the window opens and runs as soon as the delivery queue is released, so this is + * generous by three orders of magnitude. + */ + private static final long STALE_WORK_WINDOW_MILLIS = 3_000; + + /** + * Long enough that nothing times out on its own: a parked Publish request, and a + * CreateSubscription held at the gate below, must stay held until the test releases them. + */ + private static final long REQUEST_TIMEOUT_MILLIS = 60_000; + + /** The client keeps one more PublishRequest in flight than it has Subscriptions. */ + private static final int PIPELINE_DEPTH = 2; + + private static final DataValue CURRENT_VALUE = new DataValue(Variant.ofInt32(1)); + private static final DataValue STALE_VALUE = new DataValue(Variant.ofInt32(2)); + + /** + * A NotificationMessage is processed on the Subscription's processing queue and then handed to + * its delivery queue, which is a {@code final} field of {@link OpcUaSubscription} that {@link + * OpcUaSubscription#reset()} neither drains nor replaces. An application callback that has not + * yet returned therefore holds behind it work belonging to a Subscription incarnation that may be + * gone by the time it runs — and that work addresses the live object, so it acts on whatever + * incarnation exists then. + * + *

Part 4 §5.13.1.1 makes the SubscriptionId "the Server-assigned identifier for the + * Subscription": a NotificationMessage received for one is not a statement about any other, and + * two Subscriptions created in sequence are two different Subscriptions even when the same client + * object represents them. + */ + @Nested + class StaleWorkFromAPreviousIncarnation { + + /** + * The traced failure: a Bad_Timeout StatusChangeNotification received for the previous + * Subscription removes the new Subscription's registration (it asks the live object for + * its current id) and then resets it (Bad_Timeout resets the Subscription the notification is + * delivered on). The application is left holding a Subscription object that reports INITIAL and + * has no SubscriptionId, while the Subscription it just created is still alive on the Server — + * and unreachable, because {@link OpcUaSubscription#delete()} needs the ServerState that reset + * discarded. + */ + @Test + void staleStatusChangeDoesNotTearDownTheRecreatedSubscription() throws Exception { + try (var fixture = new Fixture()) { + OpcUaSubscription subscription = fixture.createSubscription(); + OpcUaMonitoredItem item = fixture.addDataItem(subscription); + + var listener = new BlockingDeliveryListener(); + subscription.setSubscriptionListener(listener); + + UInteger idA = subscriptionId(subscription); + fixture.awaitPipelineFilled(); + + // Hold the delivery queue open with a notification the application never finishes handling. + fixture.scriptable.enqueueDataChange(idA, 1, notifications(item, CURRENT_VALUE), uint(1)); + assertTrue( + listener.awaitDeliveryStarted(AWAIT_TIMEOUT_MILLIS), + "the first notification was never delivered, so the delivery queue is not held open"); + + // Queue the Bad_Timeout status change behind it. It is processed now, delivered later. + fixture.scriptable.enqueueNotification(idA, 2, badTimeout(fixture.scriptable), uint(2)); + assertTrue( + awaitTrue( + () -> subscription.getDeliveryQueue().getQueueSize() == 1, AWAIT_TIMEOUT_MILLIS), + "the StatusChangeNotification was never queued behind the blocked delivery, so the" + + " scenario under test did not happen"); + + // The application discards the Subscription and creates another one. + subscription.reset(); + subscription.create(); + + UInteger idB = subscriptionId(subscription); + assertNotEquals( + idA, idB, "the Server reused the SubscriptionId, so nothing distinguishes the two"); + + listener.release(); + + assertFalse( + awaitTrue( + () -> subscription.getSyncState() == OpcUaSubscription.SyncState.INITIAL, + STALE_WORK_WINDOW_MILLIS), + "a Bad_Timeout StatusChangeNotification received for Subscription " + + idA + + " reset Subscription " + + idB + + ", which was created after it and has never timed out: the queued delivery" + + " task asks the live Subscription object for its current SubscriptionId, so it" + + " de-registers and tears down whatever incarnation exists when it finally runs"); + + assertEquals( + Optional.of(idB), + subscription.getSubscriptionId(), + "the Subscription created after the stale notification lost its ServerState, so" + + " delete() can no longer name it and the Server-side Subscription is" + + " unreachable"); + } + } + + /** + * The control that isolates the stale StatusChangeNotification as the cause: the identical + * script with nothing queued behind the blocked delivery leaves the recreated Subscription + * intact. + */ + @Test + void recreatedSubscriptionSurvivesWhenNoStaleWorkIsQueued() throws Exception { + try (var fixture = new Fixture()) { + OpcUaSubscription subscription = fixture.createSubscription(); + OpcUaMonitoredItem item = fixture.addDataItem(subscription); + + var listener = new BlockingDeliveryListener(); + subscription.setSubscriptionListener(listener); + + UInteger idA = subscriptionId(subscription); + fixture.awaitPipelineFilled(); + + fixture.scriptable.enqueueDataChange(idA, 1, notifications(item, CURRENT_VALUE), uint(1)); + assertTrue( + listener.awaitDeliveryStarted(AWAIT_TIMEOUT_MILLIS), + "the first notification was never delivered"); + + subscription.reset(); + subscription.create(); + + UInteger idB = subscriptionId(subscription); + + listener.release(); + + assertFalse( + awaitTrue( + () -> subscription.getSyncState() == OpcUaSubscription.SyncState.INITIAL, + STALE_WORK_WINDOW_MILLIS), + "control: reset() followed by create() must leave a usable Subscription"); + assertEquals( + Optional.of(idB), + subscription.getSubscriptionId(), + "control: reset() followed by create() must leave a usable Subscription"); + } + } + + /** + * The control that keeps the assertion above from being vacuous in the other direction: a + * Bad_Timeout StatusChangeNotification delivered to the Subscription it was received for does + * reset it. Part 4 §5.13.1.1 — the Subscription no longer exists on the Server — so the reset + * is required, and the defect is only that the reset lands on the wrong incarnation. + */ + @Test + void statusChangeResetsTheSubscriptionItWasReceivedFor() throws Exception { + try (var fixture = new Fixture()) { + OpcUaSubscription subscription = fixture.createSubscription(); + + var statusChanged = new CountDownLatch(1); + subscription.setSubscriptionListener( + new OpcUaSubscription.SubscriptionListener() { + @Override + public void onStatusChanged(OpcUaSubscription s, StatusCode status) { + statusChanged.countDown(); + } + }); + + UInteger idA = subscriptionId(subscription); + fixture.awaitPipelineFilled(); + + fixture.scriptable.enqueueNotification(idA, 1, badTimeout(fixture.scriptable), uint(1)); + + assertTrue( + statusChanged.await(AWAIT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), + "control: the Bad_Timeout StatusChangeNotification was never delivered"); + assertEquals( + OpcUaSubscription.SyncState.INITIAL, + subscription.getSyncState(), + "control: a Bad_Timeout StatusChangeNotification must reset the Subscription it was" + + " received for"); + } + } + + /** + * The same interleaving with a DataChangeNotification instead of a status change. {@link + * OpcUaSubscription#reset()} leaves the MonitoredItem map and every ClientHandle in it + * untouched, so a value received for the discarded Subscription still resolves to an item and + * is handed to the application as though it had arrived for the Subscription created since. + */ + @Test + void staleDataChangeIsNotDeliveredAfterTheSubscriptionIsRecreated() throws Exception { + try (var fixture = new Fixture()) { + OpcUaSubscription subscription = fixture.createSubscription(); + OpcUaMonitoredItem item = fixture.addDataItem(subscription); + + var listener = new BlockingDeliveryListener(); + subscription.setSubscriptionListener(listener); + + UInteger idA = subscriptionId(subscription); + fixture.awaitPipelineFilled(); + + fixture.scriptable.enqueueDataChange(idA, 1, notifications(item, CURRENT_VALUE), uint(1)); + assertTrue( + listener.awaitDeliveryStarted(AWAIT_TIMEOUT_MILLIS), + "the first notification was never delivered, so the delivery queue is not held open"); + + fixture.scriptable.enqueueDataChange(idA, 2, notifications(item, STALE_VALUE), uint(2)); + assertTrue( + awaitTrue( + () -> subscription.getDeliveryQueue().getQueueSize() == 1, AWAIT_TIMEOUT_MILLIS), + "the second DataChangeNotification was never queued behind the blocked delivery, so" + + " the scenario under test did not happen"); + + subscription.reset(); + subscription.create(); + + UInteger idB = subscriptionId(subscription); + assertNotEquals( + idA, idB, "the Server reused the SubscriptionId, so nothing distinguishes the two"); + + listener.release(); + + assertFalse( + awaitTrue(() -> listener.received(STALE_VALUE), STALE_WORK_WINDOW_MILLIS), + "a DataChangeNotification received for Subscription " + + idA + + " was delivered to the application after that Subscription had been discarded" + + " and Subscription " + + idB + + " created in its place: the application is told a MonitoredItem of the new" + + " Subscription has a value the new Subscription has never reported"); + } + } + + /** + * The control for the test above: with no reset and no recreate, the second + * DataChangeNotification is delivered. Without it the assertion there could hold simply because + * the script never produced a second delivery. + */ + @Test + void queuedDataChangeIsDeliveredWhenTheSubscriptionIsNotRecreated() throws Exception { + try (var fixture = new Fixture()) { + OpcUaSubscription subscription = fixture.createSubscription(); + OpcUaMonitoredItem item = fixture.addDataItem(subscription); + + var listener = new BlockingDeliveryListener(); + subscription.setSubscriptionListener(listener); + + UInteger idA = subscriptionId(subscription); + fixture.awaitPipelineFilled(); + + fixture.scriptable.enqueueDataChange(idA, 1, notifications(item, CURRENT_VALUE), uint(1)); + assertTrue( + listener.awaitDeliveryStarted(AWAIT_TIMEOUT_MILLIS), + "the first notification was never delivered"); + + fixture.scriptable.enqueueDataChange(idA, 2, notifications(item, STALE_VALUE), uint(2)); + + listener.release(); + + assertTrue( + awaitTrue(() -> listener.received(STALE_VALUE), AWAIT_TIMEOUT_MILLIS), + "control: a DataChangeNotification queued behind a blocked delivery must be delivered" + + " once the delivery queue drains"); + } + } + } + + /** + * {@link OpcUaSubscription#create()} is a check-then-act around a blocking service call: it tests + * {@code syncState == INITIAL}, calls CreateSubscription, and only then publishes the + * ServerState. Nothing serializes it, so two concurrent calls both pass the check and both create + * a Subscription on the Server, while the object keeps only the ServerState written last. + * + *

The Subscription whose id was overwritten is not forgotten everywhere: {@code + * PublishingManager} still holds an entry registered under it, and that entry answers questions + * about its id with the live object's current one. + */ + @Nested + class ConcurrentCreate { + + /** + * Part 4 §5.14.5.2 defines a SubscriptionAcknowledgement as a subscriptionId and the + * sequenceNumber of a NotificationMessage "received on the Subscription", and lets the Server + * "delete the Message with this sequence number from its retransmission queue". Sending the + * sequence number of one Subscription's NotificationMessage under another Subscription's id + * therefore acknowledges a message that was never received on that Subscription, while the + * message that was received is never acknowledged and stays in the Server's + * retransmission queue. + */ + @Test + void acknowledgementCarriesTheIdItsNotificationWasReceivedUnder() throws Exception { + try (var fixture = new Fixture()) { + var subscription = new OpcUaSubscription(fixture.client); + + UInteger idA = fixture.createConcurrently(subscription); + OpcUaMonitoredItem item = fixture.addDataItem(subscription); + + fixture.scriptable.enqueueDataChange(idA, 1, notifications(item, CURRENT_VALUE), uint(1)); + + assertTrue( + awaitTrue(() -> fixture.acknowledgementOfSequence(1).isPresent(), AWAIT_TIMEOUT_MILLIS), + "the client never acknowledged sequence 1, so there is no acknowledgement to inspect"); + + assertEquals( + idA, + fixture.acknowledgementOfSequence(1).orElseThrow().getSubscriptionId(), + "the NotificationMessage was received on Subscription " + + idA + + " but acknowledged under Subscription " + + subscription.getSubscriptionId().orElse(null) + + ": the acknowledgement is built from the live Subscription object's current id" + + " rather than the id the PublishingManager entry is registered under"); + } + } + + /** The control: after a single create() the acknowledgement carries that Subscription's id. */ + @Test + void acknowledgementCarriesTheSubscriptionIdAfterASingleCreate() throws Exception { + try (var fixture = new Fixture()) { + OpcUaSubscription subscription = fixture.createSubscription(); + OpcUaMonitoredItem item = fixture.addDataItem(subscription); + + UInteger id = subscriptionId(subscription); + fixture.awaitPipelineFilled(); + + fixture.scriptable.enqueueDataChange(id, 1, notifications(item, CURRENT_VALUE), uint(1)); + + assertTrue( + awaitTrue(() -> fixture.acknowledgementOfSequence(1).isPresent(), AWAIT_TIMEOUT_MILLIS), + "control: the client never acknowledged sequence 1"); + assertEquals( + id, + fixture.acknowledgementOfSequence(1).orElseThrow().getSubscriptionId(), + "control: the acknowledgement must carry the id of the Subscription the" + + " NotificationMessage was received on"); + } + } + + /** + * Whatever an {@link OpcUaSubscription} created on the Server, {@link + * OpcUaSubscription#delete()} has to be able to delete: a Server-side Subscription no client + * object can name is a resource leak that survives until its lifetime expires, and Part 4 + * §5.13.8 gives DeleteSubscriptions the SubscriptionIds as its only handle on them. + * + *

A second, concurrent create() overwrites the ServerState holding the first SubscriptionId, + * so delete() names only the second and the first is left running. + */ + @Test + void deleteRemovesEveryServerSideSubscriptionTheClientCreated() throws Exception { + try (var fixture = new Fixture()) { + var subscription = new OpcUaSubscription(fixture.client); + + UInteger idA = fixture.createConcurrently(subscription); + UInteger idB = subscriptionId(subscription); + + subscription.delete(); + + assertEquals( + Set.of(), + Set.copyOf(fixture.server.getSubscriptions().keySet()), + "delete() left a Subscription running on the Server. Two concurrent create() calls" + + " each created one (" + + idA + + " and " + + idB + + "), the second overwrote the ServerState holding the first, and nothing the" + + " client can reach still names it"); + } + } + + /** The control: a Subscription created once is removed from the Server by delete(). */ + @Test + void deleteRemovesTheServerSideSubscriptionAfterASingleCreate() throws Exception { + try (var fixture = new Fixture()) { + OpcUaSubscription subscription = fixture.createSubscription(); + + subscription.delete(); + + assertEquals( + Set.of(), + Set.copyOf(fixture.server.getSubscriptions().keySet()), + "control: delete() must remove the Subscription from the Server"); + } + } + } + + // region helpers + + private static UInteger subscriptionId(OpcUaSubscription subscription) { + return subscription.getSubscriptionId().orElseThrow(); + } + + private static List notifications( + OpcUaMonitoredItem item, DataValue value) { + + return List.of(new MonitoredItemNotification(item.getClientHandle().orElseThrow(), value)); + } + + /** A NotificationMessage body reporting that the Subscription has timed out. */ + private static ExtensionObject[] badTimeout(ScriptableSubscriptionServiceSet scriptable) { + return new ExtensionObject[] { + scriptable.encode( + new StatusChangeNotification( + new StatusCode(StatusCodes.Bad_Timeout), DiagnosticInfo.NULL_VALUE)) + }; + } + + /** + * A listener that suspends the Subscription's delivery queue inside the first {@code + * onDataReceived} callback, so a test can queue further work behind it and choose when that work + * runs. Every DataValue handed to the callback is recorded, including the one delivered while it + * is suspended. + */ + private static final class BlockingDeliveryListener + implements OpcUaSubscription.SubscriptionListener { + + private final List received = Collections.synchronizedList(new ArrayList<>()); + private final CountDownLatch deliveryStarted = new CountDownLatch(1); + private final CountDownLatch release = new CountDownLatch(1); + + @Override + public void onDataReceived( + OpcUaSubscription subscription, List items, List values) { + + received.addAll(values); + + if (deliveryStarted.getCount() > 0) { + deliveryStarted.countDown(); + + try { + if (!release.await(AWAIT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { + throw new IllegalStateException("the delivery queue was never released"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } + } + } + + boolean awaitDeliveryStarted(long timeoutMillis) throws InterruptedException { + return deliveryStarted.await(timeoutMillis, TimeUnit.MILLISECONDS); + } + + void release() { + release.countDown(); + } + + boolean received(DataValue value) { + synchronized (received) { + return received.stream().anyMatch(v -> v.getValue().equals(value.getValue())); + } + } + } + + /** + * A {@link ScriptableSubscriptionServiceSet} that can hold CreateSubscription requests at the + * Server and release them one at a time. + * + *

Holding them inside the Server handler is what makes the overlap real: the two client + * threads have each sent their request, so each has already passed {@code create()}'s {@code + * syncState == INITIAL} check, and the order in which their ServerStates are published is the + * order the test releases them in. + */ + private static final class GatedCreateServiceSet extends ScriptableSubscriptionServiceSet { + + private final AtomicInteger arrivals = new AtomicInteger(0); + private final CountDownLatch releaseFirst = new CountDownLatch(1); + private final CountDownLatch releaseSecond = new CountDownLatch(1); + + private volatile boolean gated = false; + + GatedCreateServiceSet(OpcUaServer server) { + super(server); + } + + void gateCreates() { + gated = true; + } + + int arrivedCreateCount() { + return arrivals.get(); + } + + void releaseFirstCreate() { + releaseFirst.countDown(); + } + + void releaseSecondCreate() { + releaseSecond.countDown(); + } + + void releaseAllCreates() { + releaseFirst.countDown(); + releaseSecond.countDown(); + } + + @Override + public CreateSubscriptionResponse onCreateSubscription( + ServiceRequestContext context, CreateSubscriptionRequest request) throws UaException { + + if (gated) { + int index = arrivals.getAndIncrement(); + + await(index == 0 ? releaseFirst : releaseSecond); + } + + return super.onCreateSubscription(context, request); + } + + private static void await(CountDownLatch gate) throws UaException { + try { + if (!gate.await(AWAIT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { + throw new UaException( + StatusCodes.Bad_Timeout, "the CreateSubscription gate was never released"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new UaException(StatusCodes.Bad_UnexpectedError, e); + } + } + } + + /** + * A running Server whose Publish responses are scripted and whose CreateSubscription responses + * can be gated, plus a connected client. + */ + private static final class Fixture implements AutoCloseable { + + private final OpcUaServer server; + private final OpcUaClient client; + private final GatedCreateServiceSet scriptable; + + Fixture() throws Exception { + TestServer testServer = TestServer.create(); + server = testServer.getServer(); + + scriptable = new GatedCreateServiceSet(server); + for (EndpointConfig endpoint : server.getConfig().getEndpoints()) { + server.addServiceSet(endpoint.getPath(), scriptable); + } + + server.startup().get(); + + client = + TestClient.create( + server, + cfg -> + cfg + // Long request timeout so parked or gated requests do not time out. + .setRequestTimeout(uint(REQUEST_TIMEOUT_MILLIS)) + // No Session keep-alive traffic: the only requests in flight during a test + // are the ones it scripts. + .setKeepAliveInterval(uint(REQUEST_TIMEOUT_MILLIS))); + client.connect(); + } + + OpcUaSubscription createSubscription() throws UaException { + var subscription = new OpcUaSubscription(client); + subscription.create(); + + return subscription; + } + + /** + * Add a MonitoredItem to {@code subscription}, client-side only: {@link + * OpcUaSubscription#addMonitoredItem} assigns the ClientHandle a scripted notification is + * looked up by, and no Server-side item participates in delivering one. + */ + OpcUaMonitoredItem addDataItem(OpcUaSubscription subscription) { + OpcUaMonitoredItem item = + OpcUaMonitoredItem.newDataItem(NodeIds.Server_ServerStatus_CurrentTime); + subscription.addMonitoredItem(item); + + return item; + } + + void awaitPipelineFilled() throws Exception { + assertTrue( + awaitTrue( + () -> scriptable.getParkedRequestCount() >= PIPELINE_DEPTH, AWAIT_TIMEOUT_MILLIS), + "the client did not fill its Publish pipeline"); + } + + /** + * Run two overlapping {@code create()} calls on {@code subscription} and return the + * SubscriptionId the Server assigned the first of them, i.e. the one whose ServerState the + * second overwrites. + * + *

The second call is started only once the first is inside the Server's CreateSubscription + * handler, so the overlap is fixed rather than raced. Neither call is required to reach the + * Server: an implementation that serializes {@code create()} rejects the second before it sends + * anything, and the invariants asserted by the callers hold just as well then. + */ + UInteger createConcurrently(OpcUaSubscription subscription) throws Exception { + scriptable.gateCreates(); + + var failures = new CopyOnWriteArrayList(); + Runnable create = + () -> { + try { + subscription.create(); + } catch (Exception e) { + failures.add(e); + } + }; + + var first = new Thread(create, "create-first"); + var second = new Thread(create, "create-second"); + + first.start(); + assertTrue( + awaitTrue(() -> scriptable.arrivedCreateCount() >= 1, AWAIT_TIMEOUT_MILLIS), + "the first create() never reached the Server"); + + second.start(); + awaitTrue( + () -> scriptable.arrivedCreateCount() >= 2 || !second.isAlive(), + STALE_WORK_WINDOW_MILLIS); + + scriptable.releaseFirstCreate(); + assertTrue( + awaitTrue(() -> subscription.getSubscriptionId().isPresent(), AWAIT_TIMEOUT_MILLIS), + "the first create() never completed: " + failures); + + UInteger firstId = subscriptionId(subscription); + + // Publish traffic starts only once a Subscription is registered with the PublishingManager, + // so a PublishRequest is proof that the first create() finished registering under firstId. + assertTrue( + awaitTrue(() -> scriptable.getPublishRequestCount() >= 1, AWAIT_TIMEOUT_MILLIS), + "the first create() never registered its Subscription with the PublishingManager"); + + scriptable.releaseSecondCreate(); + + join(first); + join(second); + + return firstId; + } + + /** The first acknowledgement of {@code sequenceNumber} the Server received, if any. */ + Optional acknowledgementOfSequence(long sequenceNumber) { + return scriptable.getReceivedAcknowledgements().stream() + .filter(ack -> ack.getSequenceNumber().longValue() == sequenceNumber) + .findFirst(); + } + + @Override + public void close() throws Exception { + scriptable.releaseAllCreates(); + scriptable.failParkedRequests(StatusCodes.Bad_NoSubscription); + try { + client.disconnectAsync().get(5, TimeUnit.SECONDS); + } finally { + server.shutdown().get(10, TimeUnit.SECONDS); + } + } + } + + private static void join(Thread thread) throws InterruptedException { + thread.join(AWAIT_TIMEOUT_MILLIS); + + assertFalse(thread.isAlive(), thread.getName() + " did not finish"); + } + + /** Polls {@code condition} until it holds or {@code timeoutMillis} elapses. */ + private static boolean awaitTrue(ThrowingBooleanSupplier condition, long timeoutMillis) + throws Exception { + + long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(timeoutMillis); + + while (System.nanoTime() < deadline) { + if (condition.get()) { + return true; + } + Thread.sleep(25); + } + + return condition.get(); + } + + @FunctionalInterface + private interface ThrowingBooleanSupplier { + boolean get() throws Exception; + } + + // endregion +} diff --git a/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SubscriptionKeepAliveDerivationTest.java b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SubscriptionKeepAliveDerivationTest.java new file mode 100644 index 0000000000..c1456c0f3d --- /dev/null +++ b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SubscriptionKeepAliveDerivationTest.java @@ -0,0 +1,188 @@ +/* + * Copyright (c) 2026 the Eclipse Milo Authors + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.milo.opcua.sdk.client.subscriptions; + +import static org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.Unsigned.uint; +import static org.junit.jupiter.api.Assertions.assertAll; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import java.util.concurrent.TimeUnit; +import org.eclipse.milo.opcua.sdk.client.OpcUaClient; +import org.eclipse.milo.opcua.sdk.server.EndpointConfig; +import org.eclipse.milo.opcua.sdk.server.OpcUaServer; +import org.eclipse.milo.opcua.sdk.test.DelegatingSubscriptionServiceSet; +import org.eclipse.milo.opcua.sdk.test.TestClient; +import org.eclipse.milo.opcua.sdk.test.TestServer; +import org.eclipse.milo.opcua.stack.core.UaException; +import org.eclipse.milo.opcua.stack.core.types.structured.CreateSubscriptionRequest; +import org.eclipse.milo.opcua.stack.core.types.structured.CreateSubscriptionResponse; +import org.eclipse.milo.opcua.stack.transport.server.ServiceRequestContext; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Verifies that the MaxKeepAliveCount and LifetimeCount a Subscription requests are derived from + * the PublishingInterval the caller actually asked for. + * + *

{@code OpcUaSubscription} promises a keep-alive roughly every {@code + * DEFAULT_TARGET_KEEP_ALIVE_INTERVAL} (10,000 ms) and a LifetimeCount of 5x the MaxKeepAliveCount. + * The counts are therefore a function of the PublishingInterval, and every path that sets the + * PublishingInterval must recompute them (unless the caller has opted out via {@link + * OpcUaSubscription#setLifetimeAndKeepAliveCalculated(boolean)}). + * + *

Assertions are made against the {@link CreateSubscriptionRequest} observed on the wire rather + * than against the client-side getters, because the requested counts are what actually determine + * the Server's keep-alive cadence and subscription lifetime. + */ +public class SubscriptionKeepAliveDerivationTest { + + /** + * PublishingInterval used by the tests. Deliberately 10x faster than {@code + * DEFAULT_PUBLISHING_INTERVAL} (1000 ms) so a count derived from the default is off by 10x and + * cannot be mistaken for a rounding difference. + */ + private static final double PUBLISHING_INTERVAL = 100.0; + + /** ceil(10000 / 100) — a keep-alive every 100 publishing intervals, i.e. every 10 seconds. */ + private static final long EXPECTED_MAX_KEEP_ALIVE_COUNT = 100L; + + /** 5 * 100 — the lifetime must be at least 3x the keep-alive count. */ + private static final long EXPECTED_LIFETIME_COUNT = 500L; + + private TestServer testServer; + private OpcUaServer server; + private OpcUaClient client; + private CapturingSubscriptionServiceSet serviceSet; + + @BeforeEach + void startClientAndServer() throws Exception { + testServer = TestServer.create(); + server = testServer.getServer(); + + serviceSet = new CapturingSubscriptionServiceSet(server); + for (EndpointConfig endpoint : server.getConfig().getEndpoints()) { + server.addServiceSet(endpoint.getPath(), serviceSet); + } + + server.startup().get(); + + client = TestClient.create(server, cfg -> {}); + client.connect(); + } + + @AfterEach + void stopClientAndServer() throws Exception { + try { + client.disconnectAsync().get(5, TimeUnit.SECONDS); + } finally { + server.shutdown().get(5, TimeUnit.SECONDS); + } + } + + /** + * The {@code OpcUaSubscription(OpcUaClient, double)} constructor must produce the same requested + * counts as constructing with the default interval and then calling {@code + * setPublishingInterval()}. If it does not, a Subscription created with a fast PublishingInterval + * asks the Server for a keep-alive cadence and a lifetime that are both wrong by the ratio of the + * requested interval to the 1000 ms default — a 100 ms Subscription gets keep-alives every 1000 + * ms and expires after 5000 ms of client silence instead of 50,000 ms. + */ + @Test + void publishingIntervalConstructorDerivesCountsFromThatInterval() throws UaException { + var subscription = new OpcUaSubscription(client, PUBLISHING_INTERVAL); + + try { + subscription.create(); + + CreateSubscriptionRequest request = serviceSet.lastCreateSubscriptionRequest; + assertNotNull(request, "no CreateSubscription request was observed"); + + assertAll( + () -> + assertEquals( + PUBLISHING_INTERVAL, + request.getRequestedPublishingInterval(), + "requested PublishingInterval"), + () -> + assertEquals( + uint(EXPECTED_MAX_KEEP_ALIVE_COUNT), + request.getRequestedMaxKeepAliveCount(), + "requested MaxKeepAliveCount must be derived from the constructor's" + + " PublishingInterval"), + () -> + assertEquals( + uint(EXPECTED_LIFETIME_COUNT), + request.getRequestedLifetimeCount(), + "requested LifetimeCount must be derived from the constructor's" + + " PublishingInterval")); + } finally { + subscription.delete(); + } + } + + /** + * Control for {@link #publishingIntervalConstructorDerivesCountsFromThatInterval()}: the {@code + * setPublishingInterval()} path already recomputes the counts, so the expected values above are + * not vacuous. If this test fails too, the expected numbers — not the constructor — are wrong. + */ + @Test + void setPublishingIntervalDerivesCountsFromThatInterval() throws UaException { + var subscription = new OpcUaSubscription(client); + subscription.setPublishingInterval(PUBLISHING_INTERVAL); + + try { + subscription.create(); + + CreateSubscriptionRequest request = serviceSet.lastCreateSubscriptionRequest; + assertNotNull(request, "no CreateSubscription request was observed"); + + assertAll( + () -> + assertEquals( + PUBLISHING_INTERVAL, + request.getRequestedPublishingInterval(), + "requested PublishingInterval"), + () -> + assertEquals( + uint(EXPECTED_MAX_KEEP_ALIVE_COUNT), + request.getRequestedMaxKeepAliveCount(), + "requested MaxKeepAliveCount"), + () -> + assertEquals( + uint(EXPECTED_LIFETIME_COUNT), + request.getRequestedLifetimeCount(), + "requested LifetimeCount")); + } finally { + subscription.delete(); + } + } + + /** Captures the CreateSubscription request the client puts on the wire. */ + private static class CapturingSubscriptionServiceSet extends DelegatingSubscriptionServiceSet { + + volatile CreateSubscriptionRequest lastCreateSubscriptionRequest; + + CapturingSubscriptionServiceSet(OpcUaServer server) { + super(server); + } + + @Override + public CreateSubscriptionResponse onCreateSubscription( + ServiceRequestContext context, CreateSubscriptionRequest request) throws UaException { + + lastCreateSubscriptionRequest = request; + + return super.onCreateSubscription(context, request); + } + } +} diff --git a/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SubscriptionLifecycleContentionTest.java b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SubscriptionLifecycleContentionTest.java new file mode 100644 index 0000000000..6fd6310ef6 --- /dev/null +++ b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SubscriptionLifecycleContentionTest.java @@ -0,0 +1,853 @@ +/* + * Copyright (c) 2026 the Eclipse Milo Authors + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.milo.opcua.sdk.client.subscriptions; + +import static org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.Unsigned.uint; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; +import org.eclipse.milo.opcua.sdk.client.OpcUaClient; +import org.eclipse.milo.opcua.sdk.server.EndpointConfig; +import org.eclipse.milo.opcua.sdk.server.OpcUaServer; +import org.eclipse.milo.opcua.sdk.test.ScriptableSubscriptionServiceSet; +import org.eclipse.milo.opcua.sdk.test.TestClient; +import org.eclipse.milo.opcua.sdk.test.TestServer; +import org.eclipse.milo.opcua.stack.core.StatusCodes; +import org.eclipse.milo.opcua.stack.core.UaException; +import org.eclipse.milo.opcua.stack.core.types.builtin.DiagnosticInfo; +import org.eclipse.milo.opcua.stack.core.types.builtin.ExtensionObject; +import org.eclipse.milo.opcua.stack.core.types.builtin.StatusCode; +import org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.UInteger; +import org.eclipse.milo.opcua.stack.core.types.structured.CreateSubscriptionRequest; +import org.eclipse.milo.opcua.stack.core.types.structured.CreateSubscriptionResponse; +import org.eclipse.milo.opcua.stack.core.types.structured.SetPublishingModeRequest; +import org.eclipse.milo.opcua.stack.core.types.structured.SetPublishingModeResponse; +import org.eclipse.milo.opcua.stack.core.types.structured.StatusChangeNotification; +import org.eclipse.milo.opcua.stack.transport.server.ServiceRequestContext; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +/** + * What an {@link OpcUaSubscription} lifecycle transition is allowed to hold up while it waits for + * the Server. + * + *

{@code create()}, {@code modify()}, {@code delete()}, {@code setPublishingMode(boolean)} and + * {@code reset()} serialize on one private lock, and the first four hold it across their blocking + * service call. {@code reset()} therefore waits for a full network round trip before it can do its + * work — and {@code reset()} is not called only by applications: + * + *

    + *
  • {@code PublishingManager} calls it, via {@code notifyStatusChanged}, when the Server + * reports Bad_Timeout for a Subscription (Part 4 §5.13.1.1: the Subscription no longer + * exists). That runs on the Subscription's delivery queue, which is backed by the + * transport executor. + *
  • The Session FSM calls it, via {@code notifyTransferFailed}, when TransferSubscriptions did + * not carry a Subscription over to a new Session. + *
+ * + *

Two consequences, and the two nested classes assert against them: + * + *

    + *
  1. Stall. On any executor, including the unbounded cached pool the client uses by + * default, the Bad_Timeout path waits behind whatever lifecycle call happens to be in flight + * — up to a full request timeout, 60s by default. The Subscription's delivery queue is + * stopped for that whole time, and a Session FSM callback that blocks is worse still. + *
  2. Deadlock. The threads that block are executor threads, and the response that would + * release the lock is completed by the transport executor ({@code + * AbstractUascClientTransport.handleResponse}). A client whose executor has one thread — an + * ordinary application choice; {@code OpcTcpClientTransportConfigBuilder.setExecutor} exists + * for it — therefore wedges permanently: the only thread that could complete the in-flight + * lifecycle call's response is the one blocked inside {@code reset()}, and {@code + * handleResponse} has already cancelled the request timeout that would otherwise rescue it. + *
+ * + *

A failure here manifests as a timeout, and that is the correct signal: the defect is + * precisely that a call which does no network I/O of its own does not return. Every wait below is + * bounded, every thread the tests start is a daemon, and both executors use daemon threads, so a + * permanently blocked thread cannot hang the build or keep the JVM alive; teardown tolerates a + * client too wedged to disconnect. + */ +public class SubscriptionLifecycleContentionTest { + + /** How long to wait for something that must happen. */ + private static final long AWAIT_TIMEOUT_MILLIS = 10_000; + + /** + * How long a call that performs no network I/O of its own is given to return. A loopback round + * trip completes in single-digit milliseconds, so this is generous by two orders of magnitude + * against any implementation that does not wait for one — and an implementation that does wait + * for the gated call below would need {@link #GATE_TIMEOUT_MILLIS}, so no larger value would + * change an outcome. + */ + private static final long NON_BLOCKING_WINDOW_MILLIS = 2_000; + + /** + * How long to wait for a Server-side Subscription to be cleaned up. Well under the Subscription's + * own lifetime on the Server — the default 1000ms PublishingInterval derives a LifetimeCount of + * 30, i.e. 30s — so a Subscription expiring on its own cannot satisfy the assertion. + */ + private static final long CLEANUP_WINDOW_MILLIS = 5_000; + + /** Upper bound on how long a gated Server handler holds a request, so nothing hangs forever. */ + private static final long GATE_TIMEOUT_MILLIS = 30_000; + + /** + * Long enough that nothing times out on its own: neither a parked Publish request nor a gated + * lifecycle call. The stalls asserted against below are therefore the client's own doing. + */ + private static final long REQUEST_TIMEOUT_MILLIS = 60_000; + + /** How long teardown waits for a client that may be wedged to disconnect. */ + private static final long DISCONNECT_TIMEOUT_MILLIS = 5_000; + + /** The client keeps one more PublishRequest in flight than it has Subscriptions. */ + private static final int PIPELINE_DEPTH = 2; + + /** + * {@code reset()} and {@code notifyTransferFailed()} discard the client's knowledge of a + * Subscription. Neither sends anything to the Server, so neither has any reason to wait for it. + */ + @Nested + class ResetWhileACreateIsInFlight { + + /** + * The core defect. A {@code create()} parked inside the CreateSubscription service call holds + * the lifecycle lock, so {@code reset()} — which only has local state to discard — cannot run + * until the Server answers. + * + *

Failure manifests as {@code reset()} not returning inside {@link + * #NON_BLOCKING_WINDOW_MILLIS}. + */ + @Test + void resetCompletesWhileACreateIsInFlight() throws Exception { + try (var fixture = new Fixture(cachedThreadPool())) { + var subscription = new OpcUaSubscription(fixture.client); + + Call create = fixture.startGatedCreate(subscription); + Call reset = Call.start("reset", subscription::reset); + + assertTrue( + reset.awaitFinished(NON_BLOCKING_WINDOW_MILLIS), + "reset() has not returned " + + NON_BLOCKING_WINDOW_MILLIS + + "ms after it was called, because a create() parked inside the CreateSubscription" + + " service call is holding the lifecycle lock. On the Bad_Timeout path this call" + + " is made from the Subscription's delivery queue and on the transfer-failed path" + + " from the Session FSM, so both are stopped for as long as the Server takes to" + + " answer someone else's request"); + + assertFalse( + create.isFinished(), + "the gated create() has already returned, so reset() did not overlap it and the" + + " assertion above proved nothing"); + } + } + + /** + * The same defect on the path the Session FSM takes. {@code notifyTransferFailed()} is + * dispatched when TransferSubscriptions fails to carry a Subscription over to a new Session, + * and it resets the Subscription; blocking there blocks the state machine that owns + * reconnection. + */ + @Test + void notifyTransferFailedCompletesWhileACreateIsInFlight() throws Exception { + try (var fixture = new Fixture(cachedThreadPool())) { + var subscription = new OpcUaSubscription(fixture.client); + + Call create = fixture.startGatedCreate(subscription); + Call transferFailed = + Call.start( + "transfer-failed", + () -> + subscription.notifyTransferFailed( + new StatusCode(StatusCodes.Bad_SubscriptionIdInvalid))); + + assertTrue( + transferFailed.awaitFinished(NON_BLOCKING_WINDOW_MILLIS), + "notifyTransferFailed() has not returned " + + NON_BLOCKING_WINDOW_MILLIS + + "ms after it was called: it resets the Subscription, and the reset waits for the" + + " CreateSubscription round trip a concurrent create() is holding the lifecycle" + + " lock across. This callback runs on the Session FSM"); + + assertFalse( + create.isFinished(), + "the gated create() has already returned, so notifyTransferFailed() did not overlap it" + + " and the assertion above proved nothing"); + } + } + + /** + * The control that keeps the two timing assertions above from being vacuous: with no lifecycle + * call in flight, {@code reset()} returns well inside the same window. + */ + @Test + void resetCompletesWhenNoCreateIsInFlight() throws Exception { + try (var fixture = new Fixture(cachedThreadPool())) { + OpcUaSubscription subscription = fixture.createSubscription(); + + Call reset = Call.start("reset", subscription::reset); + + assertTrue( + reset.awaitFinished(NON_BLOCKING_WINDOW_MILLIS), + "control: reset() must return promptly when no lifecycle call is in flight"); + } + } + + /** + * A {@code reset()} that supersedes an in-flight {@code create()} must not abandon the + * Subscription that call went on to create. Part 4 §5.13.8 gives DeleteSubscriptions the + * SubscriptionIds as its only handle on a Subscription, so a Subscription the Server created + * and no client object still names runs until its lifetime expires and cannot be deleted — the + * same leak the immutable-id work was about. + * + *

Nothing here is asserted about timing; this is about what is left on the Server after the + * reset has demonstrably superseded the gated create and that create has subsequently finished. + */ + @Test + void noServerSideSubscriptionSurvivesAResetIssuedDuringACreate() throws Exception { + try (var fixture = new Fixture(cachedThreadPool())) { + var subscription = new OpcUaSubscription(fixture.client); + + Call create = fixture.startGatedCreate(subscription); + Call reset = Call.start("reset", subscription::reset); + + assertTrue( + reset.awaitFinished(AWAIT_TIMEOUT_MILLIS), + "reset() never finished while create() was gated at the Server"); + assertEquals(Optional.empty(), reset.failure(), "reset() must complete successfully"); + assertFalse( + create.isFinished(), + "the gated create() finished before the Server gate was released, so reset() did not" + + " demonstrably supersede it"); + + fixture.scriptable.releaseCreates(); + + assertTrue(create.awaitFinished(AWAIT_TIMEOUT_MILLIS), "create() never finished"); + + UaException failure = + assertInstanceOf( + UaException.class, + create.failure().orElse(null), + "a create() superseded by reset() must fail"); + assertEquals( + StatusCodes.Bad_InvalidState, + failure.getStatusCode().value(), + "a create() superseded by reset() must fail with Bad_InvalidState"); + + assertFalse( + fixture.scriptable.createdSubscriptionIds().isEmpty(), + "the gated create() never created a Subscription on the Server, so there was nothing" + + " that could have been leaked and the assertion below proves nothing"); + + assertTrue( + fixture.awaitTrue( + () -> fixture.unnamedServerSubscriptions(subscription).isEmpty(), + CLEANUP_WINDOW_MILLIS), + () -> + "the Server is still running Subscription(s) " + + fixture.unnamedServerSubscriptions(subscription) + + " that no client object names: the create() that made them completed after a" + + " reset() had discarded the Subscription, so its ServerState was either" + + " overwritten or thrown away and delete() can no longer name it. Created on" + + " the Server: " + + fixture.scriptable.createdSubscriptionIds() + + ", named by the client: " + + subscription.getSubscriptionId()); + } + } + + /** + * The invariant the lifecycle lock was added for, asserted here so that removing the lock from + * around the service call cannot quietly give it up: a second {@code create()} made while the + * first is still in flight is answered {@code Bad_InvalidState} and does not create a second + * Subscription on the Server. Passes today; it is a guard, not a reproduction. + */ + @Test + void secondCreateIsRejectedWhileTheFirstIsInFlight() throws Exception { + try (var fixture = new Fixture(cachedThreadPool())) { + var subscription = new OpcUaSubscription(fixture.client); + + Call first = fixture.startGatedCreate(subscription); + Call second = Call.start("create-second", subscription::create); + + // Either the second call has been rejected client-side, or it has reached the Server. Not + // asserted: an implementation that blocks it on the lock does neither until the gate opens, + // and the assertions below hold just as well then. + fixture.awaitTrue( + () -> second.isFinished() || fixture.scriptable.gatedCreateArrivals() >= 2, + NON_BLOCKING_WINDOW_MILLIS); + + fixture.scriptable.releaseCreates(); + + assertTrue(first.awaitFinished(AWAIT_TIMEOUT_MILLIS), "the first create() never finished"); + assertTrue( + second.awaitFinished(AWAIT_TIMEOUT_MILLIS), "the second create() never finished"); + + UaException failure = + assertInstanceOf( + UaException.class, + second.failure().orElse(null), + "the second create() must be rejected while a Subscription already exists or is" + + " being created, or the Server ends up running a Subscription whose" + + " SubscriptionId the client object immediately overwrites"); + assertEquals( + StatusCodes.Bad_InvalidState, + failure.getStatusCode().value(), + "the second create() must fail with Bad_InvalidState"); + + assertEquals( + 1, + fixture.scriptable.createdSubscriptionIds().size(), + "two concurrent create() calls created " + + fixture.scriptable.createdSubscriptionIds().size() + + " Subscriptions on the Server; only one of them can be named by the single" + + " ServerState the client object holds"); + } + } + } + + /** + * The Bad_Timeout StatusChangeNotification path, which is how {@code reset()} is reached in + * production. {@code PublishingManager} delivers the notification on the Subscription's delivery + * queue and {@code notifyStatusChanged} resets the Subscription before handing the status to the + * application, so a blocked reset blocks the queue and the callback with it. + */ + @Nested + class StatusChangeWhileALifecycleCallIsInFlight { + + /** + * Hazard 2, on the shape of executor the client uses by default (an unbounded cached pool, as + * {@code Stack.sharedExecutor()} is). No thread starvation is involved: the delivery queue + * simply waits for a SetPublishingMode round trip it has nothing to do with, and until it + * returns the application is not told its Subscription has timed out and no further + * notification for that Subscription is delivered. + * + *

Failure manifests as the {@code onStatusChanged} callback not arriving inside {@link + * #NON_BLOCKING_WINDOW_MILLIS}. + */ + @Test + void badTimeoutStatusChangeIsDeliveredWhileASetPublishingModeIsInFlight() throws Exception { + try (var fixture = new Fixture(cachedThreadPool())) { + OpcUaSubscription subscription = fixture.createSubscription(); + + var statusChanged = new CountDownLatch(1); + subscription.setSubscriptionListener(statusChangeListener(statusChanged)); + + UInteger id = subscriptionId(subscription); + fixture.awaitPipelineFilled(); + + Call setPublishingMode = fixture.startGatedSetPublishingMode(subscription); + + fixture.scriptable.enqueueNotification(id, 1, badTimeout(fixture.scriptable), uint(1)); + + assertTrue( + statusChanged.await(NON_BLOCKING_WINDOW_MILLIS, TimeUnit.MILLISECONDS), + "the Bad_Timeout StatusChangeNotification was not delivered within " + + NON_BLOCKING_WINDOW_MILLIS + + "ms: notifyStatusChanged() resets the Subscription first, and that reset is" + + " waiting for the SetPublishingMode round trip the lifecycle lock is held across." + + " The Subscription's delivery queue is stopped for the whole round trip, which is" + + " bounded only by the request timeout"); + + assertFalse( + setPublishingMode.isFinished(), + "the gated setPublishingMode() has already returned, so nothing was in flight while the" + + " status change was delivered and the assertion above proved nothing"); + } + } + + /** + * Hazard 1: the same interleaving on a single-threaded transport executor is not a stall but a + * permanent deadlock. The delivery queue runs on that executor, so the thread blocked inside + * {@code reset()} is the only thread that could complete the SetPublishingMode response the + * lock-holder is waiting for, and {@code handleResponse} cancelled that request's timeout the + * moment the response arrived. + * + *

The first assertion is the stall; the second — made only once the Server has answered — is + * the deadlock: nothing can ever release the in-flight call. + */ + @Test + void badTimeoutStatusChangeIsDeliveredWhileASetPublishingModeIsInFlightOnOneThread() + throws Exception { + + try (var fixture = new Fixture(singleThreadExecutor())) { + OpcUaSubscription subscription = fixture.createSubscription(); + + var statusChanged = new CountDownLatch(1); + subscription.setSubscriptionListener(statusChangeListener(statusChanged)); + + UInteger id = subscriptionId(subscription); + fixture.awaitPipelineFilled(); + + Call setPublishingMode = fixture.startGatedSetPublishingMode(subscription); + + fixture.scriptable.enqueueNotification(id, 1, badTimeout(fixture.scriptable), uint(1)); + + assertTrue( + statusChanged.await(NON_BLOCKING_WINDOW_MILLIS, TimeUnit.MILLISECONDS), + "the Bad_Timeout StatusChangeNotification was not delivered within " + + NON_BLOCKING_WINDOW_MILLIS + + "ms: the delivery queue task is blocked inside reset(), waiting for the lifecycle" + + " lock a SetPublishingMode round trip is holding, and it is occupying the" + + " transport executor's only thread while it waits"); + + fixture.scriptable.releaseSetPublishingMode(); + + assertTrue( + setPublishingMode.awaitFinished(AWAIT_TIMEOUT_MILLIS), + "setPublishingMode() never returned although the Server answered it: the response is" + + " completed by the transport executor, whose only thread is blocked inside a" + + " reset() waiting for the lock setPublishingMode() holds. The request timeout was" + + " cancelled when the response arrived, so nothing breaks the cycle"); + } + } + + /** + * The control for both tests above: the identical script with no lifecycle call in flight + * delivers the status change. Without it, a Bad_Timeout that is never delivered at all would + * look the same as one that is delivered late. + */ + @Test + void badTimeoutStatusChangeIsDeliveredWhenNoLifecycleCallIsInFlight() throws Exception { + try (var fixture = new Fixture(cachedThreadPool())) { + OpcUaSubscription subscription = fixture.createSubscription(); + + var statusChanged = new CountDownLatch(1); + subscription.setSubscriptionListener(statusChangeListener(statusChanged)); + + UInteger id = subscriptionId(subscription); + fixture.awaitPipelineFilled(); + + fixture.scriptable.enqueueNotification(id, 1, badTimeout(fixture.scriptable), uint(1)); + + assertTrue( + statusChanged.await(NON_BLOCKING_WINDOW_MILLIS, TimeUnit.MILLISECONDS), + "control: a Bad_Timeout StatusChangeNotification must be delivered promptly when no" + + " lifecycle call is in flight"); + } + } + + /** + * The control that isolates the deadlock as the cause rather than the executor: on the same + * single-threaded executor, a SetPublishingMode that nothing holds up completes, and the + * Bad_Timeout that follows it is delivered. + */ + @Test + void badTimeoutStatusChangeIsDeliveredOnOneThreadWhenNoLifecycleCallIsInFlight() + throws Exception { + + try (var fixture = new Fixture(singleThreadExecutor())) { + OpcUaSubscription subscription = fixture.createSubscription(); + + var statusChanged = new CountDownLatch(1); + subscription.setSubscriptionListener(statusChangeListener(statusChanged)); + + UInteger id = subscriptionId(subscription); + fixture.awaitPipelineFilled(); + + Call setPublishingMode = + Call.start("set-publishing-mode", () -> subscription.setPublishingMode(false)); + + assertTrue( + setPublishingMode.awaitFinished(AWAIT_TIMEOUT_MILLIS), + "control: a lifecycle call must complete on a single-threaded transport executor when" + + " nothing is occupying that thread"); + assertEquals( + Optional.empty(), + setPublishingMode.failure(), + "control: the ungated setPublishingMode() must succeed"); + + fixture.scriptable.enqueueNotification(id, 1, badTimeout(fixture.scriptable), uint(1)); + + assertTrue( + statusChanged.await(NON_BLOCKING_WINDOW_MILLIS, TimeUnit.MILLISECONDS), + "control: a Bad_Timeout StatusChangeNotification must be delivered on a" + + " single-threaded transport executor when nothing is occupying that thread"); + } + } + } + + // region helpers + + private static UInteger subscriptionId(OpcUaSubscription subscription) { + return subscription.getSubscriptionId().orElseThrow(); + } + + private static OpcUaSubscription.SubscriptionListener statusChangeListener( + CountDownLatch statusChanged) { + + return new OpcUaSubscription.SubscriptionListener() { + @Override + public void onStatusChanged(OpcUaSubscription subscription, StatusCode status) { + statusChanged.countDown(); + } + }; + } + + /** A NotificationMessage body reporting that the Subscription has timed out. */ + private static ExtensionObject[] badTimeout(ScriptableSubscriptionServiceSet scriptable) { + return new ExtensionObject[] { + scriptable.encode( + new StatusChangeNotification( + new StatusCode(StatusCodes.Bad_Timeout), DiagnosticInfo.NULL_VALUE)) + }; + } + + /** + * A lifecycle call running on its own daemon thread, with the outcome recorded however it turned + * out. + * + *

The thread is a daemon and the outcome is recorded in a {@code finally} block because these + * calls are expected to block indefinitely when the defect is present: the test has to be able to + * observe that and move on, and the JVM has to be able to exit afterwards. + */ + private static final class Call { + + private final CountDownLatch finished = new CountDownLatch(1); + private final AtomicReference failure = new AtomicReference<>(); + + private final Thread thread; + + private Call(String name, ThrowingRunnable body) { + this.thread = + new Thread( + () -> { + try { + body.run(); + } catch (Exception e) { + failure.set(e); + } finally { + finished.countDown(); + } + }, + name); + this.thread.setDaemon(true); + } + + static Call start(String name, ThrowingRunnable body) { + var call = new Call(name, body); + call.thread.start(); + + return call; + } + + boolean awaitFinished(long timeoutMillis) throws InterruptedException { + return finished.await(timeoutMillis, TimeUnit.MILLISECONDS); + } + + boolean isFinished() { + return finished.getCount() == 0; + } + + /** The Exception the call threw, if any. */ + Optional failure() { + return Optional.ofNullable(failure.get()); + } + } + + /** + * A {@link ScriptableSubscriptionServiceSet} that can hold a CreateSubscription or a + * SetPublishingMode request inside the Server handler until the test releases it, and that + * records every SubscriptionId it hands out. + * + *

Holding a request inside the handler is what makes the overlap real rather than + * raced: the client has sent the request and is waiting for the response, so it is demonstrably + * inside the service call — and therefore holding whatever the service call holds — for as long + * as the test wants. + */ + private static final class GatedSubscriptionServiceSet extends ScriptableSubscriptionServiceSet { + + private final AtomicInteger gatedCreateArrivals = new AtomicInteger(0); + private final AtomicInteger gatedSetPublishingModeArrivals = new AtomicInteger(0); + + private final CountDownLatch createGate = new CountDownLatch(1); + private final CountDownLatch setPublishingModeGate = new CountDownLatch(1); + + private final List createdSubscriptionIds = new CopyOnWriteArrayList<>(); + + private volatile boolean createsGated = false; + private volatile boolean setPublishingModeGated = false; + + GatedSubscriptionServiceSet(OpcUaServer server) { + super(server); + } + + void gateCreates() { + createsGated = true; + } + + void gateSetPublishingMode() { + setPublishingModeGated = true; + } + + int gatedCreateArrivals() { + return gatedCreateArrivals.get(); + } + + int gatedSetPublishingModeArrivals() { + return gatedSetPublishingModeArrivals.get(); + } + + void releaseCreates() { + createGate.countDown(); + } + + void releaseSetPublishingMode() { + setPublishingModeGate.countDown(); + } + + void releaseAllGates() { + releaseCreates(); + releaseSetPublishingMode(); + } + + /** Every SubscriptionId the Server has assigned, in the order it assigned them. */ + List createdSubscriptionIds() { + return List.copyOf(createdSubscriptionIds); + } + + @Override + public CreateSubscriptionResponse onCreateSubscription( + ServiceRequestContext context, CreateSubscriptionRequest request) throws UaException { + + if (createsGated) { + gatedCreateArrivals.incrementAndGet(); + await(createGate, "CreateSubscription"); + } + + CreateSubscriptionResponse response = super.onCreateSubscription(context, request); + createdSubscriptionIds.add(response.getSubscriptionId()); + + return response; + } + + @Override + public SetPublishingModeResponse onSetPublishingMode( + ServiceRequestContext context, SetPublishingModeRequest request) throws UaException { + + if (setPublishingModeGated) { + gatedSetPublishingModeArrivals.incrementAndGet(); + await(setPublishingModeGate, "SetPublishingMode"); + } + + return super.onSetPublishingMode(context, request); + } + + private static void await(CountDownLatch gate, String service) throws UaException { + try { + if (!gate.await(GATE_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { + throw new UaException( + StatusCodes.Bad_Timeout, "the " + service + " gate was never opened"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new UaException(StatusCodes.Bad_UnexpectedError, e); + } + } + } + + /** + * A running Server whose Publish responses are scripted and whose CreateSubscription and + * SetPublishingMode responses can be held, plus a client driven by a caller-supplied {@link + * ExecutorService}. + */ + private static final class Fixture implements AutoCloseable { + + private final ExecutorService executor; + private final OpcUaServer server; + private final OpcUaClient client; + private final GatedSubscriptionServiceSet scriptable; + + Fixture(ExecutorService executor) throws Exception { + this.executor = executor; + + TestServer testServer = TestServer.create(); + server = testServer.getServer(); + + scriptable = new GatedSubscriptionServiceSet(server); + for (EndpointConfig endpoint : server.getConfig().getEndpoints()) { + server.addServiceSet(endpoint.getPath(), scriptable); + } + + server.startup().get(); + + client = + TestClient.create( + server, + transportConfig -> transportConfig.setExecutor(executor), + cfg -> + cfg + // Long request timeout so parked or gated requests do not time out. + .setRequestTimeout(uint(REQUEST_TIMEOUT_MILLIS)) + // No Session keep-alive traffic: the only requests in flight during a test + // are the ones it scripts. + .setKeepAliveInterval(uint(REQUEST_TIMEOUT_MILLIS))); + client.connect(); + } + + OpcUaSubscription createSubscription() throws UaException { + var subscription = new OpcUaSubscription(client); + subscription.create(); + + return subscription; + } + + /** + * Start a {@code create()} on its own thread and return once it is inside the Server's + * CreateSubscription handler, i.e. once it is demonstrably waiting for the response. + */ + Call startGatedCreate(OpcUaSubscription subscription) throws Exception { + scriptable.gateCreates(); + + Call call = Call.start("create", subscription::create); + + assertTrue( + awaitTrue(() -> scriptable.gatedCreateArrivals() >= 1, AWAIT_TIMEOUT_MILLIS), + "the create() never reached the Server, so it is not inside the CreateSubscription" + + " service call and the scenario under test did not happen"); + + return call; + } + + /** + * Start a {@code setPublishingMode()} on its own thread and return once it is inside the + * Server's SetPublishingMode handler. + */ + Call startGatedSetPublishingMode(OpcUaSubscription subscription) throws Exception { + scriptable.gateSetPublishingMode(); + + Call call = Call.start("set-publishing-mode", () -> subscription.setPublishingMode(false)); + + assertTrue( + awaitTrue(() -> scriptable.gatedSetPublishingModeArrivals() >= 1, AWAIT_TIMEOUT_MILLIS), + "the setPublishingMode() never reached the Server, so it is not inside the" + + " SetPublishingMode service call and the scenario under test did not happen"); + + return call; + } + + void awaitPipelineFilled() throws Exception { + assertTrue( + awaitTrue( + () -> scriptable.getParkedRequestCount() >= PIPELINE_DEPTH, AWAIT_TIMEOUT_MILLIS), + "the client did not fill its Publish pipeline"); + } + + /** + * The Subscriptions the Server is running that {@code subscription} cannot name, i.e. the ones + * no DeleteSubscriptions call the client could make would name either. + */ + Set unnamedServerSubscriptions(OpcUaSubscription subscription) { + Optional named = subscription.getSubscriptionId(); + + return server.getSubscriptions().keySet().stream() + .filter(id -> named.filter(id::equals).isEmpty()) + .collect(Collectors.toUnmodifiableSet()); + } + + /** Polls {@code condition} until it holds or {@code timeoutMillis} elapses. */ + boolean awaitTrue(ThrowingBooleanSupplier condition, long timeoutMillis) throws Exception { + long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(timeoutMillis); + + while (System.nanoTime() < deadline) { + if (condition.get()) { + return true; + } + Thread.sleep(25); + } + + return condition.get(); + } + + @Override + public void close() throws Exception { + scriptable.releaseAllGates(); + scriptable.failParkedRequests(StatusCodes.Bad_NoSubscription); + try { + client.disconnectAsync().get(DISCONNECT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); + } catch (TimeoutException ignored) { + // A client whose executor thread is wedged behind a blocking reset() cannot run the + // disconnect it is asked for. Tolerated so teardown does not mask the assertion that + // detected the wedge; shutting the Server and the executor down below releases what can be + // released, and every thread involved is a daemon. + } finally { + try { + server.shutdown().get(10, TimeUnit.SECONDS); + } finally { + executor.shutdownNow(); + } + } + } + } + + /** + * The shape of {@code Stack.sharedExecutor()}: an unbounded cached pool. Daemon threads, so a + * task left blocked by a failing test cannot keep the JVM alive. + */ + private static ExecutorService cachedThreadPool() { + return new ThreadPoolExecutor( + 0, + Integer.MAX_VALUE, + 60L, + TimeUnit.SECONDS, + new SynchronousQueue<>(), + daemonThreadFactory("lifecycle-contention-pool")); + } + + /** + * A single-threaded transport executor, which {@code + * OpcTcpClientTransportConfigBuilder.setExecutor} exists to allow. + */ + private static ExecutorService singleThreadExecutor() { + return Executors.newSingleThreadExecutor(daemonThreadFactory("lifecycle-contention-single")); + } + + private static ThreadFactory daemonThreadFactory(String namePrefix) { + var counter = new AtomicInteger(0); + + return runnable -> { + var thread = new Thread(runnable, namePrefix + "-" + counter.incrementAndGet()); + thread.setDaemon(true); + + return thread; + }; + } + + @FunctionalInterface + private interface ThrowingRunnable { + void run() throws Exception; + } + + @FunctionalInterface + private interface ThrowingBooleanSupplier { + boolean get() throws Exception; + } + + // endregion +} diff --git a/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SubscriptionModifyRetryTest.java b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SubscriptionModifyRetryTest.java new file mode 100644 index 0000000000..ea62979973 --- /dev/null +++ b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SubscriptionModifyRetryTest.java @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2026 the Eclipse Milo Authors + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.milo.opcua.sdk.client.subscriptions; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.eclipse.milo.opcua.sdk.client.OpcUaClient; +import org.eclipse.milo.opcua.sdk.client.subscriptions.OpcUaSubscription.SyncState; +import org.eclipse.milo.opcua.sdk.server.EndpointConfig; +import org.eclipse.milo.opcua.sdk.server.OpcUaServer; +import org.eclipse.milo.opcua.sdk.test.DelegatingSubscriptionServiceSet; +import org.eclipse.milo.opcua.sdk.test.TestClient; +import org.eclipse.milo.opcua.sdk.test.TestServer; +import org.eclipse.milo.opcua.stack.core.StatusCodes; +import org.eclipse.milo.opcua.stack.core.UaException; +import org.eclipse.milo.opcua.stack.core.types.structured.ModifySubscriptionRequest; +import org.eclipse.milo.opcua.stack.core.types.structured.ModifySubscriptionResponse; +import org.eclipse.milo.opcua.stack.transport.server.ServiceRequestContext; +import org.junit.jupiter.api.Test; + +/** + * Verifies that a failed {@link OpcUaSubscription#modify()} leaves the pending modifications intact + * so the next {@code modify()} retries them. + * + *

{@code modify()} takes the pending {@code Modifications} and clears the field before + * calling the ModifySubscription service. If the service call fails the Subscription is left in + * {@code UNSYNCHRONIZED} state with nothing left to synchronize, so the caller's requested + * parameters are lost even though the Subscription still advertises that it has changes to push. A + * transient service failure is exactly the case a caller is expected to retry, so the retry must + * send the same parameters that the first attempt sent. + */ +public class SubscriptionModifyRetryTest { + + /** + * A {@link DelegatingSubscriptionServiceSet} that fails the first {@code n} ModifySubscription + * calls with a service fault and records every request it receives. + */ + private static class FlakyModifySubscriptionServiceSet extends DelegatingSubscriptionServiceSet { + + private final List requests = + Collections.synchronizedList(new ArrayList<>()); + + private final AtomicInteger failuresRemaining; + + FlakyModifySubscriptionServiceSet(OpcUaServer server, int failureCount) { + super(server); + + this.failuresRemaining = new AtomicInteger(failureCount); + } + + List getRequests() { + return List.copyOf(requests); + } + + @Override + public ModifySubscriptionResponse onModifySubscription( + ServiceRequestContext context, ModifySubscriptionRequest request) throws UaException { + + requests.add(request); + + if (failuresRemaining.getAndDecrement() > 0) { + throw new UaException( + StatusCodes.Bad_TooManyOperations, "scripted ModifySubscription failure"); + } + + return super.onModifySubscription(context, request); + } + } + + @Test + void modifyAfterFailedModifyRetriesTheSamePendingModifications() throws Exception { + TestServer testServer = TestServer.create(); + OpcUaServer server = testServer.getServer(); + + var serviceSet = new FlakyModifySubscriptionServiceSet(server, 1); + for (EndpointConfig endpoint : server.getConfig().getEndpoints()) { + server.addServiceSet(endpoint.getPath(), serviceSet); + } + + server.startup().get(); + + OpcUaClient client = TestClient.create(server, cfg -> {}); + client.connect(); + + try { + var subscription = new OpcUaSubscription(client); + subscription.create(); + + double originalInterval = subscription.getRevisedPublishingInterval().orElseThrow(); + double requestedInterval = originalInterval * 2.0; + + subscription.setPublishingInterval(requestedInterval); + assertEquals(SyncState.UNSYNCHRONIZED, subscription.getSyncState()); + + // First attempt: the server rejects ModifySubscription with a service fault. + UaException failure = assertThrows(UaException.class, subscription::modify); + assertEquals( + StatusCodes.Bad_TooManyOperations, + failure.getStatusCode().value(), + "first modify() should have failed with the scripted fault"); + + assertEquals( + 1, serviceSet.getRequests().size(), "server should have seen exactly one modify attempt"); + assertEquals( + requestedInterval, + serviceSet.getRequests().get(0).getRequestedPublishingInterval().doubleValue(), + "control: the first attempt did request the new PublishingInterval"); + + assertEquals( + SyncState.UNSYNCHRONIZED, + subscription.getSyncState(), + "a failed modify() must leave the Subscription unsynchronized"); + + // Second attempt: succeeds, and must re-send the modifications the first attempt lost. + subscription.modify(); + + assertEquals(SyncState.SYNCHRONIZED, subscription.getSyncState()); + assertEquals( + 2, + serviceSet.getRequests().size(), + "server should have seen exactly two modify attempts"); + assertEquals( + requestedInterval, + serviceSet.getRequests().get(1).getRequestedPublishingInterval().doubleValue(), + "the retry must re-send the pending PublishingInterval, not fall back to the server" + + " value"); + } finally { + try { + client.disconnectAsync().get(2, TimeUnit.SECONDS); + } finally { + server.shutdown().get(2, TimeUnit.SECONDS); + } + } + } +} diff --git a/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SubscriptionNotificationDeliveryTest.java b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SubscriptionNotificationDeliveryTest.java new file mode 100644 index 0000000000..420886637f --- /dev/null +++ b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SubscriptionNotificationDeliveryTest.java @@ -0,0 +1,526 @@ +/* + * Copyright (c) 2026 the Eclipse Milo Authors + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.milo.opcua.sdk.client.subscriptions; + +import static org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.Unsigned.uint; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import org.eclipse.milo.opcua.sdk.client.OpcUaClient; +import org.eclipse.milo.opcua.sdk.server.EndpointConfig; +import org.eclipse.milo.opcua.sdk.server.OpcUaServer; +import org.eclipse.milo.opcua.sdk.test.ScriptableSubscriptionServiceSet; +import org.eclipse.milo.opcua.sdk.test.TestClient; +import org.eclipse.milo.opcua.sdk.test.TestServer; +import org.eclipse.milo.opcua.stack.core.NodeIds; +import org.eclipse.milo.opcua.stack.core.StatusCodes; +import org.eclipse.milo.opcua.stack.core.UaException; +import org.eclipse.milo.opcua.stack.core.types.builtin.DataValue; +import org.eclipse.milo.opcua.stack.core.types.builtin.DiagnosticInfo; +import org.eclipse.milo.opcua.stack.core.types.builtin.ExtensionObject; +import org.eclipse.milo.opcua.stack.core.types.builtin.StatusCode; +import org.eclipse.milo.opcua.stack.core.types.builtin.Variant; +import org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.UInteger; +import org.eclipse.milo.opcua.stack.core.types.structured.DataChangeNotification; +import org.eclipse.milo.opcua.stack.core.types.structured.EventFieldList; +import org.eclipse.milo.opcua.stack.core.types.structured.EventNotificationList; +import org.eclipse.milo.opcua.stack.core.types.structured.MonitoredItemNotification; +import org.eclipse.milo.opcua.stack.core.types.structured.StatusChangeNotification; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +/** + * Delivery guarantees of the client-side notification fan-out: {@code + * OpcUaSubscription.notifyDataReceived}, {@code notifyEventsReceived}, {@code + * notifyKeepAliveReceived} and {@code notifyStatusChanged}, driven through {@code + * PublishingManager.deliverNotificationMessage}. + * + *

Publish responses are scripted with {@link ScriptableSubscriptionServiceSet}, so the + * MonitoredItems used here only need to exist on the client: {@link + * OpcUaSubscription#addMonitoredItem(OpcUaMonitoredItem)} assigns the ClientHandle the fan-out + * looks notifications up by, and no Server-side item participates in delivering a scripted + * notification. + */ +public class SubscriptionNotificationDeliveryTest { + + private static final DataValue VALUE_1 = new DataValue(Variant.ofInt32(1)); + private static final DataValue VALUE_2 = new DataValue(Variant.ofInt32(2)); + + private static final String DATA_CALLBACK = "onDataReceived"; + private static final String KEEP_ALIVE_CALLBACK = "onKeepAliveReceived"; + + /** + * The application's {@link OpcUaSubscription.SubscriptionListener} and the per-item {@link + * OpcUaMonitoredItem.DataValueListener}s are independent sinks for the same + * DataChangeNotification. A failure in one must not cost the others their notification, and must + * not consume the rest of the NotificationMessage. + */ + @Nested + class ListenerFailureIsolation { + + @Test + void monitoredItemListenerIsNotifiedWhenSubscriptionListenerThrows() throws Exception { + try (var fixture = new Fixture()) { + OpcUaSubscription subscription = fixture.createSubscription(); + + var item = OpcUaMonitoredItem.newDataItem(NodeIds.Server_ServerStatus_CurrentTime); + subscription.addMonitoredItem(item); + + var received = new AtomicReference(); + var itemNotified = new CountDownLatch(1); + item.setDataValueListener( + (i, value) -> { + received.set(value); + itemNotified.countDown(); + }); + + subscription.setSubscriptionListener( + new OpcUaSubscription.SubscriptionListener() { + @Override + public void onDataReceived( + OpcUaSubscription s, List items, List values) { + throw new IllegalStateException("application SubscriptionListener failure"); + } + }); + + fixture.scriptable.enqueueDataChange( + subscriptionId(subscription), + 1, + List.of(new MonitoredItemNotification(clientHandle(item), VALUE_1))); + + assertTrue( + itemNotified.await(5, TimeUnit.SECONDS), + "the MonitoredItem's DataValueListener was never notified: the throwing" + + " SubscriptionListener aborted the fan-out"); + assertEquals(VALUE_1.getValue(), received.get().getValue()); + } + } + + @Test + void laterMonitoredItemListenersAreNotifiedWhenAnEarlierOneThrows() throws Exception { + try (var fixture = new Fixture()) { + OpcUaSubscription subscription = fixture.createSubscription(); + + var throwingItem = OpcUaMonitoredItem.newDataItem(NodeIds.Server_ServerStatus_CurrentTime); + var observedItem = OpcUaMonitoredItem.newDataItem(NodeIds.Server_ServerStatus_StartTime); + subscription.addMonitoredItem(throwingItem); + subscription.addMonitoredItem(observedItem); + + throwingItem.setDataValueListener( + (i, value) -> { + throw new IllegalStateException("application DataValueListener failure"); + }); + + var received = new AtomicReference(); + var itemNotified = new CountDownLatch(1); + observedItem.setDataValueListener( + (i, value) -> { + received.set(value); + itemNotified.countDown(); + }); + + fixture.scriptable.enqueueDataChange( + subscriptionId(subscription), + 1, + List.of( + new MonitoredItemNotification(clientHandle(throwingItem), VALUE_1), + new MonitoredItemNotification(clientHandle(observedItem), VALUE_2))); + + assertTrue( + itemNotified.await(5, TimeUnit.SECONDS), + "the second MonitoredItem's DataValueListener was never notified: the first item's" + + " listener threw and abandoned the rest of the DataChangeNotification"); + assertEquals(VALUE_2.getValue(), received.get().getValue()); + } + } + + // A NotificationMessage carries a sequence of NotificationData elements. A listener failure in + // one element must not discard the elements behind it: here the StatusChangeNotification that + // tells the application the Subscription was transferred away. + @Test + void remainingNotificationDataIsDeliveredWhenAMonitoredItemListenerThrows() throws Exception { + try (var fixture = new Fixture()) { + OpcUaSubscription subscription = fixture.createSubscription(); + + var item = OpcUaMonitoredItem.newDataItem(NodeIds.Server_ServerStatus_CurrentTime); + subscription.addMonitoredItem(item); + item.setDataValueListener( + (i, value) -> { + throw new IllegalStateException("application DataValueListener failure"); + }); + + var received = new AtomicReference(); + var statusChanged = new CountDownLatch(1); + subscription.setSubscriptionListener( + new OpcUaSubscription.SubscriptionListener() { + @Override + public void onStatusChanged(OpcUaSubscription s, StatusCode status) { + received.set(status); + statusChanged.countDown(); + } + }); + + ExtensionObject[] notificationData = { + fixture.scriptable.encode( + new DataChangeNotification( + new MonitoredItemNotification[] { + new MonitoredItemNotification(clientHandle(item), VALUE_1) + }, + null)), + fixture.scriptable.encode( + new StatusChangeNotification( + new StatusCode(StatusCodes.Good_SubscriptionTransferred), + DiagnosticInfo.NULL_VALUE)) + }; + + fixture.scriptable.enqueueNotification(subscriptionId(subscription), 1, notificationData); + + assertTrue( + statusChanged.await(5, TimeUnit.SECONDS), + "onStatusChanged was never called: the throwing DataValueListener escaped the" + + " NotificationData loop and the StatusChangeNotification was dropped"); + assertEquals(StatusCodes.Good_SubscriptionTransferred, received.get().value()); + } + } + + // The event fan-out has the same structure as the data fan-out and needs the same isolation. + @Test + void monitoredItemEventListenerIsNotifiedWhenSubscriptionListenerThrows() throws Exception { + try (var fixture = new Fixture()) { + OpcUaSubscription subscription = fixture.createSubscription(); + + var item = OpcUaMonitoredItem.newEventItem(NodeIds.Server); + subscription.addMonitoredItem(item); + + var received = new AtomicReference(); + var itemNotified = new CountDownLatch(1); + item.setEventValueListener( + (i, eventValues) -> { + received.set(eventValues); + itemNotified.countDown(); + }); + + subscription.setSubscriptionListener( + new OpcUaSubscription.SubscriptionListener() { + @Override + public void onEventReceived( + OpcUaSubscription s, List items, List fields) { + throw new IllegalStateException("application SubscriptionListener failure"); + } + }); + + ExtensionObject[] notificationData = { + fixture.scriptable.encode( + new EventNotificationList( + new EventFieldList[] { + new EventFieldList( + clientHandle(item), new Variant[] {Variant.ofString("event")}) + })) + }; + + fixture.scriptable.enqueueNotification(subscriptionId(subscription), 1, notificationData); + + assertTrue( + itemNotified.await(5, TimeUnit.SECONDS), + "the MonitoredItem's EventValueListener was never notified: the throwing" + + " SubscriptionListener aborted the fan-out"); + assertEquals(Variant.ofString("event"), received.get()[0]); + } + } + + // The item and value Lists handed to onDataReceived are the very instances the per-item fan-out + // then iterates in lockstep. If they are not defensively copied, an application that mutates + // them silently corrupts delivery: removing an entry shifts the items while the values keep + // their positions, pairing every remaining item with the wrong DataValue. + @Test + void monitoredItemFanOutIsUnaffectedByListenerMutatingTheNotificationLists() throws Exception { + try (var fixture = new Fixture()) { + OpcUaSubscription subscription = fixture.createSubscription(); + + var firstItem = OpcUaMonitoredItem.newDataItem(NodeIds.Server_ServerStatus_CurrentTime); + var secondItem = OpcUaMonitoredItem.newDataItem(NodeIds.Server_ServerStatus_StartTime); + subscription.addMonitoredItem(firstItem); + subscription.addMonitoredItem(secondItem); + + var firstReceived = new AtomicReference(); + var secondReceived = new AtomicReference(); + var itemsNotified = new CountDownLatch(2); + firstItem.setDataValueListener( + (i, value) -> { + firstReceived.set(value); + itemsNotified.countDown(); + }); + secondItem.setDataValueListener( + (i, value) -> { + secondReceived.set(value); + itemsNotified.countDown(); + }); + + subscription.setSubscriptionListener( + new OpcUaSubscription.SubscriptionListener() { + @Override + public void onDataReceived( + OpcUaSubscription s, List items, List values) { + try { + items.remove(0); + } catch (UnsupportedOperationException expected) { + // An immutable List is one correct answer; a defensive copy is the other. Either + // way the fan-out that follows must be unaffected. + } + } + }); + + fixture.scriptable.enqueueDataChange( + subscriptionId(subscription), + 1, + List.of( + new MonitoredItemNotification(clientHandle(firstItem), VALUE_1), + new MonitoredItemNotification(clientHandle(secondItem), VALUE_2))); + + assertTrue( + itemsNotified.await(5, TimeUnit.SECONDS), + "not every MonitoredItem was notified: the SubscriptionListener mutated the List the" + + " fan-out iterates"); + assertEquals( + VALUE_1.getValue(), + firstReceived.get().getValue(), + "first item was paired with the wrong DataValue"); + assertEquals( + VALUE_2.getValue(), + secondReceived.get().getValue(), + "second item was paired with the wrong DataValue"); + } + } + } + + /** + * The SDK's own delivery contract: notifications are handed to the application in the order they + * were received, and the next PublishRequest is withheld until the application has finished + * processing the current one — the backpressure mechanism documented on {@code + * SubscriptionListener.onDataReceived}. Both properties hold only if every callback for a + * NotificationMessage runs inside the delivery task submitted for that message. + */ + @Nested + class CallbackOrdering { + + @Test + void keepAliveIsDeliveredBeforeTheNotificationThatFollowedIt() throws Exception { + try (var fixture = new Fixture()) { + OpcUaSubscription subscription = fixture.createSubscription(); + + // A second Subscription raises the in-flight Publish limit to min(2 + 1, max) = 3 so all + // three scripted responses can be outstanding at once. + fixture.createSubscription(); + + var item = OpcUaMonitoredItem.newDataItem(NodeIds.Server_ServerStatus_CurrentTime); + subscription.addMonitoredItem(item); + + List observed = Collections.synchronizedList(new ArrayList()); + var firstDeliveryStarted = new CountDownLatch(1); + var release = new CountDownLatch(1); + var allDelivered = new CountDownLatch(3); + + subscription.setSubscriptionListener( + new OpcUaSubscription.SubscriptionListener() { + @Override + public void onDataReceived( + OpcUaSubscription s, List items, List values) { + observed.add(DATA_CALLBACK); + allDelivered.countDown(); + + if (firstDeliveryStarted.getCount() > 0) { + firstDeliveryStarted.countDown(); + // Hold the delivery queue open so the two responses that follow queue up behind + // this one instead of racing it. + awaitRelease(release); + } + } + + @Override + public void onKeepAliveReceived(OpcUaSubscription s) { + observed.add(KEEP_ALIVE_CALLBACK); + allDelivered.countDown(); + } + }); + + UInteger subscriptionId = subscriptionId(subscription); + + assertTrue( + awaitTrue(() -> fixture.scriptable.getPublishRequestCount() >= 3, 5000), + "fewer than three Publish requests were in flight"); + + try { + // Sequence numbers 1, 2, 2: contiguous, so no Republish is triggered. + fixture.scriptable.enqueueDataChange( + subscriptionId, + 1, + List.of(new MonitoredItemNotification(clientHandle(item), VALUE_1))); + assertTrue( + firstDeliveryStarted.await(5, TimeUnit.SECONDS), + "the first data notification was not delivered"); + + fixture.scriptable.enqueueKeepAlive(subscriptionId, 2); + assertTrue( + awaitTrue(() -> subscription.getDeliveryQueue().getQueueSize() == 1, 5000), + "the keep-alive was not queued for delivery"); + + fixture.scriptable.enqueueDataChange( + subscriptionId, + 2, + List.of(new MonitoredItemNotification(clientHandle(item), VALUE_2))); + assertTrue( + awaitTrue(() -> subscription.getDeliveryQueue().getQueueSize() == 2, 5000), + "the second data notification was not queued for delivery"); + } finally { + release.countDown(); + } + + assertTrue( + allDelivered.await(5, TimeUnit.SECONDS), + "not every scripted notification was delivered"); + + assertEquals( + List.of(DATA_CALLBACK, KEEP_ALIVE_CALLBACK, DATA_CALLBACK), + List.copyOf(observed), + "callbacks were delivered out of order: the keep-alive was re-queued behind the data" + + " notification that arrived after it"); + } + } + + @Test + void publishRequestIsNotSentUntilTheKeepAliveCallbackReturns() throws Exception { + try (var fixture = new Fixture()) { + OpcUaSubscription subscription = fixture.createSubscription(); + + var inKeepAlive = new CountDownLatch(1); + var release = new CountDownLatch(1); + + subscription.setSubscriptionListener( + new OpcUaSubscription.SubscriptionListener() { + @Override + public void onKeepAliveReceived(OpcUaSubscription s) { + inKeepAlive.countDown(); + awaitRelease(release); + } + }); + + assertTrue( + awaitTrue(() -> fixture.scriptable.getPublishRequestCount() >= 2, 5000), + "fewer than two Publish requests were in flight"); + + int publishRequestsBefore = fixture.scriptable.getPublishRequestCount(); + + fixture.scriptable.enqueueKeepAlive(subscriptionId(subscription), 1); + + boolean publishRequestSent; + try { + assertTrue(inKeepAlive.await(5, TimeUnit.SECONDS), "the keep-alive was not delivered"); + + publishRequestSent = + awaitTrue( + () -> fixture.scriptable.getPublishRequestCount() > publishRequestsBefore, 2000); + } finally { + release.countDown(); + } + + assertFalse( + publishRequestSent, + "a Publish request was sent while onKeepAliveReceived was still running: the" + + " pending-publish permit was released when the callback was queued rather than" + + " when it ran"); + } + } + } + + private static UInteger subscriptionId(OpcUaSubscription subscription) { + return subscription.getSubscriptionId().orElseThrow(); + } + + private static UInteger clientHandle(OpcUaMonitoredItem item) { + return item.getClientHandle().orElseThrow(); + } + + /** Block the calling delivery thread until {@code release} is counted down. */ + private static void awaitRelease(CountDownLatch release) { + try { + if (!release.await(10, TimeUnit.SECONDS)) { + throw new IllegalStateException("timed out waiting to be released"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } + } + + private static boolean awaitTrue(BooleanSupplierThrowing condition, long timeoutMillis) + throws Exception { + long deadline = System.currentTimeMillis() + timeoutMillis; + while (System.currentTimeMillis() < deadline) { + if (condition.get()) { + return true; + } + Thread.sleep(25); + } + return condition.get(); + } + + @FunctionalInterface + private interface BooleanSupplierThrowing { + boolean get() throws Exception; + } + + /** A running Server whose Publish responses are scripted, plus a connected client. */ + private static final class Fixture implements AutoCloseable { + + private final OpcUaServer server; + private final OpcUaClient client; + private final ScriptableSubscriptionServiceSet scriptable; + + Fixture() throws Exception { + TestServer testServer = TestServer.create(); + server = testServer.getServer(); + + scriptable = new ScriptableSubscriptionServiceSet(server); + for (EndpointConfig endpoint : server.getConfig().getEndpoints()) { + server.addServiceSet(endpoint.getPath(), scriptable); + } + + server.startup().get(); + + // Long request timeout so parked Publish requests do not time out during the test. + client = TestClient.create(server, cfg -> cfg.setRequestTimeout(uint(60_000))); + client.connect(); + } + + OpcUaSubscription createSubscription() throws UaException { + var subscription = new OpcUaSubscription(client); + subscription.create(); + return subscription; + } + + @Override + public void close() throws Exception { + scriptable.failParkedRequests(StatusCodes.Bad_NoSubscription); + try { + client.disconnectAsync().get(5, TimeUnit.SECONDS); + } finally { + server.shutdown().get(5, TimeUnit.SECONDS); + } + } + } +} diff --git a/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SubscriptionParameterResetRaceTest.java b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SubscriptionParameterResetRaceTest.java new file mode 100644 index 0000000000..4e727c2542 --- /dev/null +++ b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SubscriptionParameterResetRaceTest.java @@ -0,0 +1,255 @@ +/* + * Copyright (c) 2026 the Eclipse Milo Authors + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.milo.opcua.sdk.client.subscriptions; + +import static org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.Unsigned.uint; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import org.eclipse.milo.opcua.sdk.client.OpcUaClient; +import org.eclipse.milo.opcua.sdk.client.subscriptions.OpcUaSubscription.SyncState; +import org.eclipse.milo.opcua.sdk.server.OpcUaServer; +import org.eclipse.milo.opcua.sdk.test.TestClient; +import org.eclipse.milo.opcua.sdk.test.TestServer; +import org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.UInteger; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * A parameter setter racing {@code OpcUaSubscription.reset()}. + * + *

The setters — {@code setPublishingInterval} and the rest — do a check-then-act: if the + * Subscription exists on the Server they record the new value as a pending modification and mark + * the Subscription {@code UNSYNCHRONIZED} so the next {@code modify()} sends it. Before 93bedb32c + * they did that without {@code lifecycleLock}, while {@code reset()} — which clears the pending + * modifications, drops the {@code ServerState} and returns the Subscription to {@code INITIAL} — + * holds it throughout. + * + *

A setter that read the state before the reset and wrote it afterwards therefore left the + * object in a state no legitimate sequence can produce: {@code UNSYNCHRONIZED} with no {@code + * ServerState} at all. Every transition answers {@code Bad_InvalidState} there — {@code create()} + * because the state is not {@code INITIAL}, {@code modify()} and {@code delete()} because there is + * no {@code ServerState} to name a Subscription with — so the object is permanently unusable, and + * the application's only recourse is to throw it away. {@code reset()} is not only an application + * call: {@code PublishingManager} makes it when the Server reports Bad_Timeout for a Subscription + * and the Session FSM makes it when a Subscription could not be transferred, so the losing side of + * this race is ordinary SDK behavior racing an ordinary application call. + * + *

The setters now take the lock for exactly that check-then-act, so a setter either runs + * entirely before the reset — and has its work discarded, which is correct — or entirely after it, + * and finds the Subscription {@code INITIAL} and leaves it alone. + * + *

Why this is a stress loop and not a deterministic interleaving. The window is the + * setter's own read-modify-write, and a setter performs no I/O and takes no callback, so there is + * no seam a test could park it in: the interleaving cannot be forced, only provoked. It is provoked + * very efficiently, though, because the window is nearly the whole of the setter's body — a thread + * calling it in a loop is inside the window a large fraction of the time — so a single {@code + * reset()} lands in it with high probability, and {@link #ROUNDS} rounds make missing it {@link + * #ROUNDS} times over the only way this test can pass without the fix. The bad state is also + * sticky: once reached, every subsequent setter call re-marks it {@code UNSYNCHRONIZED}, so there + * is no chance of the evidence being tidied away before the assertion runs. Everything is bounded — + * {@link #ROUNDS} rounds, two round trips each, and daemon setter threads that are stopped and + * joined before anything is asserted. + */ +public class SubscriptionParameterResetRaceTest { + + /** + * How many create/race/reset rounds to run. Each is two round trips against a loopback Server, so + * the whole test is a fraction of a second of network time. + */ + private static final int ROUNDS = 40; + + /** + * How many threads call the setter while the reset runs. More than one so that a reset landing in + * a gap between one thread's iterations still lands inside another's window. + */ + private static final int SETTER_THREADS = 2; + + /** How long to wait for the setter threads to start, and for them to stop when asked. */ + private static final long AWAIT_TIMEOUT_MILLIS = 10_000; + + private TestServer testServer; + private OpcUaServer server; + private OpcUaClient client; + private OpcUaSubscription subscription; + + @BeforeEach + void startClientAndServer() throws Exception { + testServer = TestServer.create(); + server = testServer.getServer(); + server.startup().get(); + + client = TestClient.create(server, cfg -> cfg.setRequestTimeout(uint(10_000))); + client.connect(); + + subscription = new OpcUaSubscription(client); + } + + @AfterEach + void stopClientAndServer() throws Exception { + try { + client.disconnectAsync().get(5, TimeUnit.SECONDS); + } finally { + server.shutdown().get(5, TimeUnit.SECONDS); + } + } + + /** + * The defect: {@code reset()} racing {@code setPublishingInterval} must never leave the + * Subscription {@code UNSYNCHRONIZED} with no SubscriptionId, and must always leave it usable. + * + *

Each round creates the Subscription, starts the setter threads, resets, stops the setter + * threads, and asserts. The {@code create()} at the head of the next round is itself the + * assertion that the previous round left the object usable rather than answering {@code + * Bad_InvalidState} forever. + */ + @Test + void aParameterSetterRacingResetLeavesTheSubscriptionUsable() throws Exception { + for (int round = 1; round <= ROUNDS; round++) { + subscription.create(); + + UInteger subscriptionId = subscription.getSubscriptionId().orElseThrow(); + + SetterThreads setters = SetterThreads.start(subscription); + try { + subscription.reset(); + } finally { + setters.stopAndJoin(); + } + + assertEquals( + SyncState.INITIAL, + subscription.getSyncState(), + "round " + + round + + ": reset() returned the Subscription to INITIAL and every setter call after it" + + " found it there, so nothing may have re-marked it. A setter that read the state" + + " before the reset and wrote it after has left it " + + subscription.getSyncState() + + " with SubscriptionId " + + subscription.getSubscriptionId() + + " — a state in which create(), modify() and delete() all answer Bad_InvalidState" + + " and the object can never be used again"); + + // Distinct from the SyncState assertion above: this is the combination that makes the object + // unusable rather than merely mislabelled, and it is what every transition trips over. + assertTrue( + subscription.getSyncState() != SyncState.UNSYNCHRONIZED + || subscription.getSubscriptionId().isPresent(), + "round " + + round + + ": the Subscription reports UNSYNCHRONIZED with no SubscriptionId. There is no" + + " Subscription for the pending modifications to be applied to, so modify() and" + + " delete() answer Bad_InvalidState for want of a ServerState and create() answers" + + " it because the state is not INITIAL"); + + // reset() only discards the client's knowledge of the Subscription; the Server is still + // running it and nothing the client object can do names it any more. + client.deleteSubscriptions(List.of(subscriptionId)); + } + + // The Subscription must still be usable after every round, by the transitions themselves and + // not + // only by what getSyncState() reports. + subscription.create(); + + assertEquals( + SyncState.SYNCHRONIZED, + subscription.getSyncState(), + "the Subscription must still be creatable after the races above"); + + subscription.setPublishingInterval(500.0); + subscription.modify(); + + assertEquals( + SyncState.SYNCHRONIZED, + subscription.getSyncState(), + "the Subscription must still be modifiable after the races above"); + + subscription.delete(); + } + + // region fixture helpers + + /** + * Daemon threads calling {@code setPublishingInterval} in a tight loop, so that a {@code reset()} + * made while they run has a good chance of landing inside one of their check-then-act windows. + * + *

{@code setPublishingInterval} is used because it is the widest of the setters: with the + * LifetimeCount and MaxKeepAliveCount derived from the interval (the default), it performs three + * separate check-then-acts, one of its own and one in each of the setters it delegates to. + */ + private static final class SetterThreads { + + private final AtomicBoolean stopped = new AtomicBoolean(false); + private final AtomicLong iterations = new AtomicLong(0); + private final List threads = new ArrayList<>(SETTER_THREADS); + + static SetterThreads start(OpcUaSubscription subscription) throws Exception { + var setters = new SetterThreads(); + + for (int i = 0; i < SETTER_THREADS; i++) { + // A different interval per thread, so no call is a no-op the JIT could hoist away. + double publishingInterval = 100.0 + i; + + var thread = + new Thread( + () -> { + while (!setters.stopped.get()) { + subscription.setPublishingInterval(publishingInterval); + setters.iterations.incrementAndGet(); + } + }, + "parameter-setter-" + i); + thread.setDaemon(true); + + setters.threads.add(thread); + thread.start(); + } + + // The reset has to race a setter that is already running, not one that is still starting. + assertTrue( + setters.awaitIterations(SETTER_THREADS), + "the setter threads never called setPublishingInterval, so nothing raced the reset()"); + + return setters; + } + + void stopAndJoin() throws InterruptedException { + stopped.set(true); + + for (Thread thread : threads) { + thread.join(AWAIT_TIMEOUT_MILLIS); + } + } + + private boolean awaitIterations(long count) throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(AWAIT_TIMEOUT_MILLIS); + + while (System.nanoTime() < deadline) { + if (iterations.get() >= count) { + return true; + } + Thread.sleep(1); + } + + return iterations.get() >= count; + } + } + + // endregion +} diff --git a/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SubscriptionStaleDeliveryTest.java b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SubscriptionStaleDeliveryTest.java new file mode 100644 index 0000000000..2b8488d20b --- /dev/null +++ b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SubscriptionStaleDeliveryTest.java @@ -0,0 +1,483 @@ +/* + * Copyright (c) 2026 the Eclipse Milo Authors + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.milo.opcua.sdk.client.subscriptions; + +import static org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.Unsigned.uint; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.eclipse.milo.opcua.sdk.client.OpcUaClient; +import org.eclipse.milo.opcua.sdk.server.EndpointConfig; +import org.eclipse.milo.opcua.sdk.server.OpcUaServer; +import org.eclipse.milo.opcua.sdk.test.ScriptableSubscriptionServiceSet; +import org.eclipse.milo.opcua.sdk.test.TestClient; +import org.eclipse.milo.opcua.sdk.test.TestServer; +import org.eclipse.milo.opcua.stack.core.NodeIds; +import org.eclipse.milo.opcua.stack.core.StatusCodes; +import org.eclipse.milo.opcua.stack.core.types.builtin.DataValue; +import org.eclipse.milo.opcua.stack.core.types.builtin.DiagnosticInfo; +import org.eclipse.milo.opcua.stack.core.types.builtin.ExtensionObject; +import org.eclipse.milo.opcua.stack.core.types.builtin.StatusCode; +import org.eclipse.milo.opcua.stack.core.types.builtin.Variant; +import org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.UInteger; +import org.eclipse.milo.opcua.stack.core.types.structured.DataChangeNotification; +import org.eclipse.milo.opcua.stack.core.types.structured.MonitoredItemNotification; +import org.eclipse.milo.opcua.stack.core.types.structured.StatusChangeNotification; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +/** + * What happens to a NotificationMessage whose Subscription is discarded while the message is still + * on its way to the application. + * + *

The delivery queue belongs to the {@link OpcUaSubscription} object, not to any one + * Subscription it has represented: {@link OpcUaSubscription#reset()} neither drains nor replaces + * it. An application callback that has not returned therefore holds behind it work belonging to a + * Subscription that may be gone — and gone Subscriptions are replaced, by a {@link + * OpcUaSubscription#create()} on the same object. + * + *

{@link SubscriptionIdentityTest} covers the binding that makes such work recognisable as + * stale: a {@code PublishingManager} entry is bound to the SubscriptionId it was registered under, + * so a message received on one Subscription is never mistaken for a message on the Subscription + * that replaced it. The two cases here are what must then happen to it: notifications the + * application will never be given have to be reported as lost, and a Bad_Timeout that goes stale + * while its own NotificationMessage is being delivered — the guard in {@code + * deliverNotificationMessage} is evaluated once, before the first callback, and application + * callbacks take as long as they take — must not tear down the replacement. + */ +public class SubscriptionStaleDeliveryTest { + + /** How long to wait for something that must happen. */ + private static final long AWAIT_TIMEOUT_MILLIS = 10_000; + + /** + * How long to watch for stale work that must not take effect. Every task involved is already + * queued when the window opens and runs as soon as the delivery queue is released. + */ + private static final long STALE_WORK_WINDOW_MILLIS = 3_000; + + /** + * Long enough that nothing times out on its own: the PublishRequests parked at the Server must + * stay parked until the test decides how they end. + */ + private static final long REQUEST_TIMEOUT_MILLIS = 60_000; + + /** The client keeps one more PublishRequest in flight than it has Subscriptions. */ + private static final int PIPELINE_DEPTH = 2; + + private static final DataValue CURRENT_VALUE = new DataValue(Variant.ofInt32(1)); + private static final DataValue STALE_VALUE = new DataValue(Variant.ofInt32(2)); + + /** + * Part 4 §5.14.5.2 lets a Server "delete the Message with this sequence number from its + * retransmission queue" once the client acknowledges it, and the client acknowledges a + * NotificationMessage when it is received, not when it is delivered. A message discarded + * on its way to the application is therefore data the client asked the Server to forget and then + * dropped: the application is never given it, and no Republish can ever get it back. + * + *

{@code onNotificationDataLost} is the one way an application can find out. Part 4 §5.14.1.1 + * makes the client's sequence-number accounting the detector of everything else that goes + * missing; this is the one loss the accounting cannot see, because the message did arrive and was + * accounted for. + */ + @Nested + class NotificationDataThatCanNoLongerBeAttributed { + + /** + * Control: the identical script with nothing discarded. It proves the fixture does not report + * lost data of its own accord, so the test below fails because of the discard. + */ + @Test + void nothingIsReportedLostWhenTheQueuedNotificationMessageIsStillAttributable() + throws Exception { + + try (var fixture = new Fixture()) { + fixture.awaitPipelineFilled(); + + fixture.enqueueDataChange(1, CURRENT_VALUE); + assertTrue( + fixture.listener.awaitDeliveryStarted(AWAIT_TIMEOUT_MILLIS), + "the first notification was never delivered, so the delivery queue is not held open"); + + fixture.enqueueDataChange(2, STALE_VALUE); + fixture.awaitQueuedDelivery(); + + fixture.listener.release(); + + assertTrue( + fixture.awaitTrue(() -> fixture.listener.received(STALE_VALUE)), + "control: a DataChangeNotification queued behind a blocked delivery must be delivered" + + " once the delivery queue drains"); + assertEquals( + 0, + fixture.listener.notificationDataLostCount(), + "control: nothing was discarded, so nothing may be reported as lost"); + } + } + + @Test + void aDiscardedNotificationMessageIsReportedAsLostData() throws Exception { + try (var fixture = new Fixture()) { + UInteger idA = fixture.subscriptionId(); + fixture.awaitPipelineFilled(); + + // Hold the delivery queue open with a notification the application never finishes handling. + fixture.enqueueDataChange(1, CURRENT_VALUE); + assertTrue( + fixture.listener.awaitDeliveryStarted(AWAIT_TIMEOUT_MILLIS), + "the first notification was never delivered, so the delivery queue is not held open"); + + // Queued behind it: received, acknowledged, and not yet delivered. + fixture.enqueueDataChange(2, STALE_VALUE); + fixture.awaitQueuedDelivery(); + + // The application discards the Subscription the queued message belongs to. + fixture.subscription.reset(); + fixture.subscription.create(); + + assertNotEquals( + idA, + fixture.subscriptionId(), + "the Server reused the SubscriptionId, so nothing distinguishes the two"); + + fixture.listener.release(); + + assertTrue( + fixture.awaitTrue(() -> fixture.listener.notificationDataLostCount() >= 1), + "a NotificationMessage received on Subscription " + + idA + + " was discarded, correctly, because that Subscription no longer exists — but the" + + " application was never told. The client had already acknowledged it, so the" + + " Server may have deleted its only copy: the notifications in it are gone and" + + " nothing reports them lost"); + } + } + } + + /** + * Part 4 §5.13.1.1 makes the SubscriptionId "the Server-assigned identifier for the + * Subscription", so a Bad_Timeout StatusChangeNotification is a statement about the Subscription + * it was received on and about no other. Applied to a Subscription created since, it discards a + * Subscription that is alive on the Server and leaves it unreachable: {@link + * OpcUaSubscription#delete()} needs the ServerState that {@link OpcUaSubscription#reset()} throws + * away. + * + *

The window is inside a single NotificationMessage. {@code deliverNotificationMessage} checks + * that the message's Subscription still exists once, before it hands anything to the application, + * and then walks the NotificationData in order. A DataChangeNotification ahead of the + * StatusChangeNotification puts an application callback of arbitrary duration between the check + * and the teardown, which is time enough for the application to discard the Subscription and + * create another. + */ + @Nested + class BadTimeoutThatGoesStaleMidMessage { + + /** + * Control: the identical NotificationMessage with no reset in the middle of it. It proves the + * Bad_Timeout in the second half of the message is still acted on after the blocking callback + * returns — Part 4 §5.13.1.1 requires the teardown when the Subscription really is the one that + * timed out — so the test below fails because the teardown landed on the wrong Subscription and + * not because it stopped happening. + */ + @Test + void badTimeoutBehindABlockingCallbackResetsTheSubscriptionItWasReceivedFor() throws Exception { + try (var fixture = new Fixture()) { + fixture.awaitPipelineFilled(); + + fixture.enqueueDataChangeThenBadTimeout(1, CURRENT_VALUE); + assertTrue( + fixture.listener.awaitDeliveryStarted(AWAIT_TIMEOUT_MILLIS), + "the DataChangeNotification at the head of the message was never delivered"); + + fixture.listener.release(); + + // Wait on the report, not on the reset: notifyStatusChanged() resets the Subscription + // before it tells the application, so syncState reaching INITIAL does not mean the + // StatusChangeNotification has been delivered yet. + assertTrue( + fixture.awaitTrue(() -> !fixture.listener.statusChanges().isEmpty()), + "control: the Bad_Timeout must be reported to the application, even when an application" + + " callback in the same NotificationMessage ran first"); + assertEquals( + List.of(new StatusCode(StatusCodes.Bad_Timeout)), + fixture.listener.statusChanges(), + "control: the Bad_Timeout, and nothing else, must be reported to the application"); + assertEquals( + OpcUaSubscription.SyncState.INITIAL, + fixture.subscription.getSyncState(), + "control: a Bad_Timeout StatusChangeNotification must reset the Subscription it was" + + " received for; the reset happens before the report, so it has already happened" + + " by the time the report arrives"); + } + } + + @Test + void aStaleBadTimeoutDoesNotTearDownTheSubscriptionCreatedWhileItWasBeingDelivered() + throws Exception { + + try (var fixture = new Fixture()) { + UInteger idA = fixture.subscriptionId(); + fixture.awaitPipelineFilled(); + + // One NotificationMessage carrying a value and then the Subscription's death notice. + fixture.enqueueDataChangeThenBadTimeout(1, CURRENT_VALUE); + assertTrue( + fixture.listener.awaitDeliveryStarted(AWAIT_TIMEOUT_MILLIS), + "the DataChangeNotification at the head of the message was never delivered, so the" + + " Bad_Timeout behind it is not held up by an application callback"); + + // The application discards the timed-out Subscription and creates another, which is what it + // is expected to do — from onDataReceived's point of view the Subscription is simply gone. + fixture.subscription.reset(); + fixture.subscription.create(); + + UInteger idB = fixture.subscriptionId(); + assertNotEquals( + idA, idB, "the Server reused the SubscriptionId, so nothing distinguishes the two"); + + fixture.listener.release(); + + assertFalse( + fixture.awaitTrue( + () -> fixture.subscription.getSyncState() == OpcUaSubscription.SyncState.INITIAL, + STALE_WORK_WINDOW_MILLIS), + "a Bad_Timeout StatusChangeNotification received on Subscription " + + idA + + " tore down Subscription " + + idB + + ", which was created while that message was being delivered and has never timed" + + " out: the guard at the head of the delivery is stale by the time the" + + " StatusChangeNotification behind the application callback is reached"); + assertEquals( + Optional.of(idB), + fixture.subscription.getSubscriptionId(), + "the Subscription created while the stale notification was being delivered lost its" + + " ServerState, so delete() can no longer name it and the Server-side" + + " Subscription is unreachable"); + } + } + } + + // region helpers + + /** + * A listener that suspends the Subscription's delivery queue inside the first {@code + * onDataReceived} callback, so a test can let work pile up behind it — or interrupt the delivery + * of the very NotificationMessage it belongs to — and choose when that work continues. + */ + private static final class BlockingDeliveryListener + implements OpcUaSubscription.SubscriptionListener { + + private final List received = Collections.synchronizedList(new ArrayList<>()); + private final List statusChanges = Collections.synchronizedList(new ArrayList<>()); + private final AtomicInteger notificationDataLost = new AtomicInteger(0); + private final CountDownLatch deliveryStarted = new CountDownLatch(1); + private final CountDownLatch release = new CountDownLatch(1); + + @Override + public void onDataReceived( + OpcUaSubscription subscription, List items, List values) { + + received.addAll(values); + + if (deliveryStarted.getCount() > 0) { + deliveryStarted.countDown(); + + try { + if (!release.await(AWAIT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { + throw new IllegalStateException("the delivery queue was never released"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } + } + } + + @Override + public void onStatusChanged(OpcUaSubscription subscription, StatusCode status) { + statusChanges.add(status); + } + + @Override + public void onNotificationDataLost(OpcUaSubscription subscription) { + notificationDataLost.incrementAndGet(); + } + + boolean awaitDeliveryStarted(long timeoutMillis) throws InterruptedException { + return deliveryStarted.await(timeoutMillis, TimeUnit.MILLISECONDS); + } + + void release() { + release.countDown(); + } + + boolean received(DataValue value) { + synchronized (received) { + return received.stream().anyMatch(v -> v.getValue().equals(value.getValue())); + } + } + + List statusChanges() { + return List.copyOf(statusChanges); + } + + int notificationDataLostCount() { + return notificationDataLost.get(); + } + } + + /** + * A running Server whose Publish responses are scripted, plus a connected client holding one + * Subscription with one MonitoredItem and a {@link BlockingDeliveryListener}. + */ + private static final class Fixture implements AutoCloseable { + + private final OpcUaServer server; + private final OpcUaClient client; + private final ScriptableSubscriptionServiceSet scriptable; + + private final OpcUaSubscription subscription; + private final OpcUaMonitoredItem item; + private final BlockingDeliveryListener listener = new BlockingDeliveryListener(); + + Fixture() throws Exception { + TestServer testServer = TestServer.create(); + server = testServer.getServer(); + + scriptable = new ScriptableSubscriptionServiceSet(server); + for (EndpointConfig endpoint : server.getConfig().getEndpoints()) { + server.addServiceSet(endpoint.getPath(), scriptable); + } + + server.startup().get(); + + client = + TestClient.create( + server, + cfg -> + cfg + // Long request timeout so parked Publish requests do not time out. + .setRequestTimeout(uint(REQUEST_TIMEOUT_MILLIS)) + // No Session keep-alive traffic: the only requests in flight during a test + // are the ones it scripts. + .setKeepAliveInterval(uint(REQUEST_TIMEOUT_MILLIS))); + client.connect(); + + subscription = new OpcUaSubscription(client); + subscription.setSubscriptionListener(listener); + subscription.create(); + + // The MonitoredItem only has to exist on the client: addMonitoredItem assigns the + // ClientHandle + // the notification fan-out looks scripted notifications up by, and no Server-side item + // participates in delivering one. + item = OpcUaMonitoredItem.newDataItem(NodeIds.Server_ServerStatus_CurrentTime); + subscription.addMonitoredItem(item); + } + + UInteger subscriptionId() { + return subscription.getSubscriptionId().orElseThrow(); + } + + void awaitPipelineFilled() throws Exception { + assertTrue( + awaitTrue(() -> scriptable.getParkedRequestCount() >= PIPELINE_DEPTH), + "the client did not fill its Publish pipeline"); + } + + /** Wait until exactly one delivery is waiting behind the blocked one. */ + void awaitQueuedDelivery() throws Exception { + assertTrue( + awaitTrue(() -> subscription.getDeliveryQueue().getQueueSize() == 1), + "the NotificationMessage was never queued behind the blocked delivery, so the scenario" + + " under test did not happen"); + } + + /** Script a PublishResponse carrying {@code value} for the current Subscription. */ + void enqueueDataChange(long sequenceNumber, DataValue value) { + scriptable.enqueueDataChange( + subscriptionId(), sequenceNumber, List.of(notification(value)), uint(sequenceNumber)); + } + + /** + * Script a PublishResponse whose single NotificationMessage carries {@code value} and then a + * Bad_Timeout StatusChangeNotification, in that order. + * + *

Part 4 §5.14.1.1 makes a NotificationMessage's notificationData a list of Notifications, + * so this is one message the client delivers in two callbacks — which is what puts an + * application callback between the Subscription's death notice and the check that it is still + * the current Subscription. + */ + void enqueueDataChangeThenBadTimeout(long sequenceNumber, DataValue value) { + ExtensionObject[] notificationData = { + scriptable.encode( + new DataChangeNotification( + new MonitoredItemNotification[] {notification(value)}, null)), + scriptable.encode( + new StatusChangeNotification( + new StatusCode(StatusCodes.Bad_Timeout), DiagnosticInfo.NULL_VALUE)) + }; + + scriptable.enqueueNotification( + subscriptionId(), sequenceNumber, notificationData, uint(sequenceNumber)); + } + + private MonitoredItemNotification notification(DataValue value) { + return new MonitoredItemNotification(item.getClientHandle().orElseThrow(), value); + } + + boolean awaitTrue(ThrowingBooleanSupplier condition) throws Exception { + return awaitTrue(condition, AWAIT_TIMEOUT_MILLIS); + } + + /** Polls {@code condition} until it holds or {@code timeoutMillis} elapses. */ + boolean awaitTrue(ThrowingBooleanSupplier condition, long timeoutMillis) throws Exception { + long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(timeoutMillis); + + while (System.nanoTime() < deadline) { + if (condition.get()) { + return true; + } + Thread.sleep(25); + } + + return condition.get(); + } + + @Override + public void close() throws Exception { + listener.release(); + scriptable.failParkedRequests(StatusCodes.Bad_NoSubscription); + try { + client.disconnectAsync().get(5, TimeUnit.SECONDS); + } finally { + server.shutdown().get(10, TimeUnit.SECONDS); + } + } + } + + @FunctionalInterface + private interface ThrowingBooleanSupplier { + boolean get() throws Exception; + } + + // endregion +} diff --git a/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SubscriptionTransitionHandoffTest.java b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SubscriptionTransitionHandoffTest.java new file mode 100644 index 0000000000..38d7ccde97 --- /dev/null +++ b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SubscriptionTransitionHandoffTest.java @@ -0,0 +1,391 @@ +/* + * Copyright (c) 2026 the Eclipse Milo Authors + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.milo.opcua.sdk.client.subscriptions; + +import static org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.Unsigned.uint; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; +import org.eclipse.milo.opcua.sdk.client.OpcUaClient; +import org.eclipse.milo.opcua.sdk.server.EndpointConfig; +import org.eclipse.milo.opcua.sdk.server.OpcUaServer; +import org.eclipse.milo.opcua.sdk.test.ScriptableSubscriptionServiceSet; +import org.eclipse.milo.opcua.sdk.test.TestClient; +import org.eclipse.milo.opcua.sdk.test.TestServer; +import org.eclipse.milo.opcua.stack.core.StatusCodes; +import org.eclipse.milo.opcua.stack.core.UaException; +import org.eclipse.milo.opcua.stack.core.types.structured.CreateSubscriptionRequest; +import org.eclipse.milo.opcua.stack.core.types.structured.CreateSubscriptionResponse; +import org.eclipse.milo.opcua.stack.core.types.structured.ModifySubscriptionRequest; +import org.eclipse.milo.opcua.stack.core.types.structured.ModifySubscriptionResponse; +import org.eclipse.milo.opcua.stack.core.util.Unit; +import org.eclipse.milo.opcua.stack.transport.server.ServiceRequestContext; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * How {@code OpcUaSubscription} hands the transition slot from one queued lifecycle transition to + * the next. + * + *

{@code runTransition} lets one transition hold the slot at a time and queues the rest in + * {@code transitionWaiters}; {@code endTransition} hands the slot to the one that has been waiting + * longest by completing its future. Before 93bedb32c it completed that future inline, on the + * stack of the transition that was finishing. That is fine for a transition that has to wait for + * the Server, because the wait unwinds the stack — but not every transition does: {@code + * modifyAsync()} with nothing pending to send, and every transition that is answered {@code + * Bad_InvalidState}, complete synchronously, so the hand-off re-entered {@code + * endTransition} on the same stack. N queued synchronously-completing transitions therefore unwound + * with recursion depth N, and enough of them is a StackOverflowError thrown while the slot is + * claimed and never released — a lifecycle that is frozen for good, not merely one failed call. The + * hand-off now goes through the transport executor. + * + *

Queueing transitions at all requires one to be in flight, so every test here starts by parking + * a {@code create()} inside the Server's CreateSubscription handler: that claims the slot, and + * every {@code modifyAsync()} made while it is parked is queued behind it. Once the gate opens, the + * create's completion is what has to hand the slot down the whole queue. + * + *

Failure manifests either as a queued transition completing exceptionally with a + * StackOverflowError or as the queue never draining at all; both are asserted against, and every + * wait is bounded. + */ +public class SubscriptionTransitionHandoffTest { + + /** + * How many synchronously-completing transitions to queue behind the parked create. + * + *

Chosen an order of magnitude above the depth at which the inline hand-off overflowed the + * stack of the thread that unwinds it. Measured against the pre-fix code on this JVM: 500 drained + * cleanly, 1000 did not, and the hand-off stopped after 883 and 916 transitions in two runs — so + * the threshold is around 900. Frame sizes vary between JVMs, platforms and stack-size settings, + * which is why the margin here is large rather than snug: a value near the threshold would make + * the test a stack-size probe rather than a regression test. + */ + private static final int QUEUED_TRANSITIONS = 10_000; + + /** + * How long the queued transitions are given to drain. They perform no I/O — each one finds + * nothing to send and completes immediately — so the only work is {@link #QUEUED_TRANSITIONS} + * hand-offs through the executor. + */ + private static final long DRAIN_WINDOW_MILLIS = 30_000; + + /** How long to wait for something that must happen. */ + private static final long AWAIT_TIMEOUT_MILLIS = 10_000; + + /** Upper bound on how long the gated Server handler holds a request, so nothing hangs forever. */ + private static final long GATE_TIMEOUT_MILLIS = 30_000; + + /** + * Long enough that nothing times out on its own: neither a parked Publish request nor the gated + * CreateSubscription. + */ + private static final long REQUEST_TIMEOUT_MILLIS = 60_000; + + private TestServer testServer; + private OpcUaServer server; + private OpcUaClient client; + private GatedSubscriptionServiceSet scriptable; + private OpcUaSubscription subscription; + + @BeforeEach + void startClientAndServer() throws Exception { + testServer = TestServer.create(); + server = testServer.getServer(); + + scriptable = new GatedSubscriptionServiceSet(server); + for (EndpointConfig endpoint : server.getConfig().getEndpoints()) { + server.addServiceSet(endpoint.getPath(), scriptable); + } + + server.startup().get(); + + client = + TestClient.create( + server, + cfg -> + cfg.setRequestTimeout(uint(REQUEST_TIMEOUT_MILLIS)) + // No Session keep-alive traffic: the only requests in flight during a test are + // the ones it makes. + .setKeepAliveInterval(uint(REQUEST_TIMEOUT_MILLIS))); + client.connect(); + + subscription = new OpcUaSubscription(client); + } + + @AfterEach + void stopClientAndServer() throws Exception { + scriptable.releaseCreateGate(); + scriptable.failParkedRequests(StatusCodes.Bad_NoSubscription); + try { + client.disconnectAsync().get(5, TimeUnit.SECONDS); + } catch (Exception ignored) { + // A lifecycle frozen by a StackOverflowError in the hand-off cannot be shut down cleanly. + // Tolerated so teardown does not mask the assertion that detected it; the Server shutdown + // below releases what can be released. + } finally { + server.shutdown().get(10, TimeUnit.SECONDS); + } + } + + /** + * The defect. Every queued transition must complete, and the transition slot must be free + * afterwards. + * + *

The queued transitions are {@code modifyAsync()} calls on a Subscription with no pending + * modifications, which Part 4 requires nothing of: {@code modifyTransition} returns an + * already-completed stage without calling ModifySubscription at all. That is the crux — a + * transition that waits for the Server unwinds the stack before the next hand-off, and only a + * synchronously-completing one recurses. The "no ModifySubscription reached the Server" assertion + * below is what keeps that premise honest. + */ + @Test + void queuedSynchronousTransitionsAllCompleteAndReleaseTheSlot() throws Exception { + CompletionStage create = startGatedCreate(); + + var queued = new ArrayList>(QUEUED_TRANSITIONS); + for (int i = 0; i < QUEUED_TRANSITIONS; i++) { + queued.add(subscription.modifyAsync().toCompletableFuture()); + } + + scriptable.releaseCreateGate(); + + // The create is awaited together with the queue it hands the slot to: it is the transition + // whose + // completion does the unwinding, so an error thrown there surfaces on its stage. + var all = new ArrayList<>(queued); + all.add(create.toCompletableFuture()); + + try { + CompletableFuture.allOf(all.toArray(CompletableFuture[]::new)) + .get(DRAIN_WINDOW_MILLIS, TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + fail( + completed(queued) + + " of " + + QUEUED_TRANSITIONS + + " queued transitions completed within " + + DRAIN_WINDOW_MILLIS + + "ms. The hand-off from one queued transition to the next stopped part way through," + + " which is what a StackOverflowError thrown inside endTransition() does: the" + + " transition slot is left claimed and every transition behind it waits for a slot" + + " nobody holds"); + } catch (ExecutionException e) { + fail( + "a transition completed exceptionally after " + + completed(queued) + + " of " + + QUEUED_TRANSITIONS + + " queued transitions had completed: " + + e.getCause() + + ". Transitions that complete synchronously are handed the slot one after another," + + " and an inline hand-off unwinds them with recursion depth " + + QUEUED_TRANSITIONS, + e.getCause()); + } + + assertEquals( + 0, + scriptable.modifySubscriptionArrivals(), + "premise: none of the queued modify transitions may reach the Server. A Subscription with" + + " no pending modifications has nothing to send, and it is exactly that synchronous" + + " completion that recursed — a transition that waits for a response unwinds the stack" + + " first and would not reproduce the defect"); + + // The slot has to be free for something to be able to use it again. A real ModifySubscription, + // so this asserts against the whole transition and not just the queueing. + subscription.setPublishingInterval(500.0); + + CompletableFuture modify = subscription.modifyAsync().toCompletableFuture(); + + try { + modify.get(AWAIT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + fail( + "a modify() made after the queue drained never completed: the transition slot was left" + + " claimed by a hand-off that did not finish, so no further transition on this" + + " Subscription can ever run"); + } + + assertEquals( + OpcUaSubscription.SyncState.SYNCHRONIZED, + subscription.getSyncState(), + "the Subscription must be usable after the queue has drained"); + assertEquals( + 1, + scriptable.modifySubscriptionArrivals(), + "the modify() made after the queue drained had a pending modification and must have reached" + + " the Server"); + } + + /** + * Cancelling the caller's lifecycle future must not cancel the internal completion that owns the + * transition slot. Releasing the slot on cancellation would let the queued delete run while the + * CreateSubscription call is still in flight; never releasing it after the Server answers would + * wedge every later lifecycle operation. + */ + @Test + void cancellingCallerFutureDoesNotReleaseOrWedgeTheTransitionSlot() throws Exception { + CompletionStage create = startGatedCreate(); + CompletableFuture callerFuture = create.toCompletableFuture(); + + assertTrue(callerFuture.cancel(false), "the caller must be able to cancel its future"); + assertTrue(callerFuture.isCancelled(), "cancellation must remain visible to the caller"); + + CompletableFuture queuedDelete = subscription.deleteAsync().toCompletableFuture(); + + assertFalse( + queuedDelete.isDone(), + "cancelling the caller's view must not release the slot while CreateSubscription is still" + + " in flight"); + + scriptable.releaseCreateGate(); + + try { + queuedDelete.get(AWAIT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + fail( + "the DeleteSubscription queued behind the cancelled caller future never ran after the" + + " internal CreateSubscription completed; cancellation prevented transition-slot" + + " cleanup"); + } + + assertTrue(callerFuture.isCancelled(), "internal completion must not undo caller cancellation"); + assertEquals( + OpcUaSubscription.SyncState.INITIAL, + subscription.getSyncState(), + "the queued delete must run after the actual CreateSubscription operation completes"); + } + + // region fixture helpers + + private static int completed(List> futures) { + return (int) futures.stream().filter(CompletableFuture::isDone).count(); + } + + /** + * Start a {@code createAsync()} and return once it is inside the Server's CreateSubscription + * handler, i.e. once it demonstrably holds the transition slot and is waiting for the response. + */ + private CompletionStage startGatedCreate() throws Exception { + scriptable.gateCreates(); + + CompletionStage create = subscription.createAsync(); + + assertTrue( + awaitTrue(() -> scriptable.gatedCreateArrivals() >= 1), + "the create() never reached the Server, so it is not holding the transition slot and" + + " nothing could be queued behind it"); + + return create; + } + + /** Polls {@code condition} until it holds or {@link #AWAIT_TIMEOUT_MILLIS} elapses. */ + private static boolean awaitTrue(ThrowingBooleanSupplier condition) throws Exception { + long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(AWAIT_TIMEOUT_MILLIS); + + while (System.nanoTime() < deadline) { + if (condition.get()) { + return true; + } + Thread.sleep(25); + } + + return condition.get(); + } + + @FunctionalInterface + private interface ThrowingBooleanSupplier { + boolean get() throws Exception; + } + + /** + * A {@link ScriptableSubscriptionServiceSet} that can hold a CreateSubscription inside the Server + * handler until the test releases it, and that counts the ModifySubscription requests it + * receives. + * + *

Holding the request inside the handler is what makes the transition demonstrably in + * flight rather than merely likely to be: the client has sent it and is waiting for the response, + * so it holds the transition slot for as long as the test wants. + */ + private static final class GatedSubscriptionServiceSet extends ScriptableSubscriptionServiceSet { + + private final AtomicInteger gatedCreateArrivals = new AtomicInteger(0); + private final AtomicInteger modifySubscriptionArrivals = new AtomicInteger(0); + + private final CountDownLatch createGate = new CountDownLatch(1); + + private volatile boolean createsGated = false; + + GatedSubscriptionServiceSet(OpcUaServer server) { + super(server); + } + + void gateCreates() { + createsGated = true; + } + + void releaseCreateGate() { + createGate.countDown(); + } + + int gatedCreateArrivals() { + return gatedCreateArrivals.get(); + } + + int modifySubscriptionArrivals() { + return modifySubscriptionArrivals.get(); + } + + @Override + public CreateSubscriptionResponse onCreateSubscription( + ServiceRequestContext context, CreateSubscriptionRequest request) throws UaException { + + if (createsGated) { + gatedCreateArrivals.incrementAndGet(); + + try { + if (!createGate.await(GATE_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { + throw new UaException( + StatusCodes.Bad_Timeout, "the CreateSubscription gate was never opened"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new UaException(StatusCodes.Bad_UnexpectedError, e); + } + } + + return super.onCreateSubscription(context, request); + } + + @Override + public ModifySubscriptionResponse onModifySubscription( + ServiceRequestContext context, ModifySubscriptionRequest request) throws UaException { + + modifySubscriptionArrivals.incrementAndGet(); + + return super.onModifySubscription(context, request); + } + } + + // endregion +} diff --git a/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SubscriptionWatchdogCancelRaceTest.java b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SubscriptionWatchdogCancelRaceTest.java new file mode 100644 index 0000000000..cda8379ca9 --- /dev/null +++ b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SubscriptionWatchdogCancelRaceTest.java @@ -0,0 +1,465 @@ +/* + * Copyright (c) 2026 the Eclipse Milo Authors + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.milo.opcua.sdk.client.subscriptions; + +import static org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.Unsigned.uint; +import static org.junit.jupiter.api.Assertions.assertAll; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.lang.reflect.Field; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Delayed; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import org.eclipse.milo.opcua.sdk.client.OpcUaClient; +import org.eclipse.milo.opcua.sdk.client.SessionActivityListener; +import org.eclipse.milo.opcua.sdk.client.UaSession; +import org.eclipse.milo.opcua.sdk.server.EndpointConfig; +import org.eclipse.milo.opcua.sdk.server.OpcUaServer; +import org.eclipse.milo.opcua.sdk.test.ScriptableSubscriptionServiceSet; +import org.eclipse.milo.opcua.sdk.test.TestClient; +import org.eclipse.milo.opcua.sdk.test.TestServer; +import org.eclipse.milo.opcua.stack.core.StatusCodes; +import org.eclipse.milo.opcua.stack.core.UaException; +import org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.UInteger; +import org.junit.jupiter.api.Test; + +/** + * The Subscription watchdog is re-armed and cancelled from two unrelated threads: {@code + * PublishingManager} re-arms it from the Publish completion handler (dispatched on the transport + * executor), and the Session FSM cancels it from {@code onSessionInactive()} (dispatched on the + * same executor, which is an unbounded cached pool, so the two run concurrently). + * + *

Those two entry points do not share a lock: {@code resetWatchdogTimer()} and {@code + * cancelWatchdogTimer()} are {@code synchronized} on the Subscription, but the Session callbacks + * call the {@code WatchdogTimer}'s own {@code reset()}/{@code cancel()} directly, and the {@code + * AtomicReference} holding the scheduled future provides no mutual exclusion across the + * read-cancel-schedule-store sequence a re-arm performs. + * + *

The invariant under test is the one an application depends on: once the watchdog has been + * cancelled it must not fire. A spurious {@code onWatchdogTimerElapsed()} tells the application its + * Subscription is dead when it is not, which is exactly what the callback exists to rule out. + * + *

The interleaving is forced, not raced: the client is given a {@link + * GateScheduledExecutor} that suspends the re-arming thread inside {@code schedule()} — after it + * has cancelled the current future but before it stores the new one — and the cancelling thread is + * released into that window. This makes the failure deterministic instead of probabilistic. + */ +public class SubscriptionWatchdogCancelRaceTest { + + /** Made explicit rather than relying on the default, because the delay below depends on it. */ + private static final double WATCHDOG_MULTIPLIER = 1.5; + + private static final double PUBLISHING_INTERVAL = 111.0; + + /** ceil(111 / 111) derives a MaxKeepAliveCount of 1. */ + private static final double TARGET_KEEP_ALIVE_INTERVAL = 111.0; + + private static final UInteger EXPECTED_MAX_KEEP_ALIVE_COUNT = uint(1); + + /** + * 111 * (1 + 1) * 1.5 — the watchdog delay implied by the revised parameters. Deliberately not a + * round number: {@link GateScheduledExecutor} identifies watchdog tasks by their delay, and no + * other client task is scheduled 333ms out. + */ + private static final long WATCHDOG_DELAY_MILLIS = 333; + + /** How long to watch for an expiry that must not happen. ~9x {@link #WATCHDOG_DELAY_MILLIS}. */ + private static final long SPURIOUS_EXPIRY_WINDOW_MILLIS = 3_000; + + /** How long to wait for an expiry that must happen. */ + private static final long EXPIRY_TIMEOUT_MILLIS = 10_000; + + private static final long THREAD_TIMEOUT_MILLIS = 10_000; + + /** + * A cancel that lands while a re-arm is in flight must still win: the watchdog was cancelled, so + * nothing may fire afterwards. + * + *

Today the re-arming thread stores its new future after the cancelling thread has already + * taken the old one out of the {@code AtomicReference}, so the new future survives a cancel that + * happened after it was created and fires ~{@value #WATCHDOG_DELAY_MILLIS}ms later. When the + * cancel comes from {@code cancelWatchdogTimer()} — the Session-fault path — the {@code + * WatchdogTimer} is also unreachable by then, so nothing can ever cancel that future. + */ + @Test + void watchdogDoesNotElapseWhenCancelRacesWithReset() throws Exception { + try (var fixture = new Fixture()) { + OpcUaSubscription subscription = fixture.createSubscription(); + assertRevisedParameters(subscription); + + SessionActivityListener watchdog = watchdogListener(subscription); + UaSession session = fixture.client.getSession(); + + RecordingScheduledFuture armed = fixture.gate.onlyWatchdogFuture(); + + // Thread 1: a Publish response arrived, so PublishingManager re-arms the watchdog. It is + // suspended inside schedule(), after cancelling the armed future and before storing the new + // one. + var rearm = new Thread(subscription::resetWatchdogTimer, "watchdog-rearm"); + fixture.gate.gateNextScheduleFrom(rearm); + rearm.start(); + + assertTrue( + fixture.gate.awaitGated(THREAD_TIMEOUT_MILLIS), + "the re-arming thread never reached schedule()"); + + // Thread 2: the Session left Active. This is the callback SessionFsmFactory dispatches, on + // the object it dispatches it to, and it takes no lock on the Subscription. + var cancel = new Thread(() -> watchdog.onSessionInactive(session), "watchdog-cancel"); + cancel.start(); + + // The interleaving point, observed rather than slept for: the second cancel() of the armed + // future is the cancelling thread having already emptied the AtomicReference. Bounded, + // because an implementation that serializes the two threads will not get there until the + // re-arm is released — and then there is no race and the assertion below simply holds. + armed.awaitCancelCount(2, SPURIOUS_EXPIRY_WINDOW_MILLIS); + + // Installed now: the future armed at create() has been cancelled twice over and the re-arm's + // future does not exist yet, so the only expiry this listener can observe is the one the + // race leaks. + var elapsed = new CountDownLatch(1); + subscription.setSubscriptionListener(watchdogElapsedListener(elapsed)); + + fixture.gate.release(); + joinAll(rearm, cancel); + + assertFalse( + elapsed.await(SPURIOUS_EXPIRY_WINDOW_MILLIS, TimeUnit.MILLISECONDS), + "watchdog elapsed although it had been cancelled; the re-arm that was in flight stored a" + + " future the cancel could no longer see, and the application is now told a healthy" + + " Subscription is dead"); + } + } + + /** + * The control that proves {@link #watchdogDoesNotElapseWhenCancelRacesWithReset()} is not + * vacuous: with the same gated re-arm and no cancel at all, the future the re-arm creates does + * fire and the listener does see it. + */ + @Test + void watchdogElapsesAfterResetWhenItIsNotCancelled() throws Exception { + try (var fixture = new Fixture()) { + OpcUaSubscription subscription = fixture.createSubscription(); + assertRevisedParameters(subscription); + + var rearm = new Thread(subscription::resetWatchdogTimer, "watchdog-rearm"); + fixture.gate.gateNextScheduleFrom(rearm); + rearm.start(); + + assertTrue( + fixture.gate.awaitGated(THREAD_TIMEOUT_MILLIS), + "the re-arming thread never reached schedule()"); + + var elapsed = new CountDownLatch(1); + subscription.setSubscriptionListener(watchdogElapsedListener(elapsed)); + + fixture.gate.release(); + joinAll(rearm); + + assertTrue( + elapsed.await(EXPIRY_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), + "watchdog did not elapse although it was re-armed and no Publish response was ever" + + " delivered"); + } + } + + /** + * The second control, and the one that isolates the race as the cause: the same two operations in + * a defined order — the re-arm completes, then the Session goes inactive — do cancel the + * watchdog. Only overlapping them loses the future. + */ + @Test + void watchdogDoesNotElapseWhenCancelFollowsACompletedReset() throws Exception { + try (var fixture = new Fixture()) { + OpcUaSubscription subscription = fixture.createSubscription(); + assertRevisedParameters(subscription); + + SessionActivityListener watchdog = watchdogListener(subscription); + UaSession session = fixture.client.getSession(); + + var rearm = new Thread(subscription::resetWatchdogTimer, "watchdog-rearm"); + fixture.gate.gateNextScheduleFrom(rearm); + rearm.start(); + + assertTrue( + fixture.gate.awaitGated(THREAD_TIMEOUT_MILLIS), + "the re-arming thread never reached schedule()"); + + var elapsed = new CountDownLatch(1); + subscription.setSubscriptionListener(watchdogElapsedListener(elapsed)); + + fixture.gate.release(); + joinAll(rearm); + + watchdog.onSessionInactive(session); + + assertFalse( + elapsed.await(SPURIOUS_EXPIRY_WINDOW_MILLIS, TimeUnit.MILLISECONDS), + "watchdog elapsed although it was cancelled after the re-arm had completed"); + } + } + + private static OpcUaSubscription.SubscriptionListener watchdogElapsedListener( + CountDownLatch elapsed) { + + return new OpcUaSubscription.SubscriptionListener() { + @Override + public void onWatchdogTimerElapsed(OpcUaSubscription subscription) { + elapsed.countDown(); + } + }; + } + + private static void joinAll(Thread... threads) throws InterruptedException { + for (Thread thread : threads) { + thread.join(THREAD_TIMEOUT_MILLIS); + assertFalse(thread.isAlive(), thread.getName() + " did not finish"); + } + } + + /** + * The {@code WatchdogTimer} is a {@link SessionActivityListener} registered with the client and + * is not reachable through any API. Reflection obtains the same reference the Session FSM holds + * so the test can deliver the callback the FSM delivers, to the object it delivers it to; + * everything exercised from there is production code on production state. + */ + private static SessionActivityListener watchdogListener(OpcUaSubscription subscription) + throws Exception { + + Field field = OpcUaSubscription.class.getDeclaredField("watchdogTimer"); + field.setAccessible(true); + + Object watchdogTimer = field.get(subscription); + assertNotNull(watchdogTimer, "the Subscription has no watchdog timer"); + + return (SessionActivityListener) watchdogTimer; + } + + /** + * The watchdog delay is derived from the revised parameters, so the timings above are only + * meaningful if the Server returned what was requested. Asserted up front so a Server that + * revises them fails loudly instead of producing a mystery timeout. + */ + private static void assertRevisedParameters(OpcUaSubscription subscription) { + assertAll( + () -> + assertEquals( + PUBLISHING_INTERVAL, + subscription.getRevisedPublishingInterval().orElseThrow(), + "revised PublishingInterval"), + () -> + assertEquals( + EXPECTED_MAX_KEEP_ALIVE_COUNT, + subscription.getRevisedMaxKeepAliveCount().orElseThrow(), + "revised MaxKeepAliveCount")); + } + + /** + * The client's {@code ScheduledExecutorService}, with two test affordances: + * + *

    + *
  • one nominated thread is suspended on its next {@code schedule()} call until the test + * releases it, which is what turns the race into a fixed interleaving; + *
  • futures scheduled at the watchdog delay are handed back wrapped so the test can observe + * cancellation of a specific future. + *
+ * + * Every other scheduling the client does — Session keep-alives, reconnect back-off — passes + * straight through. + */ + private static final class GateScheduledExecutor extends ScheduledThreadPoolExecutor { + + private final AtomicReference gatedThread = new AtomicReference<>(); + private final CountDownLatch gated = new CountDownLatch(1); + private final CountDownLatch released = new CountDownLatch(1); + + private final List watchdogFutures = new CopyOnWriteArrayList<>(); + + GateScheduledExecutor() { + super(4); + } + + void gateNextScheduleFrom(Thread thread) { + gatedThread.set(thread); + } + + boolean awaitGated(long timeoutMillis) throws InterruptedException { + return gated.await(timeoutMillis, TimeUnit.MILLISECONDS); + } + + void release() { + released.countDown(); + } + + /** The single watchdog future armed when the Subscription was created. */ + RecordingScheduledFuture onlyWatchdogFuture() { + assertEquals(1, watchdogFutures.size(), "expected exactly one armed watchdog future"); + + return watchdogFutures.get(0); + } + + @Override + public ScheduledFuture schedule(Runnable command, long delay, TimeUnit unit) { + if (gatedThread.compareAndSet(Thread.currentThread(), null)) { + gated.countDown(); + try { + if (!released.await(THREAD_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { + throw new IllegalStateException("gate was never released"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + ScheduledFuture future = super.schedule(command, delay, unit); + + if (unit.toMillis(delay) == WATCHDOG_DELAY_MILLIS) { + var recording = new RecordingScheduledFuture(future); + watchdogFutures.add(recording); + return recording; + } + + return future; + } + } + + /** A {@link ScheduledFuture} that counts the cancellation attempts made against it. */ + private static final class RecordingScheduledFuture implements ScheduledFuture { + + private final AtomicInteger cancelCount = new AtomicInteger(0); + + private final ScheduledFuture delegate; + + private RecordingScheduledFuture(ScheduledFuture delegate) { + this.delegate = delegate; + } + + /** + * Wait until this future has been cancelled at least {@code count} times, or the timeout + * expires. Returning early on timeout is intentional: it keeps a test that no longer produces + * the interleaving from hanging. + */ + void awaitCancelCount(int count, long timeoutMillis) throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(timeoutMillis); + + while (cancelCount.get() < count && System.nanoTime() < deadline) { + TimeUnit.MILLISECONDS.sleep(1); + } + } + + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + cancelCount.incrementAndGet(); + + return delegate.cancel(mayInterruptIfRunning); + } + + @Override + public long getDelay(TimeUnit unit) { + return delegate.getDelay(unit); + } + + @Override + public int compareTo(Delayed o) { + return delegate.compareTo(o); + } + + @Override + public boolean isCancelled() { + return delegate.isCancelled(); + } + + @Override + public boolean isDone() { + return delegate.isDone(); + } + + @Override + public Object get() throws InterruptedException, ExecutionException { + return delegate.get(); + } + + @Override + public Object get(long timeout, TimeUnit unit) + throws InterruptedException, ExecutionException, TimeoutException { + + return delegate.get(timeout, unit); + } + } + + /** + * A running Server whose Publish requests are all parked — no Publish response may re-arm the + * watchdog behind the test's back — plus a client driven by a {@link GateScheduledExecutor}. + */ + private static final class Fixture implements AutoCloseable { + + private final OpcUaServer server; + private final OpcUaClient client; + private final ScriptableSubscriptionServiceSet scriptable; + private final GateScheduledExecutor gate = new GateScheduledExecutor(); + + Fixture() throws Exception { + TestServer testServer = TestServer.create(); + server = testServer.getServer(); + + scriptable = new ScriptableSubscriptionServiceSet(server); + for (EndpointConfig endpoint : server.getConfig().getEndpoints()) { + server.addServiceSet(endpoint.getPath(), scriptable); + } + + server.startup().get(); + + // Long request timeout so the parked Publish requests do not time out during the test. + client = + TestClient.create( + server, + transportConfig -> transportConfig.setScheduledExecutor(gate), + cfg -> cfg.setRequestTimeout(uint(60_000))); + client.connect(); + } + + OpcUaSubscription createSubscription() throws UaException { + var subscription = new OpcUaSubscription(client); + subscription.setWatchdogMultiplier(WATCHDOG_MULTIPLIER); + subscription.setPublishingInterval(PUBLISHING_INTERVAL); + subscription.setTargetKeepAliveInterval(TARGET_KEEP_ALIVE_INTERVAL); + subscription.create(); + + return subscription; + } + + @Override + public void close() throws Exception { + gate.release(); + scriptable.failParkedRequests(StatusCodes.Bad_NoSubscription); + try { + client.disconnectAsync().get(5, TimeUnit.SECONDS); + } finally { + try { + server.shutdown().get(5, TimeUnit.SECONDS); + } finally { + gate.shutdownNow(); + } + } + } + } +} diff --git a/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SubscriptionWatchdogRearmOnModifyTest.java b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SubscriptionWatchdogRearmOnModifyTest.java new file mode 100644 index 0000000000..26c55e5280 --- /dev/null +++ b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SubscriptionWatchdogRearmOnModifyTest.java @@ -0,0 +1,238 @@ +/* + * Copyright (c) 2026 the Eclipse Milo Authors + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.milo.opcua.sdk.client.subscriptions; + +import static org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.Unsigned.uint; +import static org.junit.jupiter.api.Assertions.assertAll; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import org.eclipse.milo.opcua.sdk.client.OpcUaClient; +import org.eclipse.milo.opcua.sdk.server.EndpointConfig; +import org.eclipse.milo.opcua.sdk.server.OpcUaServer; +import org.eclipse.milo.opcua.sdk.test.ScriptableSubscriptionServiceSet; +import org.eclipse.milo.opcua.sdk.test.TestClient; +import org.eclipse.milo.opcua.sdk.test.TestServer; +import org.eclipse.milo.opcua.stack.core.StatusCodes; +import org.eclipse.milo.opcua.stack.core.UaException; +import org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.UInteger; +import org.junit.jupiter.api.Test; + +/** + * The Subscription watchdog fires {@code onWatchdogTimerElapsed()} when no Publish response has + * been seen for {@code revisedPublishingInterval * (revisedMaxKeepAliveCount + 1) * + * watchdogMultiplier} milliseconds — i.e. when the Server has missed the keep-alive it promised in + * its ModifySubscription/CreateSubscription response. + * + *

ModifySubscription is the only place besides create where those two parameters can change + * (Part 4 §5.14.3), so {@code modify()} must re-arm the watchdog from the revised parameters + * the Server just returned. Re-arming from the pre-modify parameters breaks the watchdog in both + * directions: slowing the Subscription down makes it cry wolf long before the Server's next + * keep-alive is due, and speeding it up leaves it armed for the old, longer delay so a genuinely + * dead Subscription goes unreported. + * + *

Every Publish request is parked by {@link ScriptableSubscriptionServiceSet} and never + * answered, because a Publish response is the other thing that re-arms the watchdog and would mask + * the arming performed by {@code modify()}. + */ +public class SubscriptionWatchdogRearmOnModifyTest { + + /** Made explicit rather than relying on the default, because the delays below depend on it. */ + private static final double WATCHDOG_MULTIPLIER = 1.5; + + private static final double FAST_PUBLISHING_INTERVAL = 100.0; + private static final double FAST_TARGET_KEEP_ALIVE_INTERVAL = 1_000.0; + + private static final double SLOW_PUBLISHING_INTERVAL = 2_000.0; + private static final double SLOW_TARGET_KEEP_ALIVE_INTERVAL = 20_000.0; + + /** + * ceil(1000 / 100) and ceil(20000 / 2000) both derive a MaxKeepAliveCount of 10, so the two + * configurations below differ only in their PublishingInterval and the Server revises neither + * count. + */ + private static final UInteger EXPECTED_MAX_KEEP_ALIVE_COUNT = uint(10); + + /** 100 * (10 + 1) * 1.5 — the watchdog delay implied by the fast configuration. */ + private static final long FAST_WATCHDOG_DELAY_MILLIS = 1_650; + + /** 2000 * (10 + 1) * 1.5 — the watchdog delay implied by the slow configuration. */ + private static final long SLOW_WATCHDOG_DELAY_MILLIS = 33_000; + + /** + * How long to watch for a watchdog expiry. Comfortably longer than {@link + * #FAST_WATCHDOG_DELAY_MILLIS} and far shorter than {@link #SLOW_WATCHDOG_DELAY_MILLIS}, so the + * two are never confused. + */ + private static final long OBSERVATION_WINDOW_MILLIS = 6_000; + + /** + * Modifying to a slower PublishingInterval must push the watchdog out. If the timer is re-armed + * before the revised parameters are installed it stays armed for the old 1650 ms delay and + * reports the Subscription dead ~31 seconds before the Server's next keep-alive is even due. + */ + @Test + void modifyToSlowerIntervalDoesNotElapseWatchdogAtThePreModifyInterval() throws Exception { + try (var fixture = new Fixture()) { + OpcUaSubscription subscription = + fixture.createSubscription(FAST_PUBLISHING_INTERVAL, FAST_TARGET_KEEP_ALIVE_INTERVAL); + + assertRevisedParameters(subscription, FAST_PUBLISHING_INTERVAL); + + subscription.setPublishingInterval(SLOW_PUBLISHING_INTERVAL); + subscription.setTargetKeepAliveInterval(SLOW_TARGET_KEEP_ALIVE_INTERVAL); + subscription.modify(); + + assertRevisedParameters(subscription, SLOW_PUBLISHING_INTERVAL); + + CountDownLatch elapsed = watchdogElapsedLatch(subscription); + + assertFalse( + elapsed.await(OBSERVATION_WINDOW_MILLIS, TimeUnit.MILLISECONDS), + "watchdog elapsed within " + + OBSERVATION_WINDOW_MILLIS + + "ms, but the revised parameters put the next expiry " + + SLOW_WATCHDOG_DELAY_MILLIS + + "ms out; it was re-armed with the pre-modify delay of " + + FAST_WATCHDOG_DELAY_MILLIS + + "ms"); + } + } + + /** + * The mirror case, and the control that proves {@link + * #modifyToSlowerIntervalDoesNotElapseWatchdogAtThePreModifyInterval()} is not vacuous: with the + * watchdog correctly re-armed the expiry does arrive, and it arrives on the revised schedule. + * Re-arming from the pre-modify parameters leaves it armed for the old 33 second delay, so a + * Subscription that has gone silent is not reported for another half minute. + */ + @Test + void modifyToFasterIntervalElapsesWatchdogAtThePostModifyInterval() throws Exception { + try (var fixture = new Fixture()) { + OpcUaSubscription subscription = + fixture.createSubscription(SLOW_PUBLISHING_INTERVAL, SLOW_TARGET_KEEP_ALIVE_INTERVAL); + + assertRevisedParameters(subscription, SLOW_PUBLISHING_INTERVAL); + + subscription.setPublishingInterval(FAST_PUBLISHING_INTERVAL); + subscription.setTargetKeepAliveInterval(FAST_TARGET_KEEP_ALIVE_INTERVAL); + subscription.modify(); + + assertRevisedParameters(subscription, FAST_PUBLISHING_INTERVAL); + + CountDownLatch elapsed = watchdogElapsedLatch(subscription); + + assertTrue( + elapsed.await(OBSERVATION_WINDOW_MILLIS, TimeUnit.MILLISECONDS), + "watchdog did not elapse within " + + OBSERVATION_WINDOW_MILLIS + + "ms, but the revised parameters put the expiry " + + FAST_WATCHDOG_DELAY_MILLIS + + "ms out; it is still armed for the pre-modify delay of " + + SLOW_WATCHDOG_DELAY_MILLIS + + "ms"); + } + } + + /** + * The watchdog delay is computed from the revised parameters, so the test's arithmetic is only + * meaningful if the Server returned the parameters that were requested. Asserted before the + * timing assertions so a Server that revises them fails loudly instead of producing a mystery + * timeout. + */ + private static void assertRevisedParameters( + OpcUaSubscription subscription, double expectedPublishingInterval) { + + assertAll( + () -> + assertEquals( + expectedPublishingInterval, + subscription.getRevisedPublishingInterval().orElseThrow(), + "revised PublishingInterval"), + () -> + assertEquals( + EXPECTED_MAX_KEEP_ALIVE_COUNT, + subscription.getRevisedMaxKeepAliveCount().orElseThrow(), + "revised MaxKeepAliveCount")); + } + + /** + * Install a listener that counts down when the watchdog elapses. + * + *

Deliberately installed after {@code modify()}: the arming performed when the + * Subscription was created may already have expired while the ModifySubscription round trip was + * in flight, and that expiry is not the one under test. {@code notifyWatchdogTimerElapsed()} + * reads the listener when the timer fires, so an expiry that predates this call cannot be + * observed. + */ + private static CountDownLatch watchdogElapsedLatch(OpcUaSubscription subscription) { + var latch = new CountDownLatch(1); + + subscription.setSubscriptionListener( + new OpcUaSubscription.SubscriptionListener() { + @Override + public void onWatchdogTimerElapsed(OpcUaSubscription subscription) { + latch.countDown(); + } + }); + + return latch; + } + + /** A running Server whose Publish requests are all parked, plus a connected client. */ + private static final class Fixture implements AutoCloseable { + + private final OpcUaServer server; + private final OpcUaClient client; + private final ScriptableSubscriptionServiceSet scriptable; + + Fixture() throws Exception { + TestServer testServer = TestServer.create(); + server = testServer.getServer(); + + scriptable = new ScriptableSubscriptionServiceSet(server); + for (EndpointConfig endpoint : server.getConfig().getEndpoints()) { + server.addServiceSet(endpoint.getPath(), scriptable); + } + + server.startup().get(); + + // Long request timeout so the parked Publish requests do not time out during the test. + client = TestClient.create(server, cfg -> cfg.setRequestTimeout(uint(60_000))); + client.connect(); + } + + OpcUaSubscription createSubscription(double publishingInterval, double targetKeepAliveInterval) + throws UaException { + + var subscription = new OpcUaSubscription(client); + subscription.setWatchdogMultiplier(WATCHDOG_MULTIPLIER); + subscription.setPublishingInterval(publishingInterval); + subscription.setTargetKeepAliveInterval(targetKeepAliveInterval); + subscription.create(); + + return subscription; + } + + @Override + public void close() throws Exception { + scriptable.failParkedRequests(StatusCodes.Bad_NoSubscription); + try { + client.disconnectAsync().get(5, TimeUnit.SECONDS); + } finally { + server.shutdown().get(5, TimeUnit.SECONDS); + } + } + } +} diff --git a/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SubscriptionWatchdogRecoveryStarvationTest.java b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SubscriptionWatchdogRecoveryStarvationTest.java new file mode 100644 index 0000000000..83822eda8b --- /dev/null +++ b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SubscriptionWatchdogRecoveryStarvationTest.java @@ -0,0 +1,338 @@ +/* + * Copyright (c) 2026 the Eclipse Milo Authors + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.milo.opcua.sdk.client.subscriptions; + +import static org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.Unsigned.uint; +import static org.junit.jupiter.api.Assertions.assertAll; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import org.eclipse.milo.opcua.sdk.client.OpcUaClient; +import org.eclipse.milo.opcua.sdk.client.SessionActivityListener; +import org.eclipse.milo.opcua.sdk.client.UaSession; +import org.eclipse.milo.opcua.sdk.server.EndpointConfig; +import org.eclipse.milo.opcua.sdk.server.OpcUaServer; +import org.eclipse.milo.opcua.sdk.test.ScriptableSubscriptionServiceSet; +import org.eclipse.milo.opcua.sdk.test.TestClient; +import org.eclipse.milo.opcua.sdk.test.TestServer; +import org.eclipse.milo.opcua.stack.core.StatusCodes; +import org.eclipse.milo.opcua.stack.core.UaException; +import org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.UInteger; +import org.eclipse.milo.opcua.stack.core.types.structured.RepublishRequest; +import org.eclipse.milo.opcua.stack.core.types.structured.RepublishResponse; +import org.junit.jupiter.api.Test; + +/** + * What the Subscription watchdog is allowed to conclude while the client is still recovering from a + * reconnect. + * + *

The watchdog exists to report a Server that has stopped honouring the keep-alive interval it + * promised in its CreateSubscription response (Part 4 §5.13.2), and the only event that feeds it is + * a PublishResponse. Part 4 §6.7 makes the client run a Republish loop before Publish handling + * resumes — "After the Republish returns Bad_MessageNotAvailable the Client shall start sending + * Publish requests with the normal Publish handling" — so for as long as that loop runs no + * PublishResponse can arrive: the client is holding its own Publish traffic shut. + * + *

Arming the watchdog when the Session becomes Active therefore starts a countdown that nothing + * is able to stop. A recovery that takes longer than the watchdog delay reports {@code + * onWatchdogTimerElapsed} for a Subscription that is recovering exactly as the specification says + * it should, which is the opposite of what the callback means: it tells the application its + * Subscription is dead when the client is in the middle of bringing it back. The watchdog has to be + * armed when Publish traffic is allowed to flow again, not when the Session becomes Active. + * + *

The recovery is held by the test rather than slowed down by a sleep: the Server does not + * answer the Republish that begins the drain until the test lets it, so the length of the recovery + * is a property of the test and not of anyone's timing. + */ +public class SubscriptionWatchdogRecoveryStarvationTest { + + /** Made explicit rather than relying on the default, because the delay below depends on it. */ + private static final double WATCHDOG_MULTIPLIER = 1.5; + + private static final double PUBLISHING_INTERVAL = 111.0; + + /** ceil(111 / 111) derives a MaxKeepAliveCount of 1. */ + private static final double TARGET_KEEP_ALIVE_INTERVAL = 111.0; + + private static final UInteger EXPECTED_MAX_KEEP_ALIVE_COUNT = uint(1); + + /** 111 * (1 + 1) * 1.5 — the watchdog delay implied by the revised parameters. */ + private static final long WATCHDOG_DELAY_MILLIS = 333; + + /** + * How long the Republish drain is held, and therefore how long the recovery lasts: ~9x {@link + * #WATCHDOG_DELAY_MILLIS}, so a watchdog armed anywhere in the recovery has ample time to fire. + */ + private static final long RECOVERY_HOLD_MILLIS = 3_000; + + /** + * The Session FSM waits one second in {@code ReactivatingWait} before its first re-activation + * attempt, and doubles the wait on each failure; this window allows for several attempts. + */ + private static final long RECONNECT_TIMEOUT_MILLIS = 20_000; + + /** How long to wait for an expiry that must happen. */ + private static final long WATCHDOG_TIMEOUT_MILLIS = 10_000; + + /** How long to wait for something else that must happen. */ + private static final long AWAIT_TIMEOUT_MILLIS = 10_000; + + /** Upper bound on how long the Republish drain is held, so nothing hangs indefinitely. */ + private static final long GATE_TIMEOUT_MILLIS = 30_000; + + /** + * A recovery that outlasts the watchdog delay must not fire the watchdog. Nothing has gone wrong: + * the Subscription is alive, the Server is answering, and the reason no PublishResponse has + * arrived is that the client is running the Republish loop Part 4 §6.7 puts ahead of resumed + * Publish handling. + */ + @Test + void watchdogDoesNotElapseWhileTheReconnectRepublishDrainHoldsPublishSuspended() + throws Exception { + + try (var fixture = new Fixture()) { + // Answers the first Publish the client sends once the Subscription is created; every + // subsequent Publish is parked. + fixture.faultSession(); + + var elapsedAfterReactivation = new CountDownLatch(1); + OpcUaSubscription subscription = fixture.createSubscription(elapsedAfterReactivation); + assertRevisedParameters(subscription); + + fixture.awaitReactivation(); + fixture.awaitRepublishDrainStarted(); + + assertFalse( + elapsedAfterReactivation.await(RECOVERY_HOLD_MILLIS, TimeUnit.MILLISECONDS), + "the watchdog elapsed while the Part 4 §6.7 Republish drain was still running: no" + + " PublishResponse can arrive while the client is holding Publish traffic shut for" + + " that drain, so a watchdog armed when the Session became Active counts down" + + " against a Subscription that is recovering normally and reports it dead after " + + WATCHDOG_DELAY_MILLIS + + "ms"); + } + } + + /** + * The control that keeps the assertion above from being vacuous: the same held recovery, + * released. Once the drain is over the client may send PublishRequests again, the Server's + * keep-alive promise applies again, and a Server that then stays silent must be reported — so the + * watchdog has to be armed at that point, and this is the assertion that it is. + */ + @Test + void watchdogElapsesOnceTheHeldRepublishDrainLetsPublishResume() throws Exception { + try (var fixture = new Fixture()) { + fixture.faultSession(); + + var elapsedAfterReactivation = new CountDownLatch(1); + OpcUaSubscription subscription = fixture.createSubscription(elapsedAfterReactivation); + assertRevisedParameters(subscription); + + fixture.awaitReactivation(); + fixture.awaitRepublishDrainStarted(); + + fixture.releaseRepublishDrain(); + + assertTrue( + elapsedAfterReactivation.await(WATCHDOG_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), + "control: the watchdog never elapsed although the Republish drain had finished, Publish" + + " traffic was allowed to resume, and no PublishResponse followed; a watchdog that" + + " is not armed when publishing resumes leaves the Subscription unsupervised for the" + + " rest of its life"); + } + } + + /** + * The watchdog delay is derived from the revised parameters, so the timings above are only + * meaningful if the Server returned what was requested. Asserted up front so a Server that + * revises them fails loudly instead of producing a mystery timeout. + */ + private static void assertRevisedParameters(OpcUaSubscription subscription) { + assertAll( + () -> + assertEquals( + PUBLISHING_INTERVAL, + subscription.getRevisedPublishingInterval().orElseThrow(), + "revised PublishingInterval"), + () -> + assertEquals( + EXPECTED_MAX_KEEP_ALIVE_COUNT, + subscription.getRevisedMaxKeepAliveCount().orElseThrow(), + "revised MaxKeepAliveCount")); + } + + // region fixture + + /** + * A running Server that parks every Publish request it is not scripted to answer and holds the + * first Republish request until the test releases it, plus a connected client. + */ + private static final class Fixture implements AutoCloseable { + + /** Counted down when the Republish that begins the reconnect drain reaches the Server. */ + private final CountDownLatch republishStarted = new CountDownLatch(1); + + /** Releases the held Republish; always counted down by {@link #close()}. */ + private final CountDownLatch republishGate = new CountDownLatch(1); + + private final CountDownLatch sessionInactive = new CountDownLatch(1); + private final CountDownLatch sessionReactivated = new CountDownLatch(1); + + /** + * {@code true} once the Session has become Active again. Only expiries that follow the return + * to Active are counted: an expiry armed before the fault proves nothing about recovery, and + * these tests must not be able to pass or fail on one. + */ + private final AtomicBoolean reactivated = new AtomicBoolean(false); + + private final OpcUaServer server; + private final OpcUaClient client; + private final ScriptableSubscriptionServiceSet scriptable; + + Fixture() throws Exception { + TestServer testServer = TestServer.create(); + server = testServer.getServer(); + + scriptable = new ScriptableSubscriptionServiceSet(server); + for (EndpointConfig endpoint : server.getConfig().getEndpoints()) { + server.addServiceSet(endpoint.getPath(), scriptable); + } + + server.startup().get(); + + client = + TestClient.create( + server, + cfg -> + cfg.setRequestTimeout(uint(60_000)) + // No Session keep-alive traffic: the only requests in flight during a test + // are + // the ones it scripts. + .setKeepAliveInterval(uint(60_000))); + client.connect(); + + client.addSessionActivityListener( + new SessionActivityListener() { + @Override + public void onSessionInactive(UaSession session) { + sessionInactive.countDown(); + } + + @Override + public void onSessionActive(UaSession session) { + if (sessionInactive.getCount() == 0) { + reactivated.set(true); + sessionReactivated.countDown(); + } + } + }); + + scriptable.setRepublishResponder(this::respondToRepublish); + } + + /** + * Answer the next Publish request with a Bad_SessionIdInvalid ServiceFault, which {@code + * SessionFsmFactory}'s SessionFaultListener classifies as a Session error and turns into a + * reconnect. The Server-side Session is untouched, so re-activation succeeds and the + * Subscription survives it. + */ + void faultSession() { + scriptable.enqueueServiceFault(StatusCodes.Bad_SessionIdInvalid); + } + + /** + * Create a Subscription whose watchdog delay is {@value #WATCHDOG_DELAY_MILLIS}ms and whose + * listener counts down {@code elapsedAfterReactivation} for every expiry reported after the + * Session has become Active again. + * + *

The listener is installed before {@code create()} so that no expiry can be missed by + * having been reported before it was there. + */ + OpcUaSubscription createSubscription(CountDownLatch elapsedAfterReactivation) + throws UaException { + + var subscription = new OpcUaSubscription(client); + subscription.setWatchdogMultiplier(WATCHDOG_MULTIPLIER); + subscription.setPublishingInterval(PUBLISHING_INTERVAL); + subscription.setTargetKeepAliveInterval(TARGET_KEEP_ALIVE_INTERVAL); + subscription.setSubscriptionListener( + new OpcUaSubscription.SubscriptionListener() { + @Override + public void onWatchdogTimerElapsed(OpcUaSubscription s) { + if (reactivated.get()) { + elapsedAfterReactivation.countDown(); + } + } + }); + subscription.create(); + + return subscription; + } + + /** + * Hold the Republish that begins the reconnect drain until {@link #releaseRepublishDrain()}, + * then answer it Bad_MessageNotAvailable — the termination condition of the Part 4 §6.7 loop, + * since the Server is holding nothing for retransmission. + */ + private RepublishResponse respondToRepublish(RepublishRequest request) throws UaException { + republishStarted.countDown(); + + try { + if (!republishGate.await(GATE_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { + throw new UaException(StatusCodes.Bad_Timeout); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new UaException(StatusCodes.Bad_UnexpectedError, e); + } + + throw new UaException(StatusCodes.Bad_MessageNotAvailable); + } + + void awaitReactivation() throws Exception { + assertTrue( + sessionInactive.await(RECONNECT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), + "the scripted Bad_SessionIdInvalid Publish fault did not take the Session out of Active"); + assertTrue( + sessionReactivated.await(RECONNECT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), + "the Session never became Active again"); + } + + void awaitRepublishDrainStarted() throws Exception { + assertTrue( + republishStarted.await(AWAIT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), + "precondition: the Republish drain Part 4 §6.7 requires never reached the Server, so" + + " nothing is holding Publish traffic suspended and there is no recovery to" + + " outlast"); + } + + void releaseRepublishDrain() { + republishGate.countDown(); + } + + @Override + public void close() throws Exception { + republishGate.countDown(); + scriptable.failParkedRequests(StatusCodes.Bad_NoSubscription); + try { + client.disconnectAsync().get(5, TimeUnit.SECONDS); + } finally { + server.shutdown().get(5, TimeUnit.SECONDS); + } + } + } + + // endregion +} diff --git a/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SubscriptionWatchdogRegistrationRaceTest.java b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SubscriptionWatchdogRegistrationRaceTest.java new file mode 100644 index 0000000000..587cdbb0d0 --- /dev/null +++ b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SubscriptionWatchdogRegistrationRaceTest.java @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2026 the Eclipse Milo Authors + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.milo.opcua.sdk.client.subscriptions; + +import static org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.Unsigned.uint; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyDouble; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import org.eclipse.milo.opcua.sdk.client.OpcUaClient; +import org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.UByte; +import org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.UInteger; +import org.eclipse.milo.opcua.stack.core.types.structured.CreateSubscriptionResponse; +import org.eclipse.milo.opcua.stack.core.types.structured.ResponseHeader; +import org.eclipse.milo.opcua.stack.transport.client.OpcClientTransport; +import org.eclipse.milo.opcua.stack.transport.client.OpcClientTransportConfig; +import org.junit.jupiter.api.Test; + +/** The watchdog hand-off between reconnect recovery and a concurrently created Subscription. */ +public class SubscriptionWatchdogRegistrationRaceTest { + + /** + * Recovery re-arms the Subscriptions already in the PublishingManager registry, while creation + * independently checks whether publishing is still suspended. If recovery resumes after that + * check but before registration, both paths miss the new Subscription and its watchdog remains + * unarmed forever. This fixture makes registration the exact point at which recovery finishes: + * registering before the final check guarantees one of the two paths arms the timer. + */ + @Test + void createArmsWatchdogWhenRecoveryResumesAtRegistration() throws Exception { + OpcUaClient client = mock(OpcUaClient.class); + PublishingManager publishingManager = mock(PublishingManager.class); + OpcClientTransport transport = mock(OpcClientTransport.class); + OpcClientTransportConfig transportConfig = mock(OpcClientTransportConfig.class); + ExecutorService executor = mock(ExecutorService.class); + ScheduledExecutorService scheduledExecutor = mock(ScheduledExecutorService.class); + ScheduledFuture scheduledFuture = mock(ScheduledFuture.class); + + when(client.getPublishingManager()).thenReturn(publishingManager); + when(client.getTransport()).thenReturn(transport); + when(transport.getConfig()).thenReturn(transportConfig); + when(transportConfig.getExecutor()).thenReturn(executor); + when(transportConfig.getScheduledExecutor()).thenReturn(scheduledExecutor); + doReturn(scheduledFuture) + .when(scheduledExecutor) + .schedule(any(Runnable.class), anyLong(), eq(TimeUnit.MILLISECONDS)); + + var response = + new CreateSubscriptionResponse( + mock(ResponseHeader.class), uint(1), 1_000.0, uint(50), uint(10)); + + when(client.createSubscriptionAsync( + anyDouble(), + any(UInteger.class), + any(UInteger.class), + any(UInteger.class), + anyBoolean(), + any(UByte.class))) + .thenReturn(CompletableFuture.completedFuture(response)); + + var publishingSuspended = new AtomicBoolean(true); + when(publishingManager.isPublishingSuspended()) + .thenAnswer(invocation -> publishingSuspended.get()); + doAnswer( + invocation -> { + // Models resumePublishing() completing its registry sweep immediately before this + // new entry becomes visible to that sweep. + publishingSuspended.set(false); + return null; + }) + .when(publishingManager) + .addSubscription(any(OpcUaSubscription.class)); + + var subscription = new OpcUaSubscription(client); + + subscription.createAsync().toCompletableFuture().get(5, TimeUnit.SECONDS); + + verify(publishingManager).addSubscription(same(subscription)); + verify(scheduledExecutor).schedule(any(Runnable.class), anyLong(), eq(TimeUnit.MILLISECONDS)); + } +} diff --git a/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SubscriptionWatchdogSessionFaultTest.java b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SubscriptionWatchdogSessionFaultTest.java new file mode 100644 index 0000000000..818c1849e3 --- /dev/null +++ b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SubscriptionWatchdogSessionFaultTest.java @@ -0,0 +1,242 @@ +/* + * Copyright (c) 2026 the Eclipse Milo Authors + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.milo.opcua.sdk.client.subscriptions; + +import static org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.Unsigned.uint; +import static org.junit.jupiter.api.Assertions.assertAll; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import org.eclipse.milo.opcua.sdk.client.OpcUaClient; +import org.eclipse.milo.opcua.sdk.client.SessionActivityListener; +import org.eclipse.milo.opcua.sdk.client.UaSession; +import org.eclipse.milo.opcua.sdk.server.EndpointConfig; +import org.eclipse.milo.opcua.sdk.server.OpcUaServer; +import org.eclipse.milo.opcua.sdk.test.ScriptableSubscriptionServiceSet; +import org.eclipse.milo.opcua.sdk.test.TestClient; +import org.eclipse.milo.opcua.sdk.test.TestServer; +import org.eclipse.milo.opcua.stack.core.StatusCodes; +import org.eclipse.milo.opcua.stack.core.UaException; +import org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.UInteger; +import org.junit.jupiter.api.Test; + +/** + * A Publish that fails with {@code Bad_SessionIdInvalid} or {@code Bad_SessionClosed} means the + * Session is gone, not the Subscription. {@code SessionFsmFactory.SessionFaultListener} + * classifies both as Session errors and drives the Session FSM out of {@code Active} and back + * through re-activation, after which the Subscription is still alive on the Server and Publish + * traffic resumes on it. + * + *

The watchdog is the only mechanism that tells an application the Server has stopped honouring + * the keep-alive interval it promised in its CreateSubscription response (Part 4 §5.13.2 — the + * Server must send a keep-alive Publish response when MaxKeepAliveCount publishing cycles pass with + * no notifications). A transient Session fault must therefore suspend the watchdog, not + * destroy it: if it is destroyed, the Subscription runs unsupervised for the rest of its life and a + * Server that later goes silent is never reported, even though Publish traffic itself recovered. + * + *

Every Publish is parked by {@link ScriptableSubscriptionServiceSet} except the first, which is + * answered with the Session fault, so no Publish response can re-arm the watchdog and the only + * possible re-arming is the one the Session's return to {@code Active} is supposed to perform. + */ +public class SubscriptionWatchdogSessionFaultTest { + + /** Made explicit rather than relying on the default, because the delay below depends on it. */ + private static final double WATCHDOG_MULTIPLIER = 1.5; + + private static final double PUBLISHING_INTERVAL = 111.0; + + /** ceil(111 / 111) derives a MaxKeepAliveCount of 1. */ + private static final double TARGET_KEEP_ALIVE_INTERVAL = 111.0; + + private static final UInteger EXPECTED_MAX_KEEP_ALIVE_COUNT = uint(1); + + /** 111 * (1 + 1) * 1.5 — the watchdog delay implied by the revised parameters. */ + private static final long WATCHDOG_DELAY_MILLIS = 333; + + /** + * The Session FSM waits one second in {@code ReactivatingWait} before its first re-activation + * attempt, and doubles the wait on each failure; this window allows for several attempts. + */ + private static final long RECONNECT_TIMEOUT_MILLIS = 20_000; + + /** Two orders of magnitude more than {@link #WATCHDOG_DELAY_MILLIS}. */ + private static final long WATCHDOG_TIMEOUT_MILLIS = 10_000; + + /** + * A Publish failing with {@code Bad_SessionIdInvalid} takes the Session down and brings it back; + * it says nothing about the Subscription. Once the Session is {@code Active} again the Server's + * keep-alive promise applies again, so the watchdog must be armed again and must elapse when the + * Server stays silent — exactly as it would have before the fault. + */ + @Test + void watchdogElapsesAfterSessionFaultAndReactivation() throws Exception { + try (var fixture = new Fixture()) { + var sessionInactive = new CountDownLatch(1); + var sessionReactivated = new CountDownLatch(1); + var reactivated = new AtomicBoolean(false); + + fixture.client.addSessionActivityListener( + new SessionActivityListener() { + @Override + public void onSessionInactive(UaSession session) { + sessionInactive.countDown(); + } + + @Override + public void onSessionActive(UaSession session) { + if (sessionInactive.getCount() == 0) { + reactivated.set(true); + sessionReactivated.countDown(); + } + } + }); + + // Answers the first Publish the client sends once the Subscription is created; every + // subsequent Publish is parked. + fixture.scriptable.enqueueServiceFault(StatusCodes.Bad_SessionIdInvalid); + + OpcUaSubscription subscription = fixture.createSubscription(); + assertRevisedParameters(subscription); + + // Only expiries that follow the return to Active are counted. An expiry armed before the + // fault proves nothing about recovery, and this test must not be able to pass on one. + var elapsedAfterReactivation = new CountDownLatch(1); + subscription.setSubscriptionListener( + new OpcUaSubscription.SubscriptionListener() { + @Override + public void onWatchdogTimerElapsed(OpcUaSubscription subscription) { + if (reactivated.get()) { + elapsedAfterReactivation.countDown(); + } + } + }); + + // Preconditions: the fault really did drive the Session FSM out of Active and back, which is + // what makes the watchdog assertion below meaningful. + assertTrue( + sessionInactive.await(RECONNECT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), + "the Bad_SessionIdInvalid Publish fault did not take the Session out of Active"); + assertTrue( + sessionReactivated.await(RECONNECT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), + "the Session was never re-activated"); + + assertTrue( + elapsedAfterReactivation.await(WATCHDOG_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), + "watchdog never elapsed after the Session was re-activated, although no Publish response" + + " has been received and the expiry was due " + + WATCHDOG_DELAY_MILLIS + + "ms later; the Session fault destroyed the watchdog instead of suspending it"); + } + } + + /** + * The control that proves {@link #watchdogElapsesAfterSessionFaultAndReactivation()} is not + * vacuous: in this fixture, with no Session fault at all, a Subscription whose Publish requests + * go unanswered does report the expiry, and reports it on the schedule the revised parameters + * imply. + */ + @Test + void watchdogElapsesWhenPublishResponsesStop() throws Exception { + try (var fixture = new Fixture()) { + var elapsed = new CountDownLatch(1); + + OpcUaSubscription subscription = fixture.newSubscription(); + subscription.setSubscriptionListener( + new OpcUaSubscription.SubscriptionListener() { + @Override + public void onWatchdogTimerElapsed(OpcUaSubscription subscription) { + elapsed.countDown(); + } + }); + subscription.create(); + + assertRevisedParameters(subscription); + + assertTrue( + elapsed.await(WATCHDOG_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), + "watchdog did not elapse within " + + WATCHDOG_TIMEOUT_MILLIS + + "ms although no Publish response was ever delivered"); + } + } + + /** + * The watchdog delay is derived from the revised parameters, so the timings above are only + * meaningful if the Server returned what was requested. Asserted up front so a Server that + * revises them fails loudly instead of producing a mystery timeout. + */ + private static void assertRevisedParameters(OpcUaSubscription subscription) { + assertAll( + () -> + assertEquals( + PUBLISHING_INTERVAL, + subscription.getRevisedPublishingInterval().orElseThrow(), + "revised PublishingInterval"), + () -> + assertEquals( + EXPECTED_MAX_KEEP_ALIVE_COUNT, + subscription.getRevisedMaxKeepAliveCount().orElseThrow(), + "revised MaxKeepAliveCount")); + } + + /** A running Server whose Publish requests are all parked, plus a connected client. */ + private static final class Fixture implements AutoCloseable { + + private final OpcUaServer server; + private final OpcUaClient client; + private final ScriptableSubscriptionServiceSet scriptable; + + Fixture() throws Exception { + TestServer testServer = TestServer.create(); + server = testServer.getServer(); + + scriptable = new ScriptableSubscriptionServiceSet(server); + for (EndpointConfig endpoint : server.getConfig().getEndpoints()) { + server.addServiceSet(endpoint.getPath(), scriptable); + } + + server.startup().get(); + + // Long request timeout so the parked Publish requests do not time out during the test. + client = TestClient.create(server, cfg -> cfg.setRequestTimeout(uint(60_000))); + client.connect(); + } + + OpcUaSubscription newSubscription() { + var subscription = new OpcUaSubscription(client); + subscription.setWatchdogMultiplier(WATCHDOG_MULTIPLIER); + subscription.setPublishingInterval(PUBLISHING_INTERVAL); + subscription.setTargetKeepAliveInterval(TARGET_KEEP_ALIVE_INTERVAL); + + return subscription; + } + + OpcUaSubscription createSubscription() throws UaException { + OpcUaSubscription subscription = newSubscription(); + subscription.create(); + + return subscription; + } + + @Override + public void close() throws Exception { + scriptable.failParkedRequests(StatusCodes.Bad_NoSubscription); + try { + client.disconnectAsync().get(5, TimeUnit.SECONDS); + } finally { + server.shutdown().get(5, TimeUnit.SECONDS); + } + } + } +} diff --git a/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/test/DelegatingMonitoredItemServiceSet.java b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/test/DelegatingMonitoredItemServiceSet.java new file mode 100644 index 0000000000..897a9fa368 --- /dev/null +++ b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/test/DelegatingMonitoredItemServiceSet.java @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2026 the Eclipse Milo Authors + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.milo.opcua.sdk.test; + +import java.util.Objects; +import org.eclipse.milo.opcua.sdk.server.OpcUaServer; +import org.eclipse.milo.opcua.sdk.server.servicesets.MonitoredItemServiceSet; +import org.eclipse.milo.opcua.sdk.server.servicesets.impl.DefaultMonitoredItemServiceSet; +import org.eclipse.milo.opcua.stack.core.UaException; +import org.eclipse.milo.opcua.stack.core.types.structured.CreateMonitoredItemsRequest; +import org.eclipse.milo.opcua.stack.core.types.structured.CreateMonitoredItemsResponse; +import org.eclipse.milo.opcua.stack.core.types.structured.DeleteMonitoredItemsRequest; +import org.eclipse.milo.opcua.stack.core.types.structured.DeleteMonitoredItemsResponse; +import org.eclipse.milo.opcua.stack.core.types.structured.ModifyMonitoredItemsRequest; +import org.eclipse.milo.opcua.stack.core.types.structured.ModifyMonitoredItemsResponse; +import org.eclipse.milo.opcua.stack.core.types.structured.SetMonitoringModeRequest; +import org.eclipse.milo.opcua.stack.core.types.structured.SetMonitoringModeResponse; +import org.eclipse.milo.opcua.stack.core.types.structured.SetTriggeringRequest; +import org.eclipse.milo.opcua.stack.core.types.structured.SetTriggeringResponse; +import org.eclipse.milo.opcua.stack.transport.server.ServiceRequestContext; + +/** + * A {@link MonitoredItemServiceSet} that forwards every operation to a delegate. + * + *

Mirrors {@link DelegatingSubscriptionServiceSet}. Subclass and override individual {@code on*} + * methods to observe the MonitoredItem requests a client sends, or to fail them deterministically, + * while leaving the remainder of the real server behavior intact. Register with {@code + * server.addServiceSet(path, serviceSet)}; a later registration replaces the default handlers for + * that path. + */ +public class DelegatingMonitoredItemServiceSet implements MonitoredItemServiceSet { + + private final MonitoredItemServiceSet delegate; + + public DelegatingMonitoredItemServiceSet(OpcUaServer server) { + this(new DefaultMonitoredItemServiceSet(server)); + } + + public DelegatingMonitoredItemServiceSet(MonitoredItemServiceSet delegate) { + this.delegate = Objects.requireNonNull(delegate, "delegate"); + } + + @Override + public CreateMonitoredItemsResponse onCreateMonitoredItems( + ServiceRequestContext context, CreateMonitoredItemsRequest request) throws UaException { + return delegate.onCreateMonitoredItems(context, request); + } + + @Override + public ModifyMonitoredItemsResponse onModifyMonitoredItems( + ServiceRequestContext context, ModifyMonitoredItemsRequest request) throws UaException { + return delegate.onModifyMonitoredItems(context, request); + } + + @Override + public DeleteMonitoredItemsResponse onDeleteMonitoredItems( + ServiceRequestContext context, DeleteMonitoredItemsRequest request) throws UaException { + return delegate.onDeleteMonitoredItems(context, request); + } + + @Override + public SetMonitoringModeResponse onSetMonitoringMode( + ServiceRequestContext context, SetMonitoringModeRequest request) throws UaException { + return delegate.onSetMonitoringMode(context, request); + } + + @Override + public SetTriggeringResponse onSetTriggering( + ServiceRequestContext context, SetTriggeringRequest request) throws UaException { + return delegate.onSetTriggering(context, request); + } +} diff --git a/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/test/DelegatingSubscriptionServiceSet.java b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/test/DelegatingSubscriptionServiceSet.java new file mode 100644 index 0000000000..a00a2f4f28 --- /dev/null +++ b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/test/DelegatingSubscriptionServiceSet.java @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2026 the Eclipse Milo Authors + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.milo.opcua.sdk.test; + +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import org.eclipse.milo.opcua.sdk.server.OpcUaServer; +import org.eclipse.milo.opcua.sdk.server.servicesets.SubscriptionServiceSet; +import org.eclipse.milo.opcua.sdk.server.servicesets.impl.DefaultSubscriptionServiceSet; +import org.eclipse.milo.opcua.stack.core.UaException; +import org.eclipse.milo.opcua.stack.core.types.structured.CreateSubscriptionRequest; +import org.eclipse.milo.opcua.stack.core.types.structured.CreateSubscriptionResponse; +import org.eclipse.milo.opcua.stack.core.types.structured.DeleteSubscriptionsRequest; +import org.eclipse.milo.opcua.stack.core.types.structured.DeleteSubscriptionsResponse; +import org.eclipse.milo.opcua.stack.core.types.structured.ModifySubscriptionRequest; +import org.eclipse.milo.opcua.stack.core.types.structured.ModifySubscriptionResponse; +import org.eclipse.milo.opcua.stack.core.types.structured.PublishRequest; +import org.eclipse.milo.opcua.stack.core.types.structured.PublishResponse; +import org.eclipse.milo.opcua.stack.core.types.structured.RepublishRequest; +import org.eclipse.milo.opcua.stack.core.types.structured.RepublishResponse; +import org.eclipse.milo.opcua.stack.core.types.structured.SetPublishingModeRequest; +import org.eclipse.milo.opcua.stack.core.types.structured.SetPublishingModeResponse; +import org.eclipse.milo.opcua.stack.core.types.structured.TransferSubscriptionsRequest; +import org.eclipse.milo.opcua.stack.core.types.structured.TransferSubscriptionsResponse; +import org.eclipse.milo.opcua.stack.transport.server.ServiceRequestContext; + +/** + * A {@link SubscriptionServiceSet} that forwards every operation to a delegate. + * + *

Mirrors {@link DelegatingSessionServiceSet}. Subclass and override individual {@code on*} + * methods to deterministically control the server responses the client's {@code PublishingManager} + * and {@code OpcUaSubscription} observe, while leaving the remainder of the real server behavior + * intact. Register with {@code server.addServiceSet(path, serviceSet)}; a later registration + * replaces the default handlers for that path. + * + * @see ScriptableSubscriptionServiceSet + */ +public class DelegatingSubscriptionServiceSet implements SubscriptionServiceSet { + + private final SubscriptionServiceSet delegate; + + public DelegatingSubscriptionServiceSet(OpcUaServer server) { + this(new DefaultSubscriptionServiceSet(server)); + } + + public DelegatingSubscriptionServiceSet(SubscriptionServiceSet delegate) { + this.delegate = Objects.requireNonNull(delegate, "delegate"); + } + + @Override + public CreateSubscriptionResponse onCreateSubscription( + ServiceRequestContext context, CreateSubscriptionRequest request) throws UaException { + return delegate.onCreateSubscription(context, request); + } + + @Override + public ModifySubscriptionResponse onModifySubscription( + ServiceRequestContext context, ModifySubscriptionRequest request) throws UaException { + return delegate.onModifySubscription(context, request); + } + + @Override + public DeleteSubscriptionsResponse onDeleteSubscriptions( + ServiceRequestContext context, DeleteSubscriptionsRequest request) throws UaException { + return delegate.onDeleteSubscriptions(context, request); + } + + @Override + public TransferSubscriptionsResponse onTransferSubscriptions( + ServiceRequestContext context, TransferSubscriptionsRequest request) throws UaException { + return delegate.onTransferSubscriptions(context, request); + } + + @Override + public SetPublishingModeResponse onSetPublishingMode( + ServiceRequestContext context, SetPublishingModeRequest request) throws UaException { + return delegate.onSetPublishingMode(context, request); + } + + @Override + public RepublishResponse onRepublish(ServiceRequestContext context, RepublishRequest request) + throws UaException { + return delegate.onRepublish(context, request); + } + + @Override + public CompletableFuture onPublish( + ServiceRequestContext context, PublishRequest request) { + return delegate.onPublish(context, request); + } +} diff --git a/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/test/ScriptableSubscriptionServiceSet.java b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/test/ScriptableSubscriptionServiceSet.java new file mode 100644 index 0000000000..58f6d3d4c1 --- /dev/null +++ b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/test/ScriptableSubscriptionServiceSet.java @@ -0,0 +1,390 @@ +/* + * Copyright (c) 2026 the Eclipse Milo Authors + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.milo.opcua.sdk.test; + +import static org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.Unsigned.uint; + +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.ReentrantLock; +import org.eclipse.milo.opcua.sdk.server.OpcUaServer; +import org.eclipse.milo.opcua.stack.core.UaException; +import org.eclipse.milo.opcua.stack.core.encoding.EncodingContext; +import org.eclipse.milo.opcua.stack.core.types.UaStructuredType; +import org.eclipse.milo.opcua.stack.core.types.builtin.DateTime; +import org.eclipse.milo.opcua.stack.core.types.builtin.ExtensionObject; +import org.eclipse.milo.opcua.stack.core.types.builtin.StatusCode; +import org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.UInteger; +import org.eclipse.milo.opcua.stack.core.types.structured.DataChangeNotification; +import org.eclipse.milo.opcua.stack.core.types.structured.MonitoredItemNotification; +import org.eclipse.milo.opcua.stack.core.types.structured.NotificationMessage; +import org.eclipse.milo.opcua.stack.core.types.structured.PublishRequest; +import org.eclipse.milo.opcua.stack.core.types.structured.PublishResponse; +import org.eclipse.milo.opcua.stack.core.types.structured.RepublishRequest; +import org.eclipse.milo.opcua.stack.core.types.structured.RepublishResponse; +import org.eclipse.milo.opcua.stack.core.types.structured.ResponseHeader; +import org.eclipse.milo.opcua.stack.core.types.structured.SubscriptionAcknowledgement; +import org.eclipse.milo.opcua.stack.transport.server.ServiceRequestContext; + +/** + * A {@link DelegatingSubscriptionServiceSet} that gives a test deterministic control over the + * {@code Publish} and {@code Republish} responses the client observes. + * + *

The client's {@code PublishingManager} continuously pipelines Publish requests. Each incoming + * request is matched, in FIFO order, against a script of {@link PublishResponder}s enqueued by the + * test. When the script is empty, the request is parked (an uncompleted future is returned) + * and fulfilled as soon as the test enqueues another responder. Publish is dispatched + * asynchronously by the server, so parked requests do not block any server thread. + * + *

This lets a test drive precise sequence-number scenarios — initial keep-alive, gaps, rollover, + * duplicate keep-alives — and inspect exactly which acknowledgements the client sends back. + * + *

{@code Create}/{@code Modify}/{@code Delete}/{@code Transfer}/{@code SetPublishingMode} + * delegate to the real server by default, so a real server-side subscription exists (useful for + * transfer scenarios); override those methods in a subclass or via the delegate to script them too. + * + *

All state is guarded for concurrent access from server I/O threads and the test thread. + */ +public class ScriptableSubscriptionServiceSet extends DelegatingSubscriptionServiceSet { + + /** Produces the response (normal or exceptional) for a single Publish request. */ + @FunctionalInterface + public interface PublishResponder { + CompletableFuture respondTo(PublishRequest request); + } + + /** Produces the response for a single Republish request. */ + @FunctionalInterface + public interface RepublishResponder { + RepublishResponse respondTo(RepublishRequest request) throws UaException; + } + + private record ParkedRequest(PublishRequest request, CompletableFuture future) {} + + private final ReentrantLock lock = new ReentrantLock(); + private final ArrayDeque responders = new ArrayDeque<>(); + private final ArrayDeque parked = new ArrayDeque<>(); + + private final List receivedAcknowledgements = + Collections.synchronizedList(new ArrayList<>()); + private final AtomicInteger publishRequestCount = new AtomicInteger(0); + + private volatile RepublishResponder republishResponder; + + private final EncodingContext encodingContext; + + public ScriptableSubscriptionServiceSet(OpcUaServer server) { + super(server); + this.encodingContext = server.getStaticEncodingContext(); + } + + // region Publish scripting + + /** + * Enqueue a responder that will service the next (or next parked) Publish request. + * + * @param responder the {@link PublishResponder} to enqueue. + */ + public void enqueue(PublishResponder responder) { + ParkedRequest parkedRequest; + lock.lock(); + try { + parkedRequest = parked.poll(); + if (parkedRequest == null) { + responders.add(responder); + return; + } + } finally { + lock.unlock(); + } + + // Fulfill the parked request outside the lock. + fulfill(responder, parkedRequest); + } + + /** + * Enqueue a keep-alive Publish response (no notification data) with the given sequence number. + */ + public void enqueueKeepAlive( + UInteger subscriptionId, long sequenceNumber, UInteger... available) { + enqueueNotification(subscriptionId, sequenceNumber, null, available); + } + + /** Enqueue a data-change Publish response carrying the given monitored-item notifications. */ + public void enqueueDataChange( + UInteger subscriptionId, + long sequenceNumber, + List items, + UInteger... available) { + + ExtensionObject[] notificationData = { + encode(new DataChangeNotification(items.toArray(MonitoredItemNotification[]::new), null)) + }; + + enqueueNotification(subscriptionId, sequenceNumber, notificationData, available); + } + + /** + * Enqueue a Publish response with arbitrary notification data. A {@code null} or empty {@code + * notificationData} produces a keep-alive. + */ + public void enqueueNotification( + UInteger subscriptionId, + long sequenceNumber, + ExtensionObject[] notificationData, + UInteger... available) { + + enqueue( + request -> + CompletableFuture.completedFuture( + buildPublishResponse( + request, subscriptionId, sequenceNumber, notificationData, available, false))); + } + + /** Enqueue a service-level failure (ServiceFault) for the next Publish request. */ + public void enqueueServiceFault(long statusCode) { + enqueue(request -> CompletableFuture.failedFuture(new UaException(statusCode))); + } + + /** + * Build a well-formed {@link PublishResponse} for {@code request}, echoing its request handle and + * acknowledging every acknowledgement in the request with {@code Good}. + */ + public PublishResponse buildPublishResponse( + PublishRequest request, + UInteger subscriptionId, + long sequenceNumber, + ExtensionObject[] notificationData, + UInteger[] available, + boolean moreNotifications) { + + SubscriptionAcknowledgement[] acks = request.getSubscriptionAcknowledgements(); + int ackCount = acks != null ? acks.length : 0; + var results = new StatusCode[ackCount]; + Arrays.fill(results, StatusCode.GOOD); + + return buildPublishResponse( + request, + subscriptionId, + sequenceNumber, + notificationData, + available, + moreNotifications, + results); + } + + /** + * Build a {@link PublishResponse} for {@code request} whose acknowledgement results are {@code + * results} rather than all {@code Good}. + * + *

Part 4 §5.14.5.2: "The size and order of the list matches the size and order of the + * subscriptionAcknowledgements request parameter." Callers scripting a rejected acknowledgement + * are responsible for honouring that, since it is the pairing the client has to rely on to tell + * which acknowledgement failed. + * + * @param results one {@link StatusCode} per acknowledgement in {@code request}, in the same + * order. + */ + public PublishResponse buildPublishResponse( + PublishRequest request, + UInteger subscriptionId, + long sequenceNumber, + ExtensionObject[] notificationData, + UInteger[] available, + boolean moreNotifications, + StatusCode[] results) { + + var responseHeader = + new ResponseHeader( + DateTime.now(), + request.getRequestHeader().getRequestHandle(), + StatusCode.GOOD, + null, + null, + null); + + ExtensionObject[] data = + (notificationData == null || notificationData.length == 0) ? null : notificationData; + + var notificationMessage = new NotificationMessage(uint(sequenceNumber), DateTime.now(), data); + + UInteger[] availableSequenceNumbers = + (available == null || available.length == 0) ? null : available; + + return new PublishResponse( + responseHeader, + subscriptionId, + availableSequenceNumbers, + moreNotifications, + notificationMessage, + results, + null); + } + + // endregion + + // region Republish scripting + + /** + * Install a responder for Republish requests. When {@code null} (the default), Republish + * delegates to the real server. + */ + public void setRepublishResponder(RepublishResponder responder) { + this.republishResponder = responder; + } + + /** + * Build a {@link RepublishResponse} carrying {@code notificationData} at {@code sequenceNumber}. + */ + public RepublishResponse buildRepublishResponse( + RepublishRequest request, long sequenceNumber, ExtensionObject[] notificationData) { + + var responseHeader = + new ResponseHeader( + DateTime.now(), + request.getRequestHeader().getRequestHandle(), + StatusCode.GOOD, + null, + null, + null); + + var notificationMessage = + new NotificationMessage(uint(sequenceNumber), DateTime.now(), notificationData); + + return new RepublishResponse(responseHeader, notificationMessage); + } + + // endregion + + // region Inspection + + /** + * @return every {@link SubscriptionAcknowledgement} the client has sent, in arrival order. + */ + public List getReceivedAcknowledgements() { + return List.copyOf(receivedAcknowledgements); + } + + /** + * @return the number of Publish requests received so far. + */ + public int getPublishRequestCount() { + return publishRequestCount.get(); + } + + /** + * The number of Publish requests currently parked, i.e. received but not yet matched to a + * responder. A test that needs the client's Publish pipeline to be full before it starts + * scripting can wait on this rather than on elapsed time. + * + * @return the number of Publish requests currently parked. + */ + public int getParkedRequestCount() { + lock.lock(); + try { + return parked.size(); + } finally { + lock.unlock(); + } + } + + /** + * Fail every currently parked Publish request. Call from test teardown to avoid leaking futures + * held by the server dispatch. + */ + public void failParkedRequests(long statusCode) { + List toFail; + lock.lock(); + try { + toFail = new ArrayList<>(parked); + parked.clear(); + } finally { + lock.unlock(); + } + toFail.forEach(p -> p.future().completeExceptionally(new UaException(statusCode))); + } + + /** + * Encode a structured value into an {@link ExtensionObject} using the server encoding context. + */ + public ExtensionObject encode(UaStructuredType value) { + return ExtensionObject.encode(encodingContext, value); + } + + // endregion + + @Override + public CompletableFuture onPublish( + ServiceRequestContext context, PublishRequest request) { + + SubscriptionAcknowledgement[] acks = request.getSubscriptionAcknowledgements(); + if (acks != null) { + Collections.addAll(receivedAcknowledgements, acks); + } + publishRequestCount.incrementAndGet(); + + PublishResponder responder; + lock.lock(); + try { + responder = responders.poll(); + if (responder == null) { + var future = new CompletableFuture(); + parked.add(new ParkedRequest(request, future)); + return future; + } + } finally { + lock.unlock(); + } + + // Same guard as fulfill(): a responder that throws must produce a failed future rather than + // let the exception escape into the server's async dispatch, where the response future would + // never be completed and the client's PublishRequest would hang until its timeoutHint. + try { + return responder.respondTo(request); + } catch (RuntimeException e) { + return CompletableFuture.failedFuture(e); + } + } + + @Override + public RepublishResponse onRepublish(ServiceRequestContext context, RepublishRequest request) + throws UaException { + + RepublishResponder responder = this.republishResponder; + if (responder != null) { + return responder.respondTo(request); + } + return super.onRepublish(context, request); + } + + private static void fulfill(PublishResponder responder, ParkedRequest parkedRequest) { + CompletableFuture source; + try { + source = responder.respondTo(parkedRequest.request()); + } catch (RuntimeException e) { + parkedRequest.future().completeExceptionally(e); + return; + } + source.whenComplete( + (response, ex) -> { + if (ex != null) { + parkedRequest.future().completeExceptionally(ex); + } else { + parkedRequest.future().complete(response); + } + }); + } +} diff --git a/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/test/TestClient.java b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/test/TestClient.java index 46ea4f97cc..2aa354d977 100644 --- a/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/test/TestClient.java +++ b/opc-ua-sdk/integration-tests/src/test/java/org/eclipse/milo/opcua/sdk/test/TestClient.java @@ -12,14 +12,24 @@ import static org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.Unsigned.uint; +import java.util.List; import java.util.Objects; +import java.util.concurrent.ExecutionException; import java.util.function.Consumer; +import java.util.function.Function; +import org.eclipse.milo.opcua.sdk.client.DiscoveryClient; import org.eclipse.milo.opcua.sdk.client.OpcUaClient; +import org.eclipse.milo.opcua.sdk.client.OpcUaClientConfig; import org.eclipse.milo.opcua.sdk.client.OpcUaClientConfigBuilder; import org.eclipse.milo.opcua.sdk.server.EndpointConfig; import org.eclipse.milo.opcua.sdk.server.OpcUaServer; +import org.eclipse.milo.opcua.stack.core.StatusCodes; import org.eclipse.milo.opcua.stack.core.UaException; import org.eclipse.milo.opcua.stack.core.types.builtin.LocalizedText; +import org.eclipse.milo.opcua.stack.core.types.structured.EndpointDescription; +import org.eclipse.milo.opcua.stack.transport.client.OpcClientTransport; +import org.eclipse.milo.opcua.stack.transport.client.tcp.OpcTcpClientTransportConfig; +import org.eclipse.milo.opcua.stack.transport.client.tcp.OpcTcpClientTransportConfigBuilder; public final class TestClient { @@ -28,6 +38,25 @@ private TestClient() {} public static OpcUaClient create( OpcUaServer server, Consumer configCustomizer) throws UaException { + return create(server, transportConfigBuilder -> {}, configCustomizer); + } + + /** + * Create a test client, allowing customization of both the transport config (e.g. to inject a + * controllable {@code ExecutorService}) and the client config. + * + * @param server the {@link OpcUaServer} to connect to. + * @param transportCustomizer customizes the {@link OpcTcpClientTransportConfigBuilder}. + * @param configCustomizer customizes the {@link OpcUaClientConfigBuilder}. + * @return a configured {@link OpcUaClient}. + * @throws UaException if the client could not be created. + */ + public static OpcUaClient create( + OpcUaServer server, + Consumer transportCustomizer, + Consumer configCustomizer) + throws UaException { + EndpointConfig endpoint = server.getConfig().getEndpoints().iterator().next(); return OpcUaClient.create( @@ -39,14 +68,78 @@ public static OpcUaClient create( Objects.equals( e.getSecurityPolicyUri(), endpoint.getSecurityPolicy().getUri())) .findFirst(), - transportConfigBuilder -> {}, + transportCustomizer, clientConfigBuilder -> { - clientConfigBuilder - .setApplicationName(LocalizedText.english("eclipse milo test client")) - .setApplicationUri("urn:eclipse:milo:test:client") - .setRequestTimeout(uint(5_000)); + applyDefaults(clientConfigBuilder); configCustomizer.accept(clientConfigBuilder); }); } + + /** + * Create a test client whose {@link OpcClientTransport} is built by {@code transportFactory}, so + * a test can substitute or intercept the layer that hands the SDK the Server's answers. + * + *

{@link OpcUaClient#create(String, java.util.function.Function, Consumer, Consumer)} + * constructs the transport itself and offers no seam for one, so this assembles the same + * configuration and uses the {@link OpcUaClient#OpcUaClient(OpcUaClientConfig, + * OpcClientTransport)} constructor instead. It is the only way to script a response the service + * sets cannot express, e.g. an answer to a PublishRequest that is not a PublishResponse. + * + * @param server the {@link OpcUaServer} to connect to. + * @param transportFactory builds the transport from a default {@link + * OpcTcpClientTransportConfig}. + * @param configCustomizer customizes the {@link OpcUaClientConfigBuilder}. + * @return a configured {@link OpcUaClient}. + * @throws UaException if the endpoints could not be retrieved or the client could not be created. + */ + public static OpcUaClient createWithTransport( + OpcUaServer server, + Function transportFactory, + Consumer configCustomizer) + throws UaException { + + EndpointConfig endpointConfig = server.getConfig().getEndpoints().iterator().next(); + + List endpoints; + try { + endpoints = DiscoveryClient.getEndpoints(endpointConfig.getEndpointUrl()).get(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new UaException(StatusCodes.Bad_UnexpectedError, e); + } catch (ExecutionException e) { + throw new UaException(e.getCause()); + } + + EndpointDescription endpoint = + endpoints.stream() + .filter( + e -> + Objects.equals( + e.getSecurityPolicyUri(), endpointConfig.getSecurityPolicy().getUri())) + .findFirst() + .orElseThrow( + () -> new UaException(StatusCodes.Bad_ConfigurationError, "no endpoint selected")); + + OpcUaClientConfigBuilder clientConfigBuilder = + OpcUaClientConfig.builder() + .setEndpoint(endpoint) + .setDiscoveryEndpoints(endpoints) + .setSessionEndpointValidationEnabled(false); + + applyDefaults(clientConfigBuilder); + + configCustomizer.accept(clientConfigBuilder); + + OpcTcpClientTransportConfig transportConfig = OpcTcpClientTransportConfig.newBuilder().build(); + + return new OpcUaClient(clientConfigBuilder.build(), transportFactory.apply(transportConfig)); + } + + private static void applyDefaults(OpcUaClientConfigBuilder clientConfigBuilder) { + clientConfigBuilder + .setApplicationName(LocalizedText.english("eclipse milo test client")) + .setApplicationUri("urn:eclipse:milo:test:client") + .setRequestTimeout(uint(5_000)); + } } diff --git a/opc-ua-sdk/sdk-client/src/main/java/org/eclipse/milo/opcua/sdk/client/session/SessionFsmFactory.java b/opc-ua-sdk/sdk-client/src/main/java/org/eclipse/milo/opcua/sdk/client/session/SessionFsmFactory.java index ad30dcd8ab..467e739f1f 100644 --- a/opc-ua-sdk/sdk-client/src/main/java/org/eclipse/milo/opcua/sdk/client/session/SessionFsmFactory.java +++ b/opc-ua-sdk/sdk-client/src/main/java/org/eclipse/milo/opcua/sdk/client/session/SessionFsmFactory.java @@ -39,6 +39,7 @@ import java.security.PrivateKey; import java.security.cert.CertificateEncodingException; import java.security.cert.X509Certificate; +import java.util.ArrayList; import java.util.Arrays; import java.util.LinkedList; import java.util.List; @@ -1448,10 +1449,24 @@ private static CompletableFuture transferSubscriptions( CompletableFuture transferFuture = new CompletableFuture<>(); - UInteger[] subscriptionIdsArray = - subscriptions.stream() - .flatMap(s -> s.getSubscriptionId().stream()) - .toArray(UInteger[]::new); + // A Subscription can be reset() concurrently, clearing its id, so the ids sent and the + // Subscriptions they came from are captured together: the TransferResults are indexed + // against the ids the request carried, and pairing them back through a list whose shape + // can since have changed would apply a result to the wrong Subscription. + var transferable = new ArrayList(subscriptions.size()); + var subscriptionIds = new ArrayList(subscriptions.size()); + + for (OpcUaSubscription subscription : subscriptions) { + subscription + .getSubscriptionId() + .ifPresent( + id -> { + transferable.add(subscription); + subscriptionIds.add(id); + }); + } + + UInteger[] subscriptionIdsArray = subscriptionIds.toArray(new UInteger[0]); TransferSubscriptionsRequest request = new TransferSubscriptionsRequest( @@ -1481,15 +1496,13 @@ private static CompletableFuture transferSubscriptions( if (LOGGER.isDebugEnabled()) { try { - Stream subscriptionIds = - subscriptions.stream().flatMap(s -> s.getSubscriptionId().stream()); Stream statusCodes = Stream.of(results).map(TransferResult::getStatusCode); //noinspection UnstableApiUsage String[] ss = Streams.zip( - subscriptionIds, + subscriptionIds.stream(), statusCodes, (i, s) -> { assert s != null; @@ -1509,23 +1522,38 @@ private static CompletableFuture transferSubscriptions( } } - client - .getTransport() - .getConfig() - .getExecutor() - .execute( - () -> { - for (int i = 0; i < results.length; i++) { - TransferResult result = results[i]; + // Part 4 §5.14.7.1: a successful TransferResult carries "the sequence numbers + // of the NotificationMessages that are available for retransmission", which is + // what tells the client which NotificationMessages the Republish loop Part 4 + // §6.7 requires before Publish resumes can still collect. Recorded here, inline, + // rather than dispatched: the loop runs when this Session becomes Active, and + // the list has to be in place before it does. Indexed against the + // SubscriptionIds the request was built from, which is what the results are a + // "list of results for the subscriptions to transfer" of. + for (int i = 0; i < results.length && i < subscriptionIdsArray.length; i++) { + TransferResult result = results[i]; + + if (result.getStatusCode().isGood()) { + client + .getPublishingManager() + .notifySubscriptionTransferred( + session, subscriptionIdsArray[i], result.getAvailableSequenceNumbers()); + } + } - if (!result.getStatusCode().isGood()) { - OpcUaSubscription subscription = subscriptions.get(i); + for (int i = 0; i < results.length && i < transferable.size(); i++) { + TransferResult result = results[i]; - subscription.notifyTransferFailed(result.getStatusCode()); - } - } - }); + if (!result.getStatusCode().isGood()) { + handleTransferFailure( + ctx, session, transferable.get(i), result.getStatusCode()); + } + } + // Failed Subscriptions must be reset and unregistered before this completion can + // move the FSM through Initializing and into Active. Otherwise reconnect recovery + // can still see them and issue Republish requests for SubscriptionIds that were + // not transferred to this Session. transferFuture.complete(Unit.VALUE); } else { StatusCode statusCode = @@ -1537,16 +1565,9 @@ private static CompletableFuture transferSubscriptions( LOGGER.debug("TransferSubscriptions not supported: {}", statusCode); } - client - .getTransport() - .getConfig() - .getExecutor() - .execute( - () -> { - for (OpcUaSubscription subscription : subscriptions) { - subscription.notifyTransferFailed(statusCode); - } - }); + for (OpcUaSubscription subscription : subscriptions) { + handleTransferFailure(ctx, session, subscription, statusCode); + } // Bad_ServiceUnsupported is the correct response when transfers aren't // supported but server implementations interpret the spec differently. @@ -1575,6 +1596,34 @@ private static CompletableFuture transferSubscriptions( return transferFuture; } + /** + * Reset and unregister a Subscription that was not transferred to {@code session}. + * + *

{@link OpcUaSubscription#handleTransferFailure(StatusCode)} performs non-overridable local + * teardown synchronously and dispatches the overridable notification separately. Contain internal + * failures here so one Subscription cannot leave the Session FSM in {@link State#Transferring} or + * prevent the remaining failed Subscriptions from being reset. + */ + private static void handleTransferFailure( + FsmContext ctx, + OpcUaSession session, + OpcUaSubscription subscription, + StatusCode statusCode) { + + try { + subscription.handleTransferFailure(statusCode); + } catch (Exception e) { + try (MDCCloseable ignoredInstanceId = putInstanceId(ctx); + MDCCloseable ignoredSessionId = putSessionId(session)) { + + LOGGER.warn( + "Subscription transfer-failure cleanup failed: id={}", + subscription.getSubscriptionId().orElse(null), + e); + } + } + } + private static CompletableFuture initialize( FsmContext ctx, OpcUaClient client, OpcUaSession session) { diff --git a/opc-ua-sdk/sdk-client/src/main/java/org/eclipse/milo/opcua/sdk/client/subscriptions/OpcUaSubscription.java b/opc-ua-sdk/sdk-client/src/main/java/org/eclipse/milo/opcua/sdk/client/subscriptions/OpcUaSubscription.java index 4af7269574..69df988a3d 100644 --- a/opc-ua-sdk/sdk-client/src/main/java/org/eclipse/milo/opcua/sdk/client/subscriptions/OpcUaSubscription.java +++ b/opc-ua-sdk/sdk-client/src/main/java/org/eclipse/milo/opcua/sdk/client/subscriptions/OpcUaSubscription.java @@ -13,12 +13,13 @@ import static java.util.Objects.requireNonNull; import static org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.Unsigned.ubyte; import static org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.Unsigned.uint; -import static org.eclipse.milo.opcua.stack.core.util.FutureUtils.supplyAsyncCompose; import com.google.common.primitives.Ints; import java.math.BigInteger; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collections; +import java.util.Deque; import java.util.List; import java.util.Map; import java.util.Optional; @@ -27,10 +28,11 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; import java.util.function.Predicate; +import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; import org.eclipse.milo.opcua.sdk.client.OpcUaClient; @@ -66,6 +68,43 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * A Subscription on a Server, and the client-side object that represents it. + * + *

The two are not the same thing and do not have the same lifetime. A Subscription exists on the + * Server from {@link #create()} until it is deleted, times out, or {@link #reset()} discards the + * client's knowledge of it; this object exists for as long as the application holds it, and may + * represent several Subscriptions in sequence. + * + *

Threading: the lifecycle transitions that call a service — {@link #create()}, {@link + * #modify()}, {@link #delete()} and {@link #setPublishingMode(boolean)} — are each a check of the + * current state, a service call, and an update of that state. Each is implemented once, as its + * {@code ...Async()} form composed from the client's asynchronous services; the blocking form is + * that stage awaited, so the two cannot disagree about the state they leave behind. No transition + * ever occupies a thread while it waits for the Server, including one waiting its turn, which is + * what makes the asynchronous forms safe to call from any thread — above all from the transport + * executor, whose threads are the ones that complete the responses they are waiting for. A blocking + * form, like any blocking call, must be made from a thread that is not one of those. + * + *

Transitions are serialized against each other: one runs to completion before the next begins, + * and a second concurrent {@link #create()} is answered {@code Bad_InvalidState} rather than + * creating a second Subscription on the Server that no client object could then name or delete. + * + *

{@link #reset()} is not one of them. It only discards what the client knows, sends + * nothing, and never waits for a transition that is waiting for the Server — because it is called + * from places that must not be stopped for a network round trip: the Bad_Timeout + * StatusChangeNotification path runs on this Subscription's delivery queue, and {@link + * #notifyTransferFailed(StatusCode)} runs on the Session's state machine. A {@link #reset()} that + * overtakes a transition already in flight wins: the transition's result is discarded when it + * arrives rather than applied to a Subscription this object no longer represents. + * + *

The accessors read individual volatile fields and are not serialized against any of this. The + * parameter setters take the lifecycle lock for their check-then-act on the pending modifications — + * a setter racing a {@link #reset()} must not re-mark a discarded Subscription unsynchronized — but + * the lock is never held across a service call, so they still never block on one. MonitoredItem + * management is not serialized either: it is concurrent with the lifecycle only in the sense that + * it reports {@code Bad_InvalidState} when the Subscription is gone. + */ public class OpcUaSubscription { private static final int DEFAULT_MAX_MONITORED_ITEMS_PER_CALL = 10000; @@ -76,12 +115,95 @@ public class OpcUaSubscription { private final Logger logger = LoggerFactory.getLogger(getClass()); + /** + * Guards the lifecycle state — {@link #syncState}, {@link #serverState}, {@link #modifications}, + * {@link #transitionInFlight}, {@link #transitionWaiters} and {@link #incarnation} — and + * serializes the lifecycle transitions, each of which is a check-then-act around a service call: + * without that, two of them interleave and the Server ends up in a state no single client-side + * field describes — most visibly a second Subscription created by a concurrent {@link #create()} + * whose SubscriptionId is immediately overwritten, leaving it running on the Server with nothing + * left that can name it. + * + *

Never held across a service call. A transition claims {@link #transitionInFlight}, + * releases this lock, makes the call, then takes the lock again to apply the result. Holding it + * for the round trip would make every other user of the lock — above all {@link #reset()}, which + * is called from the delivery queue and from the Session's state machine — wait out someone + * else's request timeout, and on a bounded transport executor it deadlocks: the thread that would + * complete the response is one of the threads blocked on the lock. + * + *

Nothing waits on this monitor either, for the same reason: a transition that has to wait for + * the one ahead of it is queued in {@link #transitionWaiters} instead. + * + *

The accessors do not take it. The parameter setters take it only for their short + * check-then-act on {@link #modifications} and {@link #syncState}; nothing they do under it can + * block. + */ + private final Object lifecycleLock = new Object(); + + /** + * {@code true} while a lifecycle transition holds the right to change the lifecycle state, i.e. + * from the moment it is started until the {@link CompletionStage} it returned has completed. + * Guarded by {@link #lifecycleLock}. + * + *

Deliberately not a {@link SyncState}: that enum is public API and describes what the client + * knows about the Subscription, not what it is currently asking the Server for. {@link + * #getSyncState()} therefore keeps reporting the state the Subscription had when the in-flight + * call was made, which is exactly what is still true of it until the Server answers. + */ + private boolean transitionInFlight = false; + + /** + * Lifecycle transitions that arrived while another one held {@link #transitionInFlight}, in the + * order they arrived. Each is started by completing its {@link CompletableFuture}, which {@link + * #endTransition()} does when the transition ahead of it finishes. + * + *

A queue rather than a wait on {@link #lifecycleLock}: waiting by blocking would tie up a + * thread per queued transition, and the one thread an asynchronous call might be made from — a + * transport executor thread — is the thread the in-flight transition needs to be answered on. + * + *

Guarded by {@link #lifecycleLock}. + */ + private final Deque> transitionWaiters = new ArrayDeque<>(); + + /** + * Identifies the Subscription this object currently represents. Incremented by every {@link + * #reset()}, i.e. whenever this object stops representing the Subscription it did. + * + *

A transition captures it while claiming {@link #transitionInFlight} and compares it again + * when the response arrives: a change means a {@link #reset()} discarded the Subscription the + * call was made for, so the result must not be applied to whatever this object holds now. For + * {@link #create()} it also means the Subscription the Server just created belongs to nobody, and + * has to be deleted rather than left running. + * + *

Guarded by {@link #lifecycleLock}. + */ + private long incarnation = 0L; + private volatile SyncState syncState = SyncState.INITIAL; private volatile @Nullable ServerState serverState; private volatile Modifications modifications; private volatile WatchdogTimer watchdogTimer; + /** + * Marks the executor callback dispatched by {@link #handleTransferFailure(StatusCode)} after its + * non-overridable cleanup has already run. The public callback remains overridable for + * compatibility, but the base implementation must not reset a second time and invalidate a new + * incarnation created before the asynchronous notification runs. + */ + private final ThreadLocal transferFailureAlreadyHandled = new ThreadLocal<>(); + + /** + * Guards changes that move a MonitoredItem between {@link #monitoredItems} and {@link + * #itemsToDelete}, including applying a DeleteMonitoredItems result that detaches the item. + * + *

The collections remain concurrent because notification delivery and the synchronization + * queries read them without this lock. The lock makes the multi-step bookkeeping atomic: in + * particular, adding an item back while its deletion is in flight must be ordered against the + * response that clears its ClientHandle. + */ + private final Object monitoredItemsLock = new Object(); + /** MonitoredItems added to this Subscription, by ClientHandle. */ private final Map monitoredItems = new ConcurrentHashMap<>(); @@ -122,11 +244,21 @@ public OpcUaSubscription(OpcUaClient client) { deliveryQueue = new TaskQueue(client.getTransport().getConfig().getExecutor()); } + /** + * Create a Subscription with the given PublishingInterval. + * + *

The MaxKeepAliveCount and LifetimeCount are derived from {@code publishingInterval}, as they + * would be by {@link #setPublishingInterval(Double)}. + * + * @param client the {@link OpcUaClient} this Subscription belongs to. + * @param publishingInterval the PublishingInterval to request. + */ public OpcUaSubscription(OpcUaClient client, double publishingInterval) { - this.client = client; - this.publishingInterval = publishingInterval; + this(client); - deliveryQueue = new TaskQueue(client.getTransport().getConfig().getExecutor()); + // Delegate to the setter so the MaxKeepAliveCount and LifetimeCount are derived from + // this PublishingInterval instead of being left at their default-derived values. + setPublishingInterval(publishingInterval); } /** @@ -143,10 +275,52 @@ public OpcUaClient getClient() { /** * Create this Subscription on the Server. * + *

Serialized against the other lifecycle transitions; see the class documentation. A call made + * while this Subscription already exists on the Server, including one made concurrently with the + * call that created it, fails with {@code Bad_InvalidState}. + * + *

A {@link #reset()} made while this call is waiting for the Server also fails it with {@code + * Bad_InvalidState}: the reset has discarded the Subscription being created, so the Subscription + * the Server did create is deleted again rather than installed. + * + *

Blocks until the Server has answered; this is {@link #createAsync()} awaited, so it must not + * be called from a transport executor thread. See the class documentation. + * * @throws UaException if a service- or operation-level error occurs. */ public void create() throws UaException { - if (syncState == SyncState.INITIAL) { + await(createAsync()); + } + + /** + * Create this Subscription on the Server. + * + *

Fails with {@code Bad_InvalidState} under exactly the conditions {@link #create()} throws + * it: the Subscription already exists, or a {@link #reset()} superseded the call. + * + * @return a {@link CompletionStage} that completes successfully if the Subscription was created, + * or completes exceptionally if there was a service- or operation-level error. + */ + public CompletionStage createAsync() { + return runTransition(this::createTransition); + } + + /** + * Call the CreateSubscription service and install the Subscription the Server creates. + * + *

Runs with the transition slot claimed; see {@link #runTransition(Supplier)}. + * + * @return a {@link CompletionStage} that completes when the Subscription has been created, or + * completes exceptionally if it has not been. + */ + private CompletionStage createTransition() { + long incarnation; + + synchronized (lifecycleLock) { + if (syncState != SyncState.INITIAL) { + return CompletableFuture.failedFuture(new UaException(StatusCodes.Bad_InvalidState)); + } + if (maxKeepAliveCount == null) { maxKeepAliveCount = calculateMaxKeepAliveCount(publishingInterval, DEFAULT_TARGET_KEEP_ALIVE_INTERVAL); @@ -155,166 +329,472 @@ public void create() throws UaException { lifetimeCount = calculateLifetimeCount(maxKeepAliveCount); } - CreateSubscriptionResponse response = - client.createSubscription( - publishingInterval, - lifetimeCount, - maxKeepAliveCount, - maxNotificationsPerPublish, - true, - priority); + incarnation = this.incarnation; + } - syncState = SyncState.SYNCHRONIZED; + return client + .createSubscriptionAsync( + publishingInterval, + lifetimeCount, + maxKeepAliveCount, + maxNotificationsPerPublish, + true, + priority) + .thenCompose(response -> applyCreateResponse(response, incarnation)); + } - serverState = - new ServerState( - response.getSubscriptionId(), - response.getRevisedPublishingInterval(), - response.getRevisedLifetimeCount(), - response.getRevisedMaxKeepAliveCount(), - maxNotificationsPerPublish, - priority, - true); + /** + * Install the Subscription the Server created, unless a {@link #reset()} has meanwhile discarded + * it. + * + * @param response the {@link CreateSubscriptionResponse} the Server returned. + * @param incarnation the {@link #incarnation} the call was made for. + * @return a {@link CompletionStage} that completes when the response has been dealt with, or + * completes exceptionally with {@code Bad_InvalidState} if the Subscription it describes has + * been superseded. + */ + private CompletionStage applyCreateResponse( + CreateSubscriptionResponse response, long incarnation) { + + synchronized (lifecycleLock) { + if (this.incarnation == incarnation) { + // Before the SyncState says the Subscription exists: a SYNCHRONIZED Subscription with no + // ServerState has no SubscriptionId to offer, and every operation that needs one answers + // Bad_InvalidState until it appears. + serverState = + new ServerState( + response.getSubscriptionId(), + response.getRevisedPublishingInterval(), + response.getRevisedLifetimeCount(), + response.getRevisedMaxKeepAliveCount(), + maxNotificationsPerPublish, + priority, + true); - watchdogTimer = new WatchdogTimer(); - client.addSessionActivityListener(watchdogTimer); - resetWatchdogTimer(); + syncState = SyncState.SYNCHRONIZED; - client.addSubscription(this); - client.getPublishingManager().addSubscription(this); - } else { - throw new UaException(StatusCodes.Bad_InvalidState); + watchdogTimer = new WatchdogTimer(); + client.addSessionActivityListener(watchdogTimer); + + // Registered while the lock is still held, so the SubscriptionId the PublishingManager + // binds its entry to is the one installed above and not one a reset() has since cleared. + client.addSubscription(this); + client.getPublishingManager().addSubscription(this); + + // Register before checking the suspension gate. If reconnect recovery resumes between + // these operations, either its registry sweep sees this Subscription or this check sees + // publishing allowed; there is no interval in which both can miss it. + resetWatchdogTimer(); + + return CompletableFuture.completedFuture(Unit.VALUE); + } } + + // A reset() overtook this call, so the Subscription the Server has just created is one this + // object has already been told to forget. Deleted out here rather than above: nothing that + // sends a request belongs in that critical section. + deleteAbandonedSubscription(response.getSubscriptionId()); + + return CompletableFuture.failedFuture( + new UaException( + StatusCodes.Bad_InvalidState, "the Subscription was reset while it was being created")); } /** - * Create this Subscription on the Server. + * Call the ModifySubscription service to update the Subscription's parameters on the Server. * - * @return a {@link CompletionStage} that completes successfully if the Subscription was created, - * or completes exceptionally if there was a service- or operation-level error. + *

Serialized against the other lifecycle transitions; see the class documentation. A {@link + * #reset()} made while this call is waiting for the Server supersedes it: the revised parameters + * describe a Subscription this object no longer represents, so they are discarded and the call + * fails with {@code Bad_InvalidState}. + * + *

Blocks until the Server has answered; this is {@link #modifyAsync()} awaited, so it must not + * be called from a transport executor thread. See the class documentation. + * + * @throws UaException if a service- or operation-level error occurs. */ - public CompletionStage createAsync() { - return supplyAsyncCompose( - () -> { - try { - create(); - return CompletableFuture.completedFuture(Unit.VALUE); - } catch (UaException e) { - return CompletableFuture.failedFuture(e); - } - }, - client.getTransport().getConfig().getExecutor()); + public void modify() throws UaException { + await(modifyAsync()); } /** * Call the ModifySubscription service to update the Subscription's parameters on the Server. * - * @throws UaException if a service- or operation-level error occurs. + *

Completes successfully without calling the service if there is nothing pending to send, + * exactly as {@link #modify()} returns without calling it. + * + * @return a {@link CompletionStage} that completes successfully if the Subscription was modified, + * or completes exceptionally if there was a service- or operation-level error. */ - public void modify() throws UaException { - if (syncState == SyncState.INITIAL) { - throw new UaException(StatusCodes.Bad_InvalidState); - } else if (syncState == SyncState.UNSYNCHRONIZED) { - ServerState serverState = this.serverState; + public CompletionStage modifyAsync() { + return runTransition(this::modifyTransition); + } + + /** + * Call the ModifySubscription service with the pending {@link Modifications} and install the + * parameters the Server revises them to. + * + *

Runs with the transition slot claimed; see {@link #runTransition(Supplier)}. + * + * @return a {@link CompletionStage} that completes when the Subscription has been modified, or + * completes exceptionally if it has not been. + */ + private CompletionStage modifyTransition() { + long incarnation; + ServerState serverState; + Modifications diff; + + synchronized (lifecycleLock) { + if (syncState == SyncState.INITIAL) { + return CompletableFuture.failedFuture(new UaException(StatusCodes.Bad_InvalidState)); + } else if (syncState != SyncState.UNSYNCHRONIZED) { + return CompletableFuture.completedFuture(Unit.VALUE); + } + + serverState = this.serverState; if (serverState == null) { - throw new UaException(StatusCodes.Bad_InvalidState); + return CompletableFuture.failedFuture(new UaException(StatusCodes.Bad_InvalidState)); } - Modifications diff = modifications; + diff = modifications; modifications = null; assert diff != null; - ModifySubscriptionResponse response = - client.modifySubscription( - serverState.getSubscriptionId(), - diff.publishingInterval().orElse(serverState.getPublishingInterval()), - diff.lifetimeCount().orElse(serverState.getLifetimeCount()), - diff.maxKeepAliveCount().orElse(serverState.getMaxKeepAliveCount()), - diff.maxNotificationsPerPublish().orElse(maxNotificationsPerPublish), - diff.priority().orElse(priority)); + incarnation = this.incarnation; + } + + ServerState previousState = serverState; + Modifications sentDiff = diff; - resetWatchdogTimer(); + return client + .modifySubscriptionAsync( + serverState.getSubscriptionId(), + diff.publishingInterval().orElse(serverState.getPublishingInterval()), + diff.lifetimeCount().orElse(serverState.getLifetimeCount()), + diff.maxKeepAliveCount().orElse(serverState.getMaxKeepAliveCount()), + diff.maxNotificationsPerPublish().orElse(maxNotificationsPerPublish), + diff.priority().orElse(priority)) + .whenComplete( + (response, ex) -> { + if (ex != null) { + synchronized (lifecycleLock) { + if (this.incarnation == incarnation) { + // The service call failed, so the Subscription remains UNSYNCHRONIZED. Restore + // the pending modifications so the next modify() retries them instead of + // finding nothing to send. Not restored if a reset() overtook the call: it + // cleared them on purpose, and they describe a Subscription that no longer + // exists. + restorePendingModifications(sentDiff); + } + } + } + }) + .thenCompose(response -> applyModifyResponse(response, previousState, incarnation)); + } + + /** + * Install the parameters the Server revised, unless a {@link #reset()} has meanwhile discarded + * the Subscription they belong to. + * + * @param response the {@link ModifySubscriptionResponse} the Server returned. + * @param previousState the {@link ServerState} the call was made against. + * @param incarnation the {@link #incarnation} the call was made for. + * @return a {@link CompletionStage} that completes when the response has been dealt with, or + * completes exceptionally with {@code Bad_InvalidState} if the Subscription it describes has + * been superseded. + */ + private CompletionStage applyModifyResponse( + ModifySubscriptionResponse response, ServerState previousState, long incarnation) { + + synchronized (lifecycleLock) { + if (this.incarnation != incarnation) { + return CompletableFuture.failedFuture( + new UaException( + StatusCodes.Bad_InvalidState, + "the Subscription was reset while it was being modified")); + } this.serverState = new ServerState( - serverState.getSubscriptionId(), + previousState.getSubscriptionId(), response.getRevisedPublishingInterval(), response.getRevisedLifetimeCount(), response.getRevisedMaxKeepAliveCount(), maxNotificationsPerPublish, priority, - serverState.isPublishingEnabled()); + previousState.isPublishingEnabled()); + + // Must happen after the revised parameters are installed: the watchdog delay is + // derived from the current ServerState, so re-arming any earlier would use the + // pre-modify PublishingInterval and MaxKeepAliveCount. + resetWatchdogTimer(); if (modifications == null) { syncState = SyncState.SYNCHRONIZED; } + + return CompletableFuture.completedFuture(Unit.VALUE); } } /** - * Call the ModifySubscription service to update the Subscription's parameters on the Server. + * Restore {@code diff} as the pending {@link Modifications} after a failed modify service call, + * so that a subsequent {@link #modify()} retries the same parameters. * - * @return a {@link CompletionStage} that completes successfully if the Subscription was modified, - * or completes exceptionally if there was a service- or operation-level error. + *

Any Modifications requested while the failed service call was in flight take precedence over + * the values being restored. + * + * @param diff the {@link Modifications} the failed service call attempted to apply. */ - public CompletionStage modifyAsync() { - return supplyAsyncCompose( - () -> { - try { - modify(); - return CompletableFuture.completedFuture(Unit.VALUE); - } catch (UaException e) { - return CompletableFuture.failedFuture(e); - } - }, - client.getTransport().getConfig().getExecutor()); + private void restorePendingModifications(Modifications diff) { + Modifications pending = modifications; + + if (pending == null) { + modifications = diff; + } else { + if (pending.publishingInterval == null) { + pending.publishingInterval = diff.publishingInterval; + } + if (pending.lifetimeCount == null) { + pending.lifetimeCount = diff.lifetimeCount; + } + if (pending.maxKeepAliveCount == null) { + pending.maxKeepAliveCount = diff.maxKeepAliveCount; + } + if (pending.maxNotificationsPerPublish == null) { + pending.maxNotificationsPerPublish = diff.maxNotificationsPerPublish; + } + if (pending.priority == null) { + pending.priority = diff.priority; + } + } } /** * Delete this Subscription from the Server. * + *

Serialized against the other lifecycle transitions; see the class documentation. A {@link + * #reset()} made while this call is waiting for the Server has already discarded the Subscription + * by the time the response arrives, so this call does not reset it a second time; the + * operation-level result of the DeleteSubscriptions call is reported either way. + * + *

Blocks until the Server has answered; this is {@link #deleteAsync()} awaited, so it must not + * be called from a transport executor thread. See the class documentation. + * * @throws UaException if a service- or operation-level error occurs. */ public void delete() throws UaException { - if (syncState != SyncState.INITIAL) { - ServerState serverState = this.serverState; + await(deleteAsync()); + } + + /** + * Delete this Subscription from the Server. + * + *

Completes successfully without calling the service if this Subscription does not exist on + * the Server, exactly as {@link #delete()} returns without calling it. + * + * @return a {@link CompletionStage} that completes successfully if the Subscription was deleted, + * or completes exceptionally if there was a service- or operation-level error. + */ + public CompletionStage deleteAsync() { + return runTransition(this::deleteTransition); + } + + /** + * Call the DeleteSubscriptions service and discard the Subscription it deleted. + * + *

Runs with the transition slot claimed; see {@link #runTransition(Supplier)}. + * + * @return a {@link CompletionStage} that completes when the Subscription has been deleted, or + * completes exceptionally if it has not been. + */ + private CompletionStage deleteTransition() { + long incarnation; + ServerState serverState; + + synchronized (lifecycleLock) { + if (syncState == SyncState.INITIAL) { + return CompletableFuture.completedFuture(Unit.VALUE); + } + + serverState = this.serverState; if (serverState == null) { - throw new UaException(StatusCodes.Bad_InvalidState); + return CompletableFuture.failedFuture(new UaException(StatusCodes.Bad_InvalidState)); } - DeleteSubscriptionsResponse response = - client.deleteSubscriptions(List.of(serverState.getSubscriptionId())); + incarnation = this.incarnation; + } + + return client + .deleteSubscriptionsAsync(List.of(serverState.getSubscriptionId())) + .thenCompose(response -> applyDeleteResponse(response, incarnation)); + } - StatusCode result = requireNonNull(response.getResults())[0]; + /** + * Discard the Subscription the Server deleted, unless a {@link #reset()} has meanwhile discarded + * it already. + * + * @param response the {@link DeleteSubscriptionsResponse} the Server returned. + * @param incarnation the {@link #incarnation} the call was made for. + * @return a {@link CompletionStage} that completes when the response has been dealt with, or + * completes exceptionally with the operation-level result if it was not Good. + */ + private CompletionStage applyDeleteResponse( + DeleteSubscriptionsResponse response, long incarnation) { - if (result.isGood() || result.value() == StatusCodes.Bad_SubscriptionIdInvalid) { + StatusCode result = requireNonNull(response.getResults())[0]; + + synchronized (lifecycleLock) { + // A reset() that overtook this call has already discarded the Subscription, so there is + // nothing left for this one to discard. + if (this.incarnation == incarnation + && (result.isGood() || result.value() == StatusCodes.Bad_SubscriptionIdInvalid)) { reset(); } + } - if (!result.isGood()) { - throw new UaException(result); + return result.isGood() + ? CompletableFuture.completedFuture(Unit.VALUE) + : CompletableFuture.failedFuture(new UaException(result)); + } + + /** + * Run a lifecycle transition as soon as no other one is in flight. + * + *

Nothing here blocks, and neither does anything it runs: a transition that has to wait its + * turn is queued in {@link #transitionWaiters} and started by whichever thread releases the slot. + * That is what lets the {@code ...Async()} forms be composed from the client's asynchronous + * services and awaited by the blocking ones, instead of each blocking form being dispatched onto + * the transport executor to wait there for a response only that executor can complete. + * + * @param transition the transition to run. It is called with the transition slot claimed and + * {@link #lifecycleLock} not held, must not block, and must report failure through the {@link + * CompletionStage} it returns. + * @return a {@link CompletionStage} that completes when {@code transition} has, with whatever + * {@code transition} completed with. + */ + private CompletionStage runTransition(Supplier> transition) { + // Completed to hand the transition slot to this transition: below if nothing else holds it, or + // by endTransition() once the transition ahead of it is finished. + var slot = new CompletableFuture(); + + boolean claimed; + + synchronized (lifecycleLock) { + claimed = !transitionInFlight; + + if (claimed) { + transitionInFlight = true; + } else { + transitionWaiters.add(slot); } } + + CompletionStage internalResult = + slot.thenCompose(unit -> transition.get()).whenComplete((unit, ex) -> endTransition()); + + // Callers are allowed to cancel or time out the stage they receive, but that must not cancel + // the dependent stage above: it owns the transition slot until the actual service operation + // has completed and endTransition() has handed the slot on. This extra dependent preserves the + // caller-visible result while isolating the internal cleanup from external completion. + CompletionStage callerResult = internalResult.thenApply(unit -> unit); + + // After the chain above is built, so that a slot already in hand starts the transition here + // rather than leaving it to whoever completes the stage next. + if (claimed) { + slot.complete(Unit.VALUE); + } + + return callerResult; } /** - * Delete this Subscription from the Server. + * Release the transition slot claimed by {@link #runTransition(Supplier)}, handing it straight to + * the transition that has been waiting longest if there is one. * - * @return a {@link CompletionStage} that completes successfully if the Subscription was deleted, - * or completes exceptionally if there was a service- or operation-level error. + *

Called however a transition ends, so that the next one is not left waiting on a slot nobody + * holds. Handed over rather than released and re-claimed: a slot that was briefly free would let + * a transition that arrives now overtake one that has been waiting. */ - public CompletionStage deleteAsync() { - return supplyAsyncCompose( - () -> { - try { - delete(); - return CompletableFuture.completedFuture(Unit.VALUE); - } catch (UaException e) { - return CompletableFuture.failedFuture(e); - } - }, - client.getTransport().getConfig().getExecutor()); + private void endTransition() { + CompletableFuture next; + + synchronized (lifecycleLock) { + next = transitionWaiters.poll(); + + if (next == null) { + transitionInFlight = false; + } + } + + if (next != null) { + // Completed asynchronously rather than on this stack: a queued transition that completes + // synchronously — nothing pending to send, or an immediate Bad_InvalidState — would re-enter + // this method inline, one frame per waiter, and enough waiters is a StackOverflowError that + // leaves the slot claimed forever. + next.completeAsync(() -> Unit.VALUE, client.getTransport().getConfig().getExecutor()); + } + } + + /** + * Await a lifecycle transition, so that a blocking transition is its asynchronous counterpart and + * nothing more. + * + * @param stage the {@link CompletionStage} the asynchronous transition returned. + * @throws UaException if {@code stage} completed exceptionally, or the calling thread was + * interrupted while waiting for it. + */ + private static void await(CompletionStage stage) throws UaException { + try { + stage.toCompletableFuture().get(); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + + // Rethrown rather than wrapped when it is already a UaException: the blocking and the + // asynchronous form of a transition must report the same StatusCode and the same message. + if (cause instanceof UaException uaException) { + throw uaException; + } else { + throw new UaException(cause); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new UaException(StatusCodes.Bad_UnexpectedError, e); + } + } + + /** + * Delete a Subscription the Server created for this object but which a {@link #reset()} means it + * can no longer represent. + * + *

Best effort and never awaited: the caller's result has already been discarded, and the + * reason the reset was allowed to overtake it in the first place is that its own callers cannot + * afford to wait for the Server. Part 4 §5.13.8 gives DeleteSubscriptions the SubscriptionId as + * its only handle on a Subscription, so this is the last chance anything has to name this one; if + * the attempt fails it runs on the Server until its lifetime expires, which is worth a warning + * and nothing more. + * + * @param subscriptionId the SubscriptionId of the Subscription to delete. + */ + private void deleteAbandonedSubscription(UInteger subscriptionId) { + logger.debug("id={}, deleting Subscription abandoned by a concurrent reset()", subscriptionId); + + client + .deleteSubscriptionsAsync(List.of(subscriptionId)) + .whenComplete( + (response, ex) -> { + if (ex != null) { + logger.warn("id={}, failed to delete abandoned Subscription", subscriptionId, ex); + } else { + StatusCode result = requireNonNull(response.getResults())[0]; + + if (!result.isGood() && result.value() != StatusCodes.Bad_SubscriptionIdInvalid) { + logger.warn( + "id={}, failed to delete abandoned Subscription: {}", subscriptionId, result); + } + } + }); } // endregion @@ -331,27 +811,29 @@ public CompletionStage deleteAsync() { * @throws UaRuntimeException if the Subscription has not been created yet. */ public void addMonitoredItem(OpcUaMonitoredItem item) { - Optional existingHandle = item.getClientHandle(); + synchronized (monitoredItemsLock) { + Optional existingHandle = item.getClientHandle(); - if (existingHandle.isPresent()) { - UInteger handle = existingHandle.get(); + if (existingHandle.isPresent()) { + UInteger handle = existingHandle.get(); - // O(1) check: if this item is already in the map, nothing to do - if (monitoredItems.get(handle) == item) { - return; - } + // O(1) check: if this item is already in the map, nothing to do + if (monitoredItems.get(handle) == item) { + return; + } - // Item has a handle but isn't in map - check if pending deletion - if (itemsToDelete.remove(item)) { - monitoredItems.put(handle, item); - } - // else: item has a handle from a different context, ignore - } else { - // Brand-new item with no handle - UInteger clientHandle = clientHandleSequence.nextClientHandle(); - item.setClientHandle(clientHandle); + // Item has a handle but isn't in map - check if pending deletion + if (itemsToDelete.remove(item)) { + monitoredItems.put(handle, item); + } + // else: item has a handle from a different context, ignore + } else { + // Brand-new item with no handle + UInteger clientHandle = clientHandleSequence.nextClientHandle(); + item.setClientHandle(clientHandle); - monitoredItems.put(clientHandle, item); + monitoredItems.put(clientHandle, item); + } } } @@ -377,7 +859,9 @@ public void addMonitoredItems(List items) { * @param item the MonitoredItem to remove. */ public void removeMonitoredItem(OpcUaMonitoredItem item) { - item.getClientHandle().map(monitoredItems::remove).ifPresent(itemsToDelete::add); + synchronized (monitoredItemsLock) { + item.getClientHandle().map(monitoredItems::remove).ifPresent(itemsToDelete::add); + } } /** @@ -614,21 +1098,35 @@ private List modifyMonitoredItems( /** * Delete any MonitoredItems that have been removed from the Subscription. * + *

A MonitoredItem is only removed from the pending deletion queue once the Server has reported + * an operation-level result for it. A deletion that never reached the Server, e.g. because the + * service call failed, remains pending and is attempted again by the next call. + * * @return a List of the MonitoredItems that were deleted. */ public List deleteMonitoredItems() { - List itemsToDelete = - this.itemsToDelete.stream() - .filter(item -> item.getSyncState() != OpcUaMonitoredItem.SyncState.INITIAL) - .collect(Collectors.toList()); + List itemsToDelete; + + synchronized (monitoredItemsLock) { + // Items that were never created on the Server don't need to be deleted from it. + this.itemsToDelete.removeIf( + item -> { + if (item.getSyncState() == OpcUaMonitoredItem.SyncState.INITIAL) { + item.setClientHandle(null); + return true; + } - this.itemsToDelete.clear(); + return false; + }); - if (!itemsToDelete.isEmpty()) { - return deleteMonitoredItems(itemsToDelete); - } else { + itemsToDelete = List.copyOf(this.itemsToDelete); + } + + if (itemsToDelete.isEmpty()) { return Collections.emptyList(); } + + return deleteMonitoredItems(itemsToDelete); } private List deleteMonitoredItems( @@ -658,12 +1156,16 @@ private List deleteMonitoredItems( //noinspection DuplicatedCode var monitoredItemIds = new ArrayList(partition.size()); var itemIds = new ArrayList>(partition.size()); + var clientHandles = new ArrayList>(partition.size()); - for (OpcUaMonitoredItem item : partition) { - Optional itemId = item.getMonitoredItemId(); + synchronized (monitoredItemsLock) { + for (OpcUaMonitoredItem item : partition) { + Optional itemId = item.getMonitoredItemId(); - itemIds.add(itemId); - itemId.ifPresent(monitoredItemIds::add); + itemIds.add(itemId); + clientHandles.add(item.getClientHandle()); + itemId.ifPresent(monitoredItemIds::add); + } } if (monitoredItemIds.isEmpty()) { @@ -694,7 +1196,7 @@ private List deleteMonitoredItems( if (itemIds.get(i).isPresent()) { StatusCode result = results[resultIndex++]; - item.applyDeleteResult(result); + applyMonitoredItemDeleteResult(item, clientHandles.get(i), result); serviceOperationsResults.add( new MonitoredItemServiceOperationResult(item, StatusCode.GOOD, result)); @@ -723,6 +1225,38 @@ private List deleteMonitoredItems( return serviceOperationsResults; } + /** + * Apply an operation-level DeleteMonitoredItems result without corrupting an item that was added + * back while the service call was in flight. + * + *

The Server has acted on the deletion, so the item's server state must be cleared either way. + * If the application added the item back, it remains in {@link #monitoredItems}; restore the + * ClientHandle captured for the request after {@link OpcUaMonitoredItem#applyDeleteResult} clears + * it, so a later {@link #createMonitoredItems()} can recreate the desired item. If it was not + * added back, it stays detached as before. + */ + private void applyMonitoredItemDeleteResult( + OpcUaMonitoredItem item, Optional clientHandle, StatusCode result) { + + synchronized (monitoredItemsLock) { + boolean readded = + clientHandle.map(handle -> monitoredItems.get(handle) == item).orElse(false); + + item.applyDeleteResult(result); + + if (readded) { + UInteger handle = clientHandle.orElseThrow(); + item.setClientHandle(handle); + } + + // An operation-level result means the Server acted on the deletion, either by deleting the + // item or by reporting that it was already gone. Remove the original pending deletion in the + // same critical section as applying the result, so a later remove is a distinct operation + // that this response cannot consume. + itemsToDelete.remove(item); + } + } + private UInteger getMonitoredItemPartitionSize() { return monitoredItemPartitionSize.get( () -> { @@ -868,37 +1402,20 @@ public List setMonitoringMode( /** * Set the publishing mode, i.e. enable or disable publishing, for this Subscription. * + *

Serialized against the other lifecycle transitions; see the class documentation. A {@link + * #reset()} made while this call is waiting for the Server supersedes it: the new publishing mode + * belongs to a Subscription this object no longer represents, so it is discarded and the call + * fails with {@code Bad_InvalidState}. + * + *

Blocks until the Server has answered; this is {@link #setPublishingModeAsync(boolean)} + * awaited, so it must not be called from a transport executor thread. See the class + * documentation. + * * @param enabled {@code true} to enable publishing, {@code false} to disable publishing. * @throws UaException if a service- or operation-level error occurs. */ public void setPublishingMode(boolean enabled) throws UaException { - if (syncState == SyncState.INITIAL) { - throw new UaException(StatusCodes.Bad_InvalidState); - } else { - ServerState serverState = this.serverState; - if (serverState == null) { - throw new UaException(StatusCodes.Bad_InvalidState); - } - - SetPublishingModeResponse response = - client.setPublishingMode(enabled, List.of(serverState.getSubscriptionId())); - - StatusCode result = requireNonNull(response.getResults())[0]; - - if (result.isGood()) { - this.serverState = - new ServerState( - serverState.getSubscriptionId(), - serverState.getPublishingInterval(), - serverState.getLifetimeCount(), - serverState.getMaxKeepAliveCount(), - maxNotificationsPerPublish, - priority, - enabled); - } else { - throw new UaException(result); - } - } + await(setPublishingModeAsync(enabled)); } /** @@ -909,21 +1426,101 @@ public void setPublishingMode(boolean enabled) throws UaException { * or completes exceptionally if there was a service- or operation-level error. */ public CompletionStage setPublishingModeAsync(boolean enabled) { - return supplyAsyncCompose( - () -> { - try { - setPublishingMode(enabled); - return CompletableFuture.completedFuture(Unit.VALUE); - } catch (UaException e) { - return CompletableFuture.failedFuture(e); - } - }, - client.getTransport().getConfig().getExecutor()); + return runTransition(() -> setPublishingModeTransition(enabled)); + } + + /** + * Call the SetPublishingMode service and record the publishing mode the Server accepted. + * + *

Runs with the transition slot claimed; see {@link #runTransition(Supplier)}. + * + * @param enabled {@code true} to enable publishing, {@code false} to disable publishing. + * @return a {@link CompletionStage} that completes when the publishing mode has been set, or + * completes exceptionally if it has not been. + */ + private CompletionStage setPublishingModeTransition(boolean enabled) { + long incarnation; + ServerState serverState; + + synchronized (lifecycleLock) { + if (syncState == SyncState.INITIAL) { + return CompletableFuture.failedFuture(new UaException(StatusCodes.Bad_InvalidState)); + } + + serverState = this.serverState; + if (serverState == null) { + return CompletableFuture.failedFuture(new UaException(StatusCodes.Bad_InvalidState)); + } + + incarnation = this.incarnation; + } + + ServerState previousState = serverState; + + return client + .setPublishingModeAsync(enabled, List.of(serverState.getSubscriptionId())) + .thenCompose( + response -> + applySetPublishingModeResponse(response, previousState, enabled, incarnation)); + } + + /** + * Record the publishing mode the Server accepted, unless a {@link #reset()} has meanwhile + * discarded the Subscription it belongs to. + * + * @param response the {@link SetPublishingModeResponse} the Server returned. + * @param previousState the {@link ServerState} the call was made against. + * @param enabled the publishing mode the call requested. + * @param incarnation the {@link #incarnation} the call was made for. + * @return a {@link CompletionStage} that completes when the response has been dealt with, or + * completes exceptionally with {@code Bad_InvalidState} if the Subscription it describes has + * been superseded, or with the operation-level result if it was not Good. + */ + private CompletionStage applySetPublishingModeResponse( + SetPublishingModeResponse response, + ServerState previousState, + boolean enabled, + long incarnation) { + + StatusCode result = requireNonNull(response.getResults())[0]; + + synchronized (lifecycleLock) { + if (this.incarnation != incarnation) { + return CompletableFuture.failedFuture( + new UaException( + StatusCodes.Bad_InvalidState, + "the Subscription was reset while its publishing mode was being set")); + } + + if (!result.isGood()) { + return CompletableFuture.failedFuture(new UaException(result)); + } + + this.serverState = + new ServerState( + previousState.getSubscriptionId(), + previousState.getPublishingInterval(), + previousState.getLifetimeCount(), + previousState.getMaxKeepAliveCount(), + maxNotificationsPerPublish, + priority, + enabled); + + return CompletableFuture.completedFuture(Unit.VALUE); + } } // endregion /** + * Get the current {@link SyncState} of this Subscription. + * + *

A lifecycle transition waiting for the Server is not visible here: the state does not + * advance until the Server has answered, so a Subscription being created still reports {@link + * SyncState#INITIAL}, and one being modified still reports {@link SyncState#UNSYNCHRONIZED}. That + * is what remains true of the Subscription while the request is in flight — and it is also why a + * transition can fail without the state having to be rolled back. + * * @return the current {@link SyncState} of this Subscription. */ public SyncState getSyncState() { @@ -1115,25 +1712,27 @@ public Optional getRevisedMaxKeepAliveCount() { * @see #modifyAsync() */ public void setPublishingInterval(Double publishingInterval) { - this.publishingInterval = publishingInterval; + synchronized (lifecycleLock) { + this.publishingInterval = publishingInterval; - if (syncState != SyncState.INITIAL) { - if (modifications == null) { - modifications = new Modifications(); - } + if (syncState != SyncState.INITIAL) { + if (modifications == null) { + modifications = new Modifications(); + } - modifications.publishingInterval = publishingInterval; + modifications.publishingInterval = publishingInterval; - syncState = SyncState.UNSYNCHRONIZED; - } + syncState = SyncState.UNSYNCHRONIZED; + } - if (lifetimeAndKeepAliveCalculated) { - UInteger maxKeepAliveCount = - calculateMaxKeepAliveCount(publishingInterval, DEFAULT_TARGET_KEEP_ALIVE_INTERVAL); - UInteger lifetimeCount = calculateLifetimeCount(maxKeepAliveCount); + if (lifetimeAndKeepAliveCalculated) { + UInteger maxKeepAliveCount = + calculateMaxKeepAliveCount(publishingInterval, DEFAULT_TARGET_KEEP_ALIVE_INTERVAL); + UInteger lifetimeCount = calculateLifetimeCount(maxKeepAliveCount); - setMaxKeepAliveCount(maxKeepAliveCount); - setLifetimeCount(lifetimeCount); + setMaxKeepAliveCount(maxKeepAliveCount); + setLifetimeCount(lifetimeCount); + } } } @@ -1153,16 +1752,18 @@ public void setPublishingInterval(Double publishingInterval) { * @see #modifyAsync() */ public void setLifetimeCount(UInteger lifetimeCount) { - this.lifetimeCount = lifetimeCount; + synchronized (lifecycleLock) { + this.lifetimeCount = lifetimeCount; - if (syncState != SyncState.INITIAL) { - if (modifications == null) { - modifications = new Modifications(); - } + if (syncState != SyncState.INITIAL) { + if (modifications == null) { + modifications = new Modifications(); + } - modifications.lifetimeCount = lifetimeCount; + modifications.lifetimeCount = lifetimeCount; - syncState = SyncState.UNSYNCHRONIZED; + syncState = SyncState.UNSYNCHRONIZED; + } } } @@ -1182,16 +1783,18 @@ public void setLifetimeCount(UInteger lifetimeCount) { * @see #modifyAsync() */ public void setMaxKeepAliveCount(UInteger maxKeepAliveCount) { - this.maxKeepAliveCount = maxKeepAliveCount; + synchronized (lifecycleLock) { + this.maxKeepAliveCount = maxKeepAliveCount; - if (syncState != SyncState.INITIAL) { - if (modifications == null) { - modifications = new Modifications(); - } + if (syncState != SyncState.INITIAL) { + if (modifications == null) { + modifications = new Modifications(); + } - modifications.maxKeepAliveCount = maxKeepAliveCount; + modifications.maxKeepAliveCount = maxKeepAliveCount; - syncState = SyncState.UNSYNCHRONIZED; + syncState = SyncState.UNSYNCHRONIZED; + } } } @@ -1211,16 +1814,18 @@ public void setMaxKeepAliveCount(UInteger maxKeepAliveCount) { * @see #modifyAsync() */ public void setPriority(UByte priority) { - this.priority = priority; + synchronized (lifecycleLock) { + this.priority = priority; - if (syncState != SyncState.INITIAL) { - if (modifications == null) { - modifications = new Modifications(); - } + if (syncState != SyncState.INITIAL) { + if (modifications == null) { + modifications = new Modifications(); + } - modifications.priority = priority; + modifications.priority = priority; - syncState = SyncState.UNSYNCHRONIZED; + syncState = SyncState.UNSYNCHRONIZED; + } } } @@ -1240,16 +1845,18 @@ public void setPriority(UByte priority) { * @see #modifyAsync() */ public void setMaxNotificationsPerPublish(UInteger maxNotificationsPerPublish) { - this.maxNotificationsPerPublish = maxNotificationsPerPublish; + synchronized (lifecycleLock) { + this.maxNotificationsPerPublish = maxNotificationsPerPublish; - if (syncState != SyncState.INITIAL) { - if (modifications == null) { - modifications = new Modifications(); - } + if (syncState != SyncState.INITIAL) { + if (modifications == null) { + modifications = new Modifications(); + } - modifications.maxNotificationsPerPublish = maxNotificationsPerPublish; + modifications.maxNotificationsPerPublish = maxNotificationsPerPublish; - syncState = SyncState.UNSYNCHRONIZED; + syncState = SyncState.UNSYNCHRONIZED; + } } } @@ -1364,26 +1971,105 @@ public TaskQueue getDeliveryQueue() { * cleared. If the Subscription is created again, the call {@link #synchronizeMonitoredItems()} or * {@link #createMonitoredItems()} to create the items on the Server again. * + *

MonitoredItems that were removed from the Subscription but not yet deleted from the Server + * are discarded: they belong to a Subscription that no longer exists, so there is nothing left to + * delete. + * *

This is called automatically when the Subscription is deleted, but can also be called * manually when necessary if it has been determined the Subscription no longer exists on the * Server. + * + *

Never waits for the Server, and never waits for a lifecycle transition that is waiting for + * the Server; see the class documentation for why the callers cannot afford it to. + * + *

A reset still never lands in the middle of a {@link #create()}: it either discards the + * Subscription that existed before it, or the one that call goes on to create. What it no longer + * does is wait for that call to finish first — a {@link #create()} still in flight is superseded, + * so it fails with {@code Bad_InvalidState} and the Subscription the Server created for it is + * deleted rather than left running with nothing able to name it. A {@link #modify()}, {@link + * #delete()} or {@link #setPublishingMode(boolean)} in flight is superseded the same way. */ public void reset() { - if (syncState != SyncState.INITIAL) { - cancelWatchdogTimer(); - client.removeSubscription(this); - client.getPublishingManager().removeSubscription(this); + resetInternal(); + } + + /** Perform the reset without virtual dispatch. */ + private void resetInternal() { + synchronized (lifecycleLock) { + // Unconditional, and before anything else: this is the only record a transition already + // waiting for the Server has that the Subscription it was called for is gone. A create() in + // flight has published no SyncState yet, so the SyncState check below says nothing about it, + // but its result must be discarded all the same. + incarnation++; + + if (syncState != SyncState.INITIAL) { + cancelWatchdogTimer(); + client.removeSubscription(this); + client.getPublishingManager().removeSubscription(this); + + serverState = null; + modifications = null; + + monitoredItemPartitionSize.reset(); + monitoredItems.values().forEach(OpcUaMonitoredItem::reset); + + // MonitoredItemIds are scoped to the Subscription that no longer exists, so the items + // pending deletion are already gone and their ids must never be sent again. Detach them + // completely, including the ClientHandle, so they can be added to a Subscription again. + itemsToDelete.forEach( + item -> { + item.reset(); + item.setClientHandle(null); + }); + itemsToDelete.clear(); - serverState = null; - modifications = null; + syncState = SyncState.INITIAL; + } + } + } - monitoredItemPartitionSize.reset(); - monitoredItems.values().forEach(OpcUaMonitoredItem::reset); + /** + * Get the incarnation of this object, i.e. which Subscription it currently represents. + * + *

Captured by {@link PublishingManager} when it registers an entry for this object, and + * compared again by {@link #resetIfIncarnation(long)}: a change means this object has stopped + * representing the Subscription the caller knew it as. + * + * @return the current {@link #incarnation}. + */ + long getIncarnation() { + synchronized (lifecycleLock) { + return incarnation; + } + } - syncState = SyncState.INITIAL; + /** + * {@link #reset()} this Subscription, but only if it still represents the incarnation the caller + * knew it as. + * + *

This is the teardown for a Bad_Timeout StatusChangeNotification, which is a statement about + * the Subscription the message was received on: applied unconditionally, a stale one — delivered + * behind application callbacks of arbitrary duration — would tear down a Subscription created + * after it was received. + * + * @param incarnation the {@link #incarnation} the caller is acting on behalf of. + */ + void resetIfIncarnation(long incarnation) { + synchronized (lifecycleLock) { + if (this.incarnation == incarnation) { + reset(); + } } } + /** + * Permanently cancel the watchdog timer: the pending expiry is cancelled, the {@link + * SessionActivityListener} is de-registered, and the timer is discarded. + * + *

This is for teardown of the Subscription itself, e.g. {@link #reset()}. It cannot be undone; + * a new timer is only created by {@link #create()}. To suspend the timer while the Session is + * unavailable use {@link #pauseWatchdogTimer()} instead. + */ synchronized void cancelWatchdogTimer() { WatchdogTimer watchdog = this.watchdogTimer; if (watchdog != null) { @@ -1396,7 +2082,36 @@ synchronized void cancelWatchdogTimer() { } } + /** + * Suspend the watchdog timer: the pending expiry is cancelled, but the timer remains registered + * as a {@link SessionActivityListener} and is re-armed when the Session becomes active again. + * + *

This is for temporary Session unavailability, where no PublishResponse can arrive and the + * Server's keep-alive obligation is therefore in abeyance, but the Subscription itself may well + * survive (e.g. via TransferSubscriptions once the Session is re-activated). + */ + synchronized void pauseWatchdogTimer() { + WatchdogTimer watchdog = this.watchdogTimer; + if (watchdog != null) { + watchdog.pause(); + logger.debug( + "id={}, watchdog timer paused", + getServerState().map(ServerState::getSubscriptionId).orElse(null)); + } + } + synchronized void resetWatchdogTimer() { + if (client.getPublishingManager().isPublishingSuspended()) { + // The watchdog is fed only by PublishResponses, and none can arrive while Publish traffic + // is suspended: arming it now — from a create or modify made mid-recovery, or from the + // finish of a recovery a newer activation has superseded — could only have it fire on a + // healthy Subscription. The recovery that ends the suspension re-arms it. + logger.debug( + "id={}, watchdog timer reset deferred pending reconnect recovery", + getServerState().map(ServerState::getSubscriptionId).orElse(null)); + return; + } + WatchdogTimer watchdog = this.watchdogTimer; if (watchdog != null) { watchdog.reset(); @@ -1455,13 +2170,18 @@ void notifyDataReceived(MonitoredItemNotification[] notifications) { SubscriptionListener listener = this.listener; if (listener != null) { - listener.onDataReceived(this, items, values); + // Unmodifiable views: the Lists below are the ones the fan-out iterates. + List itemsView = Collections.unmodifiableList(items); + List valuesView = Collections.unmodifiableList(values); + + deliverToListener( + "onDataReceived", () -> listener.onDataReceived(this, itemsView, valuesView)); } for (int i = 0; i < items.size(); i++) { OpcUaMonitoredItem item = items.get(i); DataValue value = values.get(i); - item.notifyDataValueReceived(value); + deliverToListener("DataValueListener", () -> item.notifyDataValueReceived(value)); } } @@ -1486,34 +2206,53 @@ void notifyEventsReceived(EventFieldList[] events) { SubscriptionListener listener = this.listener; if (listener != null) { - listener.onEventReceived(this, items, eventValuesList); + // Unmodifiable views: the Lists below are the ones the fan-out iterates. + List itemsView = Collections.unmodifiableList(items); + List fieldsView = Collections.unmodifiableList(eventValuesList); + + deliverToListener( + "onEventReceived", () -> listener.onEventReceived(this, itemsView, fieldsView)); } for (int i = 0; i < items.size(); i++) { OpcUaMonitoredItem item = items.get(i); Variant[] eventValues = eventValuesList.get(i); - item.notifyEventValuesReceived(eventValues); + deliverToListener("EventValueListener", () -> item.notifyEventValuesReceived(eventValues)); } } + /** + * Called from {@link PublishingManager} while already executing on {@link #deliveryQueue}, so the + * listener is invoked inline to keep it ordered with the data and event callbacks and inside the + * delivery task the backpressure mechanism waits on. + */ void notifyKeepAliveReceived() { SubscriptionListener listener = this.listener; if (listener != null) { - deliveryQueue.execute(() -> listener.onKeepAliveReceived(this)); + deliverToListener("onKeepAliveReceived", () -> listener.onKeepAliveReceived(this)); } } + /** + * Called from {@link PublishingManager} while already executing on {@link #deliveryQueue}, so the + * listener is invoked inline to keep it ordered with the data and event callbacks and inside the + * delivery task the backpressure mechanism waits on. + * + *

The teardown a Bad_Timeout implies is not done here: the caller applies it via {@link + * #resetIfIncarnation(long)}, guarded by the identity of the Subscription the notification was + * received on, which this object no longer knows. + */ void notifyStatusChanged(StatusCode status) { - if (status.getValue() == StatusCodes.Bad_Timeout) { - reset(); - } - SubscriptionListener listener = this.listener; if (listener != null) { - deliveryQueue.execute(() -> listener.onStatusChanged(this, status)); + deliverToListener("onStatusChanged", () -> listener.onStatusChanged(this, status)); } } + /** + * Unlike the notification callbacks above, this is called from off the {@link + * #deliveryQueue} and must therefore be enqueued onto it. + */ void notifyNotificationDataLost() { SubscriptionListener listener = this.listener; if (listener != null) { @@ -1521,8 +2260,17 @@ void notifyNotificationDataLost() { } } + /** + * Reset this Subscription and enqueue its transfer-failure listener notification. + * + *

Session transfer uses {@link #handleTransferFailure(StatusCode)} so the reset cannot be + * delayed by an override of this method. A direct call retains the established public behavior: + * it resets first, then queues the listener callback onto the {@link #deliveryQueue}. + */ public void notifyTransferFailed(StatusCode status) { - reset(); + if (!Boolean.TRUE.equals(transferFailureAlreadyHandled.get())) { + reset(); + } SubscriptionListener listener = this.listener; if (listener != null) { @@ -1530,6 +2278,66 @@ public void notifyTransferFailed(StatusCode status) { } } + /** + * Reset this Subscription before the Session FSM can become Active, then dispatch the public, + * overridable transfer-failure notification away from the FSM completion path. + * + *

This method is final because transfer cleanup is an internal ordering invariant: a subclass + * that blocks or throws from {@link #notifyTransferFailed(StatusCode)} must not keep the Session + * in its Transferring state or let reconnect recovery observe a Subscription that was not + * transferred. + * + * @param status the operation- or service-level transfer failure. + */ + public final void handleTransferFailure(StatusCode status) { + resetInternal(); + + try { + client + .getTransport() + .getConfig() + .getExecutor() + .execute( + () -> { + transferFailureAlreadyHandled.set(Boolean.TRUE); + + try { + notifyTransferFailed(status); + } catch (Exception e) { + logger.warn("notifyTransferFailed threw an unhandled Exception", e); + } finally { + transferFailureAlreadyHandled.remove(); + } + }); + } catch (Exception e) { + // Cleanup is complete. A rejected application notification must not fail Session transfer or + // leave the FSM waiting forever for work its executor will never accept. + logger.warn("could not dispatch notifyTransferFailed", e); + } + } + + /** + * Invoke an application-supplied callback, containing any Exception it throws. + * + *

The {@link SubscriptionListener} and each MonitoredItem's listener are independent sinks for + * the same notification: one of them failing must not cost the others their notification, and + * must not abort delivery of the remaining NotificationData in the same NotificationMessage. + * + * @param callback the name of the callback being invoked, for logging. + * @param delivery the callback invocation. + */ + private void deliverToListener(String callback, Runnable delivery) { + try { + delivery.run(); + } catch (Exception e) { + logger.warn( + "id={}, {} threw an unhandled Exception", + getServerState().map(ServerState::getSubscriptionId).orElse(null), + callback, + e); + } + } + private static class Modifications { private volatile @Nullable Double publishingInterval; @@ -1561,20 +2369,66 @@ private Optional priority() { private class WatchdogTimer implements SessionActivityListener { - private final AtomicReference> scheduledFuture = new AtomicReference<>(); + /** + * Guards every transition of this timer. The state below is read and written by unrelated + * threads - the transport executor completing a PublishResponse, the Session FSM notifying + * activity listeners, and the scheduled executor running an expiry - and each transition spans + * a cancel/schedule/store sequence that must be atomic as a whole. + */ + private final Object lock = new Object(); + + /** The pending expiry, or {@code null} if the timer is not armed. Guarded by {@link #lock}. */ + private @Nullable ScheduledFuture scheduledFuture; + + /** Terminal once set: a cancelled timer never arms again. Guarded by {@link #lock}. */ + private boolean cancelled = false; + + /** + * Incremented on every transition, and captured by each expiry when it is scheduled, so that an + * expiry which has already begun running - and which {@code ScheduledFuture.cancel(false)} + * therefore cannot stop - is recognised as stale and ignored. Guarded by {@link #lock}. + */ + private long epoch = 0L; void reset() { - ScheduledFuture sf = scheduledFuture.get(); - if (sf != null) sf.cancel(false); + synchronized (lock) { + if (cancelled) { + return; + } + + cancelPending(); + scheduleNext(); + } + } - scheduleNext(); + /** Cancel the pending expiry, leaving the timer able to arm again. */ + void pause() { + synchronized (lock) { + cancelPending(); + } } + /** Cancel the pending expiry permanently; subsequent calls to {@link #reset()} are no-ops. */ void cancel() { - ScheduledFuture sf = scheduledFuture.getAndSet(null); - if (sf != null) sf.cancel(false); + synchronized (lock) { + cancelled = true; + + cancelPending(); + } } + /** Must be called while holding {@link #lock}. */ + private void cancelPending() { + epoch++; + + ScheduledFuture sf = scheduledFuture; + if (sf != null) { + sf.cancel(false); + scheduledFuture = null; + } + } + + /** Must be called while holding {@link #lock}. */ private void scheduleNext() { getServerState() .ifPresent( @@ -1584,22 +2438,33 @@ private void scheduleNext() { (state.publishingInterval * (state.maxKeepAliveCount.longValue() + 1)) * watchdogMultiplier); - ScheduledFuture nextSf = + long scheduledEpoch = epoch; + + scheduledFuture = client .getTransport() .getConfig() .getScheduledExecutor() .schedule( - () -> notifyWatchdogTimerElapsed(delay), delay, TimeUnit.MILLISECONDS); - - scheduledFuture.set(nextSf); + () -> notifyWatchdogTimerElapsed(scheduledEpoch, delay), + delay, + TimeUnit.MILLISECONDS); logger.debug( "id={} watchdog timer scheduled for +{}ms", state.subscriptionId, delay); }); } - private void notifyWatchdogTimerElapsed(long delay) { + private void notifyWatchdogTimerElapsed(long scheduledEpoch, long delay) { + synchronized (lock) { + if (cancelled || scheduledEpoch != epoch) { + // This expiry was cancelled or superseded while it was already running. + return; + } + + scheduledFuture = null; + } + SubscriptionListener listener = OpcUaSubscription.this.listener; if (listener != null) { @@ -1617,17 +2482,20 @@ private void notifyWatchdogTimerElapsed(long delay) { @Override public void onSessionActive(UaSession session) { - reset(); + // Deliberately not re-armed here: the PublishingManager holds Publish traffic shut until + // the Part 4 §6.7 Republish recovery for this activation has finished, so no + // PublishResponse — the only event that feeds this timer — can arrive yet. The timer is + // re-armed, via resetWatchdogTimer(), when publishing resumes. logger.debug( - "id={}, watchdog timer reset via onSessionActive()", + "id={}, watchdog timer awaiting reconnect recovery after onSessionActive()", getServerState().map(ServerState::getSubscriptionId).orElse(null)); } @Override public void onSessionInactive(UaSession session) { - cancel(); + pause(); logger.debug( - "id={}, watchdog timer cancelled via onSessionInactive()", + "id={}, watchdog timer paused via onSessionInactive()", getServerState().map(ServerState::getSubscriptionId).orElse(null)); } } @@ -1746,6 +2614,11 @@ default void onEventReceived( /** * Called when a Subscription receives a keep-alive notification from the Server. * + *

Take care not to block unnecessarily in this callback because subscription notifications + * are processed synchronously as a backpressure mechanism. Blocking inside this callback will + * prevent subsequent notifications from being processed and new PublishRequests from being + * sent. + * * @param subscription the Subscription that received the keep-alive notification. */ default void onKeepAliveReceived(OpcUaSubscription subscription) {} @@ -1783,6 +2656,11 @@ default void onWatchdogTimerElapsed(OpcUaSubscription subscription) {} *

  • Good_Transferred: the Subscription was transferred to another Session. * * + *

    Take care not to block unnecessarily in this callback because subscription notifications + * are processed synchronously as a backpressure mechanism. Blocking inside this callback will + * prevent subsequent notifications from being processed and new PublishRequests from being + * sent. + * * @param subscription the Subscription whose status has changed. * @param status the new status of the Subscription. */ diff --git a/opc-ua-sdk/sdk-client/src/main/java/org/eclipse/milo/opcua/sdk/client/subscriptions/PublishingManager.java b/opc-ua-sdk/sdk-client/src/main/java/org/eclipse/milo/opcua/sdk/client/subscriptions/PublishingManager.java index b040a99a5a..cd8b1e6d27 100644 --- a/opc-ua-sdk/sdk-client/src/main/java/org/eclipse/milo/opcua/sdk/client/subscriptions/PublishingManager.java +++ b/opc-ua-sdk/sdk-client/src/main/java/org/eclipse/milo/opcua/sdk/client/subscriptions/PublishingManager.java @@ -15,13 +15,19 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; import org.eclipse.milo.opcua.sdk.client.OpcUaClient; import org.eclipse.milo.opcua.sdk.client.OpcUaSession; import org.eclipse.milo.opcua.sdk.client.SessionActivityListener; @@ -39,66 +45,331 @@ import org.eclipse.milo.opcua.stack.core.types.structured.NotificationMessage; import org.eclipse.milo.opcua.stack.core.types.structured.PublishRequest; import org.eclipse.milo.opcua.stack.core.types.structured.PublishResponse; +import org.eclipse.milo.opcua.stack.core.types.structured.RepublishRequest; import org.eclipse.milo.opcua.stack.core.types.structured.RepublishResponse; import org.eclipse.milo.opcua.stack.core.types.structured.RequestHeader; import org.eclipse.milo.opcua.stack.core.types.structured.StatusChangeNotification; import org.eclipse.milo.opcua.stack.core.types.structured.SubscriptionAcknowledgement; import org.eclipse.milo.opcua.stack.core.util.TaskQueue; import org.eclipse.milo.opcua.stack.core.util.Unit; +import org.jspecify.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * Keeps a Session supplied with PublishRequests and dispatches the PublishResponses that answer + * them to the Subscriptions they belong to. + * + *

    Every method is safe to call from any thread. The work one PublishResponse generates is split + * across three execution contexts, and the Javadoc of each method below names the one it runs in: + * the transport's serial PublishResponse handler, which is where the order the Server sent + * NotificationMessages in still exists and where nothing may block; each Subscription's own serial + * processing queue, where sequence-number accounting and Republish recovery happen; and each + * Subscription's own serial delivery queue, where application callbacks are invoked. + * + *

    A registration ({@link SubscriptionDetails}) is bound to the SubscriptionId it was registered + * under and is never reused: {@link OpcUaSubscription#reset()} unregisters it, and a subsequent + * {@link OpcUaSubscription#create()} registers a new one, even though the same {@link + * OpcUaSubscription} object represents both. Part 4 §5.13.1.1 makes the SubscriptionId "the + * Server-assigned identifier for the Subscription", so a NotificationMessage received under one + * SubscriptionId is not a statement about any other, and work queued for one registration must + * never be applied to another. That is why nothing here asks the live Subscription object what its + * current SubscriptionId is: the id an entry is registered under is captured once, when the entry + * is created, and every question about identity is answered from it. + */ public class PublishingManager { + /** + * Upper bound on the number of missing NotificationMessages the client will try to recover from a + * PublishResponse when the Server does not tell it what it is holding, i.e. when the response + * carries no availableSequenceNumbers. + * + *

    Recovery is one Republish call per missing message, so the gap has to be bounded by + * something: a sequence number that is far ahead — because it is corrupt, or because it comes + * from a Subscription whose state the client has lost track of — would otherwise cost up to 2^32 + * round trips. When the Server does advertise availableSequenceNumbers, that list is the bound. + */ + private static final long DEFAULT_MAX_RECOVERABLE_GAP = 64L; + + /** + * The ceiling of a {@link PendingPublishCeiling} while the Server has never refused a + * PublishRequest for holding too many, i.e. while nothing but the client's own target applies. + */ + private static final long NO_PENDING_PUBLISH_CEILING = Long.MAX_VALUE; + + /** + * The number of successful Publish round trips that must pass before the client asks a Server + * that once refused a PublishRequest whether it would now queue one more. + * + *

    Counted in the Server's own answers rather than in elapsed time: a round trip is a + * PublishRequest the Server accepted, answered, and had the answer delivered from, which is + * exactly the evidence that the condition behind the refusal may have passed, and it needs no + * clock. + * + *

    Eight is chosen to be comfortably clear of the window Part 4 §5.14.5.1 is measured in — "one + * of its outstanding Publish requests is returned" buys exactly one replacement, so the immediate + * response to the fault is a property of the first returning request — while still being a small + * number of NotificationMessages: a Subscription publishing once a second reaches the first probe + * in under ten seconds. + */ + private static final long PROBE_COOLDOWN_BASE_SUCCESSES = 8L; + + /** + * The factor the cooldown grows by when a probe is refused. + * + *

    Doubling is what bounds the cost of leniency: a Server that always refuses is asked at + * roughly successes 8, 24, 56, 120, 248, ..., so the number of probes it sees grows only + * logarithmically in the number of responses it delivers, and the interval between two probes is + * never shorter than the interval before it. + */ + private static final long PROBE_COOLDOWN_GROWTH = 2L; + + /** + * The longest cooldown a repeatedly refused probe can grow to, in successful Publish round trips. + * + *

    A cap is needed because the alternative to a bounded backoff is one that gives up: without + * it the interval doubles until a Server whose cap really was raised is never asked again. At 512 + * a Subscription publishing once a second probes about every eight minutes in the worst case, + * which costs a Server that will never accept more essentially nothing and still finds a + * condition that has passed within minutes. + */ + private static final long PROBE_COOLDOWN_MAX_SUCCESSES = 512L; + private final Logger logger = LoggerFactory.getLogger(getClass()); private final ConcurrentMap pendingCountMap = new ConcurrentHashMap<>(); private final Map subscriptionDetails = new ConcurrentHashMap<>(); - private final TaskQueue processingQueue; + /** + * Incremented every time a Subscription is registered with or unregistered from this manager. + * + *

    Each PublishRequest records the value in effect when it was sent, which is what lets a + * failure be told apart from a failure that is still relevant: a Server's answer describes the + * Subscription set the request was sent for, and once that set has changed the answer no longer + * describes the client. + */ + private final AtomicLong subscriptionGeneration = new AtomicLong(0L); + + /** + * What the client has learned from Bad_TooManyPublishRequests about how many PublishRequests the + * Server will queue for this Session, and what it has to see before asking for one more. + */ + private final AtomicReference pendingPublishCeiling = + new AtomicReference<>(PendingPublishCeiling.none(0L)); + + /** + * The number of times a Session has become Active, i.e. the number of times the client has had to + * recover the Subscriptions it holds before resuming Publish traffic. + */ + private final AtomicLong sessionActivations = new AtomicLong(0L); + + /** + * The deepest Publish pipeline Milo has permitted for this client. + * + *

    Recorded when the target is chosen, before requests can fail and release their permits. A + * Session-inactive callback is dispatched asynchronously and may run only after those failures + * and concurrent Subscription removals have made both the outstanding count and current target + * smaller, so it is too late to reconstruct the depth the outage may have lost. Keeping the + * lifetime high-water mark can overestimate a later, shallower outage, but the first + * Bad_MessageNotAvailable still terminates recovery immediately and the safety bound remains + * finite. + */ + private final AtomicLong maxPermittedPublishPipelineDepth = new AtomicLong(0L); + + /** + * Whether a Session has ever become inactive, and therefore whether an activation is a + * re-activation that the Part 4 §6.7 Republish drain has a reason to run for. + * + *

    On the first activation the Session, and every Subscription registered against it, were + * created moments earlier on that same Session: the Server cannot be holding a + * NotificationMessage the client has not collected, so the drain can only spend a round trip per + * Subscription being told Bad_MessageNotAvailable. It is not merely wasteful — the drain holds + * the publish gate shut until it ends (see {@link #isPublishingAllowed(UaSession)}), so it also + * delays the first PublishRequest of every Subscription created before the {@code + * onSessionActive} callbacks run. That window is reachable: {@code connect()} returns when the + * Session future completes, which happens in a task submitted before the callback fan-out. + */ + private final AtomicBoolean sessionEverInactive = new AtomicBoolean(false); + + /** + * The newest Session activation whose recovery has finished, one way or the other, and the + * Session that recovery ran on. + * + *

    Publish traffic is allowed only while this describes the Session a request would be sent on; + * see {@link #isPublishingAllowed(UaSession)}. The activation must have caught up with {@link + * #sessionActivations} and the Session in hand must be the one recovered: an activation is + * only counted when the {@code onSessionActive} callbacks run, but the Session future completes + * in a separate task that can release callbacks parked on it first, and in that window the + * counters alone still describe the previous activation. The Session reference is cleared the + * moment its Session becomes inactive, because a re-activation can hand back the same Session + * object, and in the same window the reference alone would still match it. + * + *

    The pair is monotonic, and the recovery of the newest activation always finishes, so the + * pipeline cannot be held shut by a recovery that is over. + */ + private final AtomicReference lastRecoveredActivation = + new AtomicReference<>(new RecoveredActivation(0L, null)); private final OpcUaClient client; public PublishingManager(OpcUaClient client) { this.client = client; - processingQueue = new TaskQueue(client.getTransport().getConfig().getExecutor()); - // When a Session gets re-activated after a connection loss we need to make sure PublishRequests - // are being sent again. + // are being sent again -- but only after every Subscription has had the chance to collect what + // the Server generated while the Session was unusable. See recoverAndResumePublishing(). client.addSessionActivityListener( new SessionActivityListener() { @Override public void onSessionActive(UaSession session) { - maybeSendPublishRequests(); + recoverAndResumePublishing(session); + } + + @Override + public void onSessionInactive(UaSession session) { + // From here on an activation is a re-activation, and the Server may be holding + // NotificationMessages generated while the Session was unusable. + sessionEverInactive.set(true); + + // The Session is gone, and the recovery that ran for it says nothing about the next + // activation — which may hand back this very Session object, re-activated. Shut the + // gate here rather than when the next activation is counted: the next Session future + // can complete, releasing callbacks parked on it, before the activation callbacks run. + lastRecoveredActivation.getAndUpdate( + r -> r.session() == session ? new RecoveredActivation(r.activation(), null) : r); } }); } + /** + * Register {@code subscription} under the SubscriptionId it currently holds. + * + *

    Called by {@link OpcUaSubscription#create()} while it holds that Subscription's lifecycle + * lock, so the id read here is the one the Server just assigned and cannot be cleared by a + * concurrent {@link OpcUaSubscription#reset()} before the entry is created. + * + * @param subscription the Subscription to register. + */ void addSubscription(OpcUaSubscription subscription) { + Executor executor = client.getTransport().getConfig().getExecutor(); + subscription .getSubscriptionId() - .ifPresent(id -> subscriptionDetails.put(id, new SubscriptionDetails(subscription))); + .ifPresent( + id -> { + SubscriptionDetails displaced = + subscriptionDetails.put(id, new SubscriptionDetails(subscription, id, executor)); + + if (displaced != null) { + // The Server has reused this SubscriptionId while an entry was still registered + // under it. Displacement removed that entry from the map, so unregister() — which + // matches by (key, value) — can never succeed for it again; without this, work + // still queued for it would be applied as if its Subscription existed, forever. + displaced.registered = false; + } + + subscriptionGeneration.incrementAndGet(); + }); + + // The client wants a deeper pipeline than it did when any ceiling was learned, and Part 4 + // §5.14.5.1 requires a Server to accept at least one more queued PublishRequest per + // Subscription, so it is worth finding out whether this one now will — immediately, rather than + // after the cooldown a probe would have to serve. + pendingPublishCeiling.set(PendingPublishCeiling.none(sessionActivations.get())); maybeSendPublishRequests(); } + /** + * Unregister every entry held for {@code subscription}. + * + *

    Entries are matched by identity rather than by the Subscription's current SubscriptionId: + * the caller is discarding this Subscription, and an entry that outlived the id it was registered + * under would keep answering PublishResponses for a Subscription the client no longer has. + * + * @param subscription the Subscription to unregister. + */ void removeSubscription(OpcUaSubscription subscription) { - subscription.getSubscriptionId().ifPresent(subscriptionDetails::remove); + for (SubscriptionDetails details : subscriptionDetails.values()) { + if (details.subscription == subscription) { + unregister(details); + } + } maybeSendPublishRequests(); } + /** + * Remove {@code details} from the registry, if it is still the entry registered under its + * SubscriptionId, and mark it unregistered so that work already queued for it is discarded rather + * than applied to whatever Subscription exists by the time it runs. + * + * @param details the entry to unregister. + * @return {@code true} if {@code details} was still the registered entry and this call removed + * it; {@code false} if something else unregistered it first. + */ + private boolean unregister(SubscriptionDetails details) { + if (subscriptionDetails.remove(details.subscriptionId, details)) { + details.registered = false; + + subscriptionGeneration.incrementAndGet(); + + return true; + } + + return false; + } + + /** + * Record what a Server said it was still holding for {@code subscriptionId} when it accepted a + * TransferSubscriptions request for it. + * + *

    Part 4 §5.14.7.1 gives each successful TransferResult "the sequence numbers of the + * NotificationMessages that are available for retransmission". It is the exact input the + * reconnect recovery needs: it says which NotificationMessages the Republish loop of Part 4 §6.7 + * can still collect, and therefore where that loop starts and where it stops. + * + *

    Called by the Session FSM while the Session that transfer was part of is still on its way to + * Active, so the list is in place before the recovery that consumes it runs. + * + * @param session the Session the Subscription was transferred to. + * @param subscriptionId the SubscriptionId that was transferred. + * @param availableSequenceNumbers the availableSequenceNumbers of its TransferResult. + */ + public void notifySubscriptionTransferred( + UaSession session, UInteger subscriptionId, UInteger @Nullable [] availableSequenceNumbers) { + + SubscriptionDetails details = subscriptionDetails.get(subscriptionId); + + if (details != null) { + details.transferredSequenceNumbers.set( + new TransferredSequenceNumbers(session, availableSequenceNumbers)); + } + } + private void maybeSendPublishRequests() { long maxPendingPublishes = getMaxPendingPublishes(); if (maxPendingPublishes > 0) { + maxPermittedPublishPipelineDepth.accumulateAndGet(maxPendingPublishes, Math::max); + client .getSessionAsync() .whenComplete( (session, ex) -> { if (session != null) { + if (!isPublishingAllowed(session)) { + // The Republish loop Part 4 §6.7 requires before Publish resumes has not + // finished yet for this Session. Tested here, once the Session is in hand, + // rather than on the way in: a caller that found no Session is parked on the + // one being established, and must not send the instant it arrives either. + // Whatever deficit builds up while the pipeline is shut is made good by + // resumePublishing(), which every recovery reaches. + logger.debug("Publish suspended pending reconnect recovery"); + return; + } + AtomicLong pendingCount = pendingCountMap.computeIfAbsent( session.getSessionId(), id -> new AtomicLong(0L)); @@ -126,107 +397,861 @@ private void maybeSendPublishRequests() { } } - void sendPublishRequest(OpcUaSession session, AtomicLong pendingCount) { - var subscriptionAcknowledgements = new ArrayList(); - - subscriptionDetails - .values() - .forEach( - subscription -> { - synchronized (subscription.availableAcknowledgements) { - subscription.availableAcknowledgements.forEach( - sequenceNumber -> - subscription - .subscription - .getSubscriptionId() - .ifPresent( - subscriptionId -> - subscriptionAcknowledgements.add( - new SubscriptionAcknowledgement( - subscriptionId, sequenceNumber)))); - subscription.availableAcknowledgements.clear(); + /** + * @param session the Session a PublishRequest would be sent on. + * @return {@code true} if PublishRequests may be sent on {@code session}, i.e. if every Session + * activation so far has had its reconnect recovery run and {@code session} is the Session the + * newest of those recoveries ran on. + */ + private boolean isPublishingAllowed(UaSession session) { + RecoveredActivation recovered = lastRecoveredActivation.get(); + + return recovered.activation() >= sessionActivations.get() && recovered.session() == session; + } + + /** + * @return {@code true} if Publish traffic is suspended, i.e. no PublishResponse can arrive: + * either no recovered Session is in hand, or an activation has been counted whose reconnect + * recovery has not finished. The watchdog timer is fed only by PublishResponses, so it must + * not be armed while this is true — see {@link OpcUaSubscription#resetWatchdogTimer()} — and + * {@link #resumePublishing} re-arms it when the suspension ends. + */ + boolean isPublishingSuspended() { + RecoveredActivation recovered = lastRecoveredActivation.get(); + + return recovered.session() == null || recovered.activation() < sessionActivations.get(); + } + + /** + * Recover every registered Subscription on the newly Active {@code session} and only then let + * PublishRequests flow again. + * + *

    Part 4 §6.7: "After re-establishing the connection the Client shall call Republish in a + * loop, starting with the next expected sequence number and incrementing the sequence number + * until the Server returns the status Bad_MessageNotAvailable. After the Republish returns + * Bad_MessageNotAvailable the Client shall start sending Publish requests with the normal Publish + * handling. This sequence ensures that the lost NotificationMessages queued in the Server are not + * overwritten by new Publish responses." A Server's retransmission queue is finite — Part 4 + * §5.14.1.1: "In the case of a retransmission queue overflow, the oldest sent NotificationMessage + * gets deleted" — so every NotificationMessage sent in answer to a resumed PublishRequest can + * evict one the client has not collected yet, which is what the ordering prevents. + * + *

    Runs on the transport's executor, as one of the Session activity callbacks, and returns + * immediately: the recovery is a chain of Republish round trips and nothing here waits for it. + * + * @param session the Session that has just become Active. + */ + private void recoverAndResumePublishing(UaSession session) { + long activation = sessionActivations.incrementAndGet(); + + // A Session has its own queue of PublishRequests, so what an earlier one was willing to hold + // says nothing about this one — and neither does how reluctant it was to accept more. Tagging + // the reset with this activation makes it atomic with respect to a delayed refusal: an old + // failure either updates the old state before this set replaces it, or sees the new tag and is + // ignored. + pendingPublishCeiling.set(PendingPublishCeiling.none(activation)); + + try { + recoverSubscriptions(session, activation) + .whenComplete((unit, ex) -> resumePublishing(activation, session)); + } catch (Exception e) { + // Nothing is going to complete the recovery that never started, and a pipeline that stays + // shut is worse than one that resumes without having drained: it never recovers at all. + logger.error("Reconnect recovery could not be started", e); + + resumePublishing(activation, session); + } + } + + /** + * Let PublishRequests flow again now that the recovery of Session activation {@code activation} + * is over, and send the ones that were not sent while it ran. + * + * @param activation the {@link #sessionActivations} value the finished recovery belongs to. + * @param session the Session that recovery ran on. + */ + private void resumePublishing(long activation, UaSession session) { + lastRecoveredActivation.getAndUpdate( + r -> r.activation() >= activation ? r : new RecoveredActivation(activation, session)); + + // The watchdog watches for the Server going quiet, but no PublishResponse — the only event + // that feeds it — can arrive while Publish traffic is suspended, so it is re-armed only now, + // not the moment the Session became active: a recovery longer than the watchdog delay must + // not fire it on a Subscription whose recovery is proceeding normally. + subscriptionDetails.values().forEach(d -> d.subscription.resetWatchdogTimer()); + + maybeSendPublishRequests(); + } + + /** + * Run the Part 4 §6.7 Republish loop for every registered Subscription, all of them at once: they + * have independent sequence numbers, independent processing queues, and independent Republish + * round trips, so one Subscription's recovery is not a reason for another's to wait. + * + * @param session the Session that has just become Active. + * @param activation the {@link #sessionActivations} value this recovery belongs to. + * @return a {@link CompletableFuture} that completes, never exceptionally, when the last of them + * is done. + */ + private CompletableFuture recoverSubscriptions(UaSession session, long activation) { + List details = List.copyOf(subscriptionDetails.values()); + + if (details.isEmpty()) { + return CompletableFuture.completedFuture(Unit.VALUE); + } + + // A transfer names what the Server still holds, so a Subscription carrying that list has + // something to collect whatever else is true; see republishUntilUnavailable(). + boolean transferred = + details.stream() + .map(d -> d.transferredSequenceNumbers.get()) + .anyMatch(t -> t != null && t.session() == session); + + if (!sessionEverInactive.get() && !transferred) { + logger.debug( + "First Session activation, so nothing predates it: skipping recovery of {}" + + " Subscription(s)", + details.size()); + + return CompletableFuture.completedFuture(Unit.VALUE); + } + + var futures = new ArrayList>(details.size()); + + for (SubscriptionDetails d : details) { + futures.add(recoverSubscription(session, d, activation)); + } + + return CompletableFuture.allOf(futures.toArray(CompletableFuture[]::new)) + .handle((unit, ex) -> Unit.VALUE); + } + + /** + * Run the Part 4 §6.7 Republish loop for one Subscription. + * + *

    The loop runs with the Subscription's processing queue paused, exactly as the reactive gap + * repair in {@link #processPublishResponse} does, which is what orders the NotificationMessages + * it collects ahead of any PublishResponse for the same Subscription: none is processed, and + * therefore none is delivered, until the loop is done. + * + * @param session the Session that has just become Active. + * @param details the {@link SubscriptionDetails} for the Subscription to recover. + * @param activation the {@link #sessionActivations} value this recovery belongs to. + * @return a {@link CompletableFuture} that completes, never exceptionally, when the loop is done. + */ + private CompletableFuture recoverSubscription( + UaSession session, SubscriptionDetails details, long activation) { + + var recovered = new CompletableFuture(); + + boolean queued = + details.processingQueue.execute( + () -> { + // Safe from here because this task is running on the queue it pauses, so no other + // task for this Subscription can be in flight. + details.processingQueue.pause(); + + try { + republishUntilUnavailable(session, details, activation) + .whenComplete((unit, ex) -> finishRecovery(details, recovered)); + } catch (Exception e) { + logger.error( + "Reconnect recovery failed, subscriptionId={}", details.subscriptionId, e); + + finishRecovery(details, recovered); } }); - RequestHeader requestHeader = - client.newRequestHeader(session.getAuthenticationToken(), getTimeoutHint()); + if (!queued) { + // The queue is shut down or saturated, so the task above will never run and there is no loop + // to wait for. + recovered.complete(Unit.VALUE); + } - UInteger requestHandle = requestHeader.getRequestHandle(); + return recovered; + } - var request = - new PublishRequest( - requestHeader, - subscriptionAcknowledgements.toArray(new SubscriptionAcknowledgement[0])); - - if (logger.isDebugEnabled()) { - String[] ackStrings = - subscriptionAcknowledgements.stream() - .map( - ack -> - String.format( - "id=%s/seq=%s", ack.getSubscriptionId(), ack.getSequenceNumber())) - .toArray(String[]::new); + /** + * Hand a Subscription's processing queue back and report its recovery finished, whatever happened + * to it. Every branch of a recovery ends here: a paused queue that is never resumed stops the + * Subscription being delivered to, and a recovery that is never reported finished stops the whole + * client's Publish traffic. + * + * @param details the {@link SubscriptionDetails} for the Subscription that was recovered. + * @param recovered the {@link CompletableFuture} to complete. + */ + private static void finishRecovery( + SubscriptionDetails details, CompletableFuture recovered) { + + try { + details.processingQueue.resume(); + } finally { + recovered.complete(Unit.VALUE); + } + } - logger.debug( - "Sending PublishRequest, requestHandle={}, acknowledgements={}", - requestHandle, - Arrays.toString(ackStrings)); + /** + * Request, one at a time, the NotificationMessages the Server generated for one Subscription + * while the client could not collect them, starting at the sequence number expected next. + * + *

    Where the Session was replaced and the Subscription transferred to it, the TransferResult + * has already named the sequence numbers the Server still holds (Part 4 §5.14.7.1), and that list + * is what the loop follows: it starts at the oldest of them the client is missing and ends where + * the list does, without spending a round trip to be told about a sequence number the Server has + * already said it does not have. Anything older than that oldest one is gone — Part 4 §5.14.1.1 + * deletes the oldest NotificationMessage first — and is reported as lost data rather than + * requested. + * + *

    Where the Session was merely re-activated there is no such list, and the loop is the one + * Part 4 §6.7 describes: increment until the Server answers Bad_MessageNotAvailable. At most one + * NotificationMessage could have been sent for each PublishRequest Milo allowed outstanding + * before the disconnect, so that pipeline depth plus the terminating request is both sufficient + * to drain everything Milo could have left unanswered and a finite bound on the loop. + * + * @param session the Session that has just become Active. + * @param details the {@link SubscriptionDetails} for the Subscription to recover. + * @param activation the {@link #sessionActivations} value this recovery belongs to. + * @return a {@link CompletableFuture} that completes, never exceptionally, when the loop is done. + */ + private CompletableFuture republishUntilUnavailable( + UaSession session, SubscriptionDetails details, long activation) { + + TransferredSequenceNumbers transferred = takeTransferredSequenceNumbers(details, session); + UInteger[] advertised = transferred != null ? transferred.sequenceNumbers() : null; + + long expectedSequenceNumber = SequenceNumbers.successor(details.lastSequenceNumber); + long firstSequenceNumber = expectedSequenceNumber; + long maxRepublishes = getBlindReconnectRecoveryLimit(); + Set heldByServer = null; + + if (advertised != null && advertised.length > 0) { + heldByServer = legalSequenceNumbers(advertised); + maxRepublishes = heldByServer.size(); + + long oldestHeld = oldestHeldAtOrAfter(heldByServer, expectedSequenceNumber); + + if (oldestHeld == SequenceNumbers.NONE) { + logger.debug( + "Nothing the Server advertised on transfer is missing, subscriptionId={}, " + + "expectedSequenceNumber={}", + details.subscriptionId, + expectedSequenceNumber); + + return CompletableFuture.completedFuture(Unit.VALUE); + } + + if (oldestHeld != expectedSequenceNumber) { + logger.warn( + "The oldest NotificationMessage the Server can retransmit after the transfer is " + + "sequenceNumber={}, so the {} starting at sequenceNumber={} are gone; treating " + + "them as lost data, subscriptionId={}", + oldestHeld, + SequenceNumbers.forwardDistance(expectedSequenceNumber, oldestHeld), + expectedSequenceNumber, + details.subscriptionId); + + details.lastSequenceNumber = SequenceNumbers.predecessor(oldestHeld); + details.subscription.notifyNotificationDataLost(); + + firstSequenceNumber = oldestHeld; + } } - client - .sendRequestAsync(request) - .whenCompleteAsync( - (response, ex) -> { - if (response instanceof PublishResponse publishResponse) { - logger.debug( - "Received PublishResponse, requestHandle={}, sequenceNumber={}", - publishResponse.getResponseHeader().getRequestHandle(), - publishResponse.getNotificationMessage().getSequenceNumber()); + return republishNext( + session, details, firstSequenceNumber, maxRepublishes, heldByServer, activation); + } - UInteger subscriptionId = publishResponse.getSubscriptionId(); - SubscriptionDetails details = subscriptionDetails.get(subscriptionId); + /** + * Consume only the TransferResult produced for {@code session}. + * + *

    A recovery task can remain queued after its Session has been superseded. In that interval a + * replacement Session can transfer the same Subscription and publish its available sequence + * numbers. Matching by Session identity before the compare-and-set keeps the stale task from + * taking the replacement's input; leaving a non-matching value in place lets the replacement's + * own recovery consume it later. + */ + private static @Nullable TransferredSequenceNumbers takeTransferredSequenceNumbers( + SubscriptionDetails details, UaSession session) { + + while (true) { + TransferredSequenceNumbers transferred = details.transferredSequenceNumbers.get(); + + if (transferred == null || transferred.session() != session) { + return null; + } - if (details != null) { - details.subscription.resetWatchdogTimer(); - } + if (details.transferredSequenceNumbers.compareAndSet(transferred, null)) { + return transferred; + } + } + } - processingQueue.execute( - () -> processPublishResponse(publishResponse, pendingCount)); - } else { - StatusCode statusCode = - UaException.extract(ex).map(UaException::getStatusCode).orElse(StatusCode.BAD); + /** + * @return the most Republish requests a blind reconnect drain may make: one for every + * PublishRequest Milo permitted or actually had outstanding immediately before the outage, + * plus one to receive the Bad_MessageNotAvailable that terminates Part 4 §6.7's loop. Unlike + * a fixed default this covers a configured pipeline deeper than 64, while remaining bounded + * by the deepest pipeline this client has permitted. + */ + private long getBlindReconnectRecoveryLimit() { + long pipelineDepth = + Math.max(maxPermittedPublishPipelineDepth.get(), getNaturalMaxPendingPublishes()); + + return Math.max(1L, pipelineDepth + 1L); + } - pendingCount.getAndUpdate(p -> (p > 0) ? p - 1 : 0); + /** + * Request the NotificationMessage with {@code sequenceNumber} and, if it arrives, the one after + * it. + * + * @param session the Session that has just become Active. + * @param details the {@link SubscriptionDetails} for the Subscription being recovered. + * @param sequenceNumber the sequence number to request. + * @param maxRepublishes the number of requests, including this one, the loop may still make. + * @param heldByServer the sequence numbers the Server said it holds, or {@code null} if it did + * not say and the loop has to run until it answers Bad_MessageNotAvailable. + * @param activation the {@link #sessionActivations} value this recovery belongs to. + * @return a {@link CompletableFuture} that completes, never exceptionally, when the loop is done. + */ + private CompletableFuture republishNext( + UaSession session, + SubscriptionDetails details, + long sequenceNumber, + long maxRepublishes, + @Nullable Set heldByServer, + long activation) { + + if (maxRepublishes <= 0 + || !details.registered + || activation != sessionActivations.get() + || (heldByServer != null && !heldByServer.contains(sequenceNumber))) { + + // Either the loop has run its course, or the Subscription is gone, or the Session this + // recovery was for has been superseded by one with a recovery of its own. + return CompletableFuture.completedFuture(Unit.VALUE); + } - long code = statusCode.value(); + return republish(session, details.subscriptionId, uint(sequenceNumber)) + .handle( + (response, ex) -> { + if (!details.registered || activation != sessionActivations.get()) { + // The response belongs to a recovery that a later Session activation superseded. + // It cannot update sequence accounting or deliver a NotificationMessage for the + // replacement activation. + return false; + } - if (code == StatusCodes.Bad_SessionClosed - || code == StatusCodes.Bad_SessionIdInvalid) { - subscriptionDetails.values().forEach(d -> d.subscription.cancelWatchdogTimer()); - } else if (code != StatusCodes.Bad_NoSubscription - && code != StatusCodes.Bad_TooManyPublishRequests) { + if (ex != null) { + StatusCode statusCode = + UaException.extract(ex).map(UaException::getStatusCode).orElse(StatusCode.BAD); - maybeSendPublishRequests(); + if (statusCode.value() == StatusCodes.Bad_MessageNotAvailable) { + // The termination condition of Part 4 §6.7's loop, and not lost data: the Server + // is holding nothing with this sequence number because it never sent one, which + // is the normal case when nothing was missed. + logger.debug( + "Reconnect recovery complete, subscriptionId={}, sequenceNumber={} is not " + + "available for retransmission", + details.subscriptionId, + sequenceNumber); + } else { + logger.warn( + "Republish service failure during reconnect recovery, subscriptionId={}, " + + "sequenceNumber={}: {}", + details.subscriptionId, + sequenceNumber, + statusCode); } - logger.debug( - "Publish service failure (requestHandle={}): {}", - requestHandle, - statusCode, - ex); + return false; } - }, - client.getTransport().getConfig().getExecutor()); + + // The processing queue is paused for the duration of the loop, so nothing else is + // accounting for this Subscription's sequence numbers and nothing else is queueing + // deliveries for it. Advancing lastSequenceNumber here is what stops the first + // PublishResponse after the recovery from finding a gap the recovery has just closed + // and delivering these NotificationMessages a second time. + details.availableAcknowledgements.add(uint(sequenceNumber)); + details.lastSequenceNumber = sequenceNumber; + + NotificationMessage notificationMessage = response.getNotificationMessage(); + + details + .subscription + .getDeliveryQueue() + .execute(() -> deliverNotificationMessage(details, notificationMessage)); + + return true; + }) + .thenCompose( + recovered -> + recovered + ? republishNext( + session, + details, + SequenceNumbers.successor(sequenceNumber), + maxRepublishes - 1, + heldByServer, + activation) + : CompletableFuture.completedFuture(Unit.VALUE)); } - private void processPublishResponse(PublishResponse response, AtomicLong pendingCount) { - UInteger subscriptionId = response.getSubscriptionId(); + /** + * Call the Republish service on {@code session}. + * + *

    Bound to the Session the recovery is for, rather than made through {@link + * OpcUaClient#republishAsync}, which resolves the Session again for every call: a recovery that + * outlives the Session it began on must not have its remaining requests silently re-issued on the + * next one, which has a recovery of its own, and must never park waiting for a Session that does + * not exist yet — a request parked there is a recovery that never finishes and a Publish pipeline + * that never reopens. + * + * @param session the Session to send the request on. + * @param subscriptionId the SubscriptionId to request a NotificationMessage of. + * @param sequenceNumber the sequence number to request. + * @return a {@link CompletableFuture} that completes with the {@link RepublishResponse}. + */ + private CompletableFuture republish( + UaSession session, UInteger subscriptionId, UInteger sequenceNumber) { - SubscriptionDetails details = subscriptionDetails.get(subscriptionId); + var request = + new RepublishRequest( + client.newRequestHeader(session.getAuthenticationToken()), + subscriptionId, + sequenceNumber); + + return client.sendRequestAsync(request).thenApply(RepublishResponse.class::cast); + } + + /** + * @param sequenceNumbers the availableSequenceNumbers of a TransferResult. + * @return the legal sequence numbers among them. + */ + private static Set legalSequenceNumbers(UInteger[] sequenceNumbers) { + var legal = new HashSet(sequenceNumbers.length); + + for (UInteger sequenceNumber : sequenceNumbers) { + if (sequenceNumber != null && SequenceNumbers.isLegal(sequenceNumber.longValue())) { + legal.add(sequenceNumber.longValue()); + } + } + + return legal; + } + + /** + * @param heldByServer the sequence numbers the Server said it holds. + * @param expectedSequenceNumber the sequence number the client expects next. + * @return the oldest of {@code heldByServer} that the client has not accounted for, or {@link + * SequenceNumbers#NONE} if the Server holds nothing the client is missing. + */ + private static long oldestHeldAtOrAfter(Set heldByServer, long expectedSequenceNumber) { + long oldest = SequenceNumbers.NONE; + long oldestDistance = Long.MAX_VALUE; + + for (long sequenceNumber : heldByServer) { + if (sequenceNumber != expectedSequenceNumber + && !SequenceNumbers.isAhead(sequenceNumber, expectedSequenceNumber)) { + // Already accounted for: received, recovered, or given up on. + continue; + } + + long distance = SequenceNumbers.forwardDistance(expectedSequenceNumber, sequenceNumber); + + if (distance < oldestDistance) { + oldestDistance = distance; + oldest = sequenceNumber; + } + } + + return oldest; + } + + void sendPublishRequest(OpcUaSession session, AtomicLong pendingCount) { + // Acknowledgements are removed from the Subscription's queue as they are drained into this + // request, so this request now owns them: if it fails they have to be put back, or the client + // has silently decided never to acknowledge those NotificationMessages. + var drainedAcknowledgements = new ArrayList(); + + // Read before the request is built, so that a Subscription registered or unregistered while it + // is in flight is seen as a change by the failure handler rather than missed. + long generation = subscriptionGeneration.get(); + + // Read for the same reason: a Session-level failure describes the Session the request was sent + // on, and once another activation has been counted it no longer describes the client. + long activation = sessionActivations.get(); + + try { + var subscriptionAcknowledgements = new ArrayList(); + + for (SubscriptionDetails details : subscriptionDetails.values()) { + List sequenceNumbers; + + synchronized (details.availableAcknowledgements) { + if (details.availableAcknowledgements.isEmpty()) { + continue; + } + + sequenceNumbers = List.copyOf(details.availableAcknowledgements); + details.availableAcknowledgements.clear(); + } + + for (UInteger sequenceNumber : sequenceNumbers) { + // Part 4 §5.14.5.2 pairs a sequenceNumber with the subscriptionId of the Subscription the + // NotificationMessage was "received on", and lets the Server delete the message with that + // sequence number from that Subscription's retransmission queue. The id the entry is + // registered under is that Subscription; the live object's current id may by now be a + // different one, under which these sequence numbers were never sent. + subscriptionAcknowledgements.add( + new SubscriptionAcknowledgement(details.subscriptionId, sequenceNumber)); + } + + drainedAcknowledgements.add(new DrainedAcknowledgements(details, sequenceNumbers)); + } + + RequestHeader requestHeader = + client.newRequestHeader(session.getAuthenticationToken(), getTimeoutHint()); + + UInteger requestHandle = requestHeader.getRequestHandle(); + + var request = + new PublishRequest( + requestHeader, + subscriptionAcknowledgements.toArray(new SubscriptionAcknowledgement[0])); + + if (logger.isDebugEnabled()) { + String[] ackStrings = + subscriptionAcknowledgements.stream() + .map( + ack -> + String.format( + "id=%s/seq=%s", ack.getSubscriptionId(), ack.getSequenceNumber())) + .toArray(String[]::new); + + logger.debug( + "Sending PublishRequest, requestHandle={}, acknowledgements={}", + requestHandle, + Arrays.toString(ackStrings)); + } + + client + .sendRequestAsync(request) + .whenComplete( + (response, ex) -> { + if (response instanceof PublishResponse publishResponse) { + // This handler runs inline on the transport's serial PublishResponse queue (see + // AbstractUascClientTransport#handleResponse), which is the only place the order + // the Server sent NotificationMessages in still exists. Queueing the work here, + // rather than hopping through the general-purpose executor first, is what carries + // that order into the Subscription's processing queue, which is itself serial. + // Nothing that can block belongs in this handler. + logger.debug( + "Received PublishResponse, requestHandle={}, sequenceNumber={}", + publishResponse.getResponseHeader().getRequestHandle(), + publishResponse.getNotificationMessage().getSequenceNumber()); + + reportRefusedAcknowledgements(subscriptionAcknowledgements, publishResponse); + + UInteger subscriptionId = publishResponse.getSubscriptionId(); + SubscriptionDetails details = subscriptionDetails.get(subscriptionId); + + boolean queued = false; + + if (details != null) { + // Cheap and non-blocking: cancels and re-schedules a timer. The watchdog + // watches for the Server going quiet, so it is reset when the response is + // received rather than when it is eventually processed. + details.subscription.resetWatchdogTimer(); + + // The entry resolved here rides along with the task: by the time the task + // runs, the entry registered under this SubscriptionId may be a different + // one, and this task was queued for this one. + queued = + details.processingQueue.execute( + () -> + processPublishResponse( + details, publishResponse, pendingCount, activation)); + } + + if (!queued) { + // Nothing to process — no entry, or a queue already shut down — but the + // permit still has to be released, and doing it here would re-enter + // sendPublishRequest() on this thread. + client + .getTransport() + .getConfig() + .getExecutor() + .execute(() -> releasePendingPublish(pendingCount, activation)); + } + } else { + // The failure path is dispatched asynchronously: it may run on a wheel timer + // thread (request timeout) or inline on the caller's thread (a request that + // fails before it is sent), and it re-enters maybeSendPublishRequests(), which + // would otherwise recurse into sendPublishRequest() on that same thread. + client + .getTransport() + .getConfig() + .getExecutor() + .execute( + () -> + handlePublishFailure( + ex, + requestHandle, + pendingCount, + drainedAcknowledgements, + generation, + activation)); + } + }); + } catch (Exception e) { + // The caller took a pending-publish permit before invoking this method. If building or + // sending the request fails synchronously no completion handler will ever run, so release + // the permit here; otherwise it leaks and Publish traffic eventually stops for good. The + // acknowledgements already drained are in the same position: nothing will put them back + // unless it happens here. + restoreAcknowledgements(drainedAcknowledgements); - if (details == null) { pendingCount.getAndUpdate(p -> (p > 0) ? p - 1 : 0); + + logger.error("Error sending PublishRequest", e); + } + } + + /** + * Report every acknowledgement the Server refused in {@code response}. + * + *

    Part 4 §5.14.5.2 gives the PublishResponse a "List of results for the acknowledgements", + * whose "size and order... matches the size and order of the subscriptionAcknowledgements request + * parameter", so the result at index {@code i} belongs to the acknowledgement at index {@code i} + * of the request that produced it. + * + *

    A refused acknowledgement is reported and then forgotten, never re-queued. The reasons a + * Server refuses one are statements that there is nothing left to acknowledge — + * Bad_SequenceNumberUnknown means it is not holding a NotificationMessage with that sequence + * number, Bad_SubscriptionIdInvalid that the Subscription itself is gone — so repeating the + * acknowledgement could only be refused the same way, forever. + * + *

    Runs inline on the transport's PublishResponse handler and performs no I/O. + * + * @param acknowledgements the acknowledgements carried by the PublishRequest, in request order. + * @param response the {@link PublishResponse} that answered it. + */ + private void reportRefusedAcknowledgements( + List acknowledgements, PublishResponse response) { + + StatusCode[] results = response.getResults(); + + if (acknowledgements.isEmpty() || results == null || results.length == 0) { + return; + } + + int count = Math.min(results.length, acknowledgements.size()); + + if (results.length != acknowledgements.size()) { + logger.debug( + "PublishResponse carried {} acknowledgement result(s) for a PublishRequest with {} " + + "acknowledgement(s); pairing the first {}", + results.length, + acknowledgements.size(), + count); + } + + for (int i = 0; i < count; i++) { + StatusCode result = results[i]; + + if (result != null && result.isBad()) { + SubscriptionAcknowledgement acknowledgement = acknowledgements.get(i); + + logger.warn( + "Server refused SubscriptionAcknowledgement, subscriptionId={}, sequenceNumber={}: {}", + acknowledgement.getSubscriptionId(), + acknowledgement.getSequenceNumber(), + result); + } + } + } + + /** + * Put back the acknowledgements drained into a PublishRequest that failed, so that a later + * PublishRequest carries them. + * + *

    They go back at the head of the queue, ahead of anything queued since, because they are + * older. A sequence number that is queued again already — a duplicate NotificationMessage can + * re-queue one while the request carrying the first acknowledgement is still in flight — is not + * added a second time. + * + * @param drainedAcknowledgements what the failed request drained, per Subscription. + */ + private static void restoreAcknowledgements( + List drainedAcknowledgements) { + + for (DrainedAcknowledgements drained : drainedAcknowledgements) { + List availableAcknowledgements = drained.details().availableAcknowledgements; + + synchronized (availableAcknowledgements) { + var restored = new ArrayList(drained.sequenceNumbers().size()); + + for (UInteger sequenceNumber : drained.sequenceNumbers()) { + if (!availableAcknowledgements.contains(sequenceNumber)) { + restored.add(sequenceNumber); + } + } + + availableAcknowledgements.addAll(0, restored); + } + } + } + + /** + * Handle a PublishRequest that failed rather than returning a PublishResponse. + * + * @param ex the failure. + * @param requestHandle the requestHandle of the PublishRequest that failed. + * @param pendingCount the pending-publish permits held for the Session the request was sent on. + * @param drainedAcknowledgements the acknowledgements the failed request was carrying. + * @param generation the {@link #subscriptionGeneration} in effect when the request was sent. + * @param activation the {@link #sessionActivations} in effect when the request was sent. + */ + private void handlePublishFailure( + Throwable ex, + UInteger requestHandle, + AtomicLong pendingCount, + List drainedAcknowledgements, + long generation, + long activation) { + + // The acknowledgements went down with the request. Part 4 §5.14.7.1: "The Client should + // acknowledge all Messages in this list for which it will not request retransmission" — and + // these are messages the client has and will not request again, so abandoning the + // acknowledgement leaves the Server holding them, and re-advertising them in + // availableSequenceNumbers, until its retransmission queue evicts them. Restoring them here, + // before the permit is released, is what puts them on the next PublishRequest. + restoreAcknowledgements(drainedAcknowledgements); + + // A null failure means the request completed with a response that was not a PublishResponse; + // there is no exception to extract a StatusCode from, but the permit release and the refill + // below still have to happen. + StatusCode statusCode = + ex == null + ? StatusCode.BAD + : UaException.extract(ex).map(UaException::getStatusCode).orElse(StatusCode.BAD); + + long outstanding = pendingCount.updateAndGet(p -> (p > 0) ? p - 1 : 0); + + long code = statusCode.value(); + + if (code == StatusCodes.Bad_SessionClosed || code == StatusCodes.Bad_SessionIdInvalid) { + // The Session is gone, not the Subscription: no PublishResponse can arrive until the Session + // is re-activated, so the watchdog must be suspended rather than destroyed. The Session FSM + // treats both codes as Session faults and reconnects, after which TransferSubscriptions may + // well keep the Subscription alive; cancelling here would de-register the watchdog's + // SessionActivityListener and leave it unable to ever arm again. Only while the answer still + // describes this client, though: a straggling failure from a Session that a later activation + // has already replaced must not disarm the watchdogs that activation's recovery re-armed. + if (activation == sessionActivations.get()) { + subscriptionDetails.values().forEach(d -> d.subscription.pauseWatchdogTimer()); + } + } else if (code == StatusCodes.Bad_NoSubscription) { + // Part 4 §5.14.8.1: when the last Subscription of a Session is deleted, "all Publish requests + // still queued for that Session are de-queued and shall be returned with Bad_NoSubscription". + // Replacing a request the Server answered that way with another one for the same Subscription + // set could only be answered the same way, so the refill is suppressed — but only while the + // answer still describes this client. A Subscription registered or unregistered since the + // request was sent makes it stale: an application that deletes its last Subscription and + // immediately creates another takes the whole de-queued burst *after* addSubscription() has + // run, and addSubscription() cannot start Publish traffic for the new Subscription because + // the de-queued requests still hold every pending-publish permit. Suppressing the refill here + // as well would leave the new Subscription with no PublishRequest ever sent for it: no other + // caller of maybeSendPublishRequests() is left to run, and the watchdog only reports the + // silence, it does not recover from it. + if (generation != subscriptionGeneration.get()) { + maybeSendPublishRequests(); + } + } else if (code == StatusCodes.Bad_TooManyPublishRequests) { + // Part 4 §5.14.5.1: after this error a Client "shall not issue another Publish request before + // one of its outstanding Publish requests is returned". Not replacing this one is the letter + // of that; remembering how many the Server was in fact holding is what stops the next + // PublishResponse to be delivered from restoring the very outstanding count that drew the + // fault, since the target would otherwise still be subscriptionCount + 1 and the refill loop + // closes the whole deficit at once. + // + // Never below one: a ceiling of zero is a pipeline that can never refill. The same clause + // requires a Server to accept at least subscriptionCount + 1 queued Publish requests, which + // is exactly what the client aims for, so a conformant Server never gets here. + // + // The ceiling is not permanent — see maybeRaisePendingPublishCeiling(long) — and this is + // where a probe that was refused is charged for: the ceiling drops back and the next probe + // costs geometrically more successful Publish round trips than the one that was refused. + long ceiling = Math.max(1L, outstanding); + + PendingPublishCeiling before = + pendingPublishCeiling.getAndUpdate( + c -> c.activation() == activation ? c.refused(ceiling) : c); + + if (before.activation() != activation) { + logger.debug( + "Ignoring Bad_TooManyPublishRequests from superseded Session activation {} " + + "(current={})", + activation, + before.activation()); + } else { + if (before.probing()) { + PendingPublishCeiling after = before.refused(ceiling); + + logger.debug( + "Server refused the probe raising the pending Publish ceiling to {}; the ceiling is " + + "{} again and the next probe costs {} successful Publish round trips", + before.ceiling(), + after.ceiling(), + after.cooldown()); + } + + if (outstanding == 0) { + // This failure was itself the last outstanding request being returned, so the clause's + // condition for issuing another is met — and nothing else is left in flight whose + // completion would refill the pipeline. Without this the pipeline stays empty until an + // unrelated event (a Subscription added or a Session re-activated) restarts it. + maybeSendPublishRequests(); + } + } + } else { maybeSendPublishRequests(); + } + + logger.debug("Publish service failure (requestHandle={}): {}", requestHandle, statusCode, ex); + } + + /** + * Process a PublishResponse. + * + *

    Runs on the Subscription's own processing queue, which is serial, so the sequence-number + * accounting below needs no further synchronization and sees PublishResponses in the order the + * Server sent them. Nothing here may block: the queue runs on the transport's executor, which is + * also what completes the responses to any request this method might make. + * + * @param details the entry the response was routed to when it was received. Not looked up again + * here: the entry now registered under the response's SubscriptionId may be a different one — + * the Server can assign a new Subscription an id it has used before — and work queued for one + * registration must never be applied to another. + * @param response the {@link PublishResponse} to process. + * @param pendingCount the pending-publish permits held for the Session the request was sent on. + * @param activation the Session activation the request was sent on. + */ + private void processPublishResponse( + SubscriptionDetails details, + PublishResponse response, + AtomicLong pendingCount, + long activation) { + + UInteger subscriptionId = details.subscriptionId; + + if (!details.registered) { + logger.debug( + "Discarding PublishResponse for a Subscription that no longer exists, " + + "subscriptionId={}", + subscriptionId); + + releasePendingPublish(pendingCount, activation); return; } @@ -237,64 +1262,140 @@ private void processPublishResponse(PublishResponse response, AtomicLong pending || notificationMessage.getNotificationData().length == 0; long receivedSequenceNumber = notificationMessage.getSequenceNumber().longValue(); - long expectedSequenceNumber = details.lastSequenceNumber + 1; logger.debug( "Processing PublishResponse, subscriptionId={}, isKeepAlive={}, " - + "lastSequenceNumber={}, receivedSequenceNumber={}, expectedSequenceNumber={}", + + "lastSequenceNumber={}, receivedSequenceNumber={}", subscriptionId, isKeepAlive, details.lastSequenceNumber, - receivedSequenceNumber, - expectedSequenceNumber); - - if (receivedSequenceNumber > expectedSequenceNumber) { - boolean republishSuccess = true; - - for (long sequenceNumber = expectedSequenceNumber; - sequenceNumber < receivedSequenceNumber; - sequenceNumber++) { - - try { - RepublishResponse republishResponse = - client.republish(subscriptionId, uint(sequenceNumber)); - - NotificationMessage republishNotificationMessage = - republishResponse.getNotificationMessage(); - - details - .subscription - .getDeliveryQueue() - .execute(() -> deliverNotificationMessage(details, republishNotificationMessage)); - } catch (UaException e) { - logger.warn("Republish service failure, sequenceNumber={}", sequenceNumber, e); - - republishSuccess = false; + receivedSequenceNumber); + + List missingSequenceNumbers = List.of(); + + if (SequenceNumbers.isLegal(receivedSequenceNumber)) { + if (!isKeepAlive) { + // Acknowledge only NotificationMessages that were actually received: Part 4 §5.14.5.2 lets + // the Server delete an acknowledged message from its retransmission queue, so + // acknowledging one that never arrived destroys the only copy of it. A message that has + // already been accounted for was still received, so it is still acknowledged — but only + // once: a duplicate copy arriving before the first acknowledgement has been drained must + // not queue a second one, which the Server could only answer Bad_SequenceNumberUnknown. + UInteger sequenceNumber = notificationMessage.getSequenceNumber(); + + synchronized (details.availableAcknowledgements) { + if (!details.availableAcknowledgements.contains(sequenceNumber)) { + details.availableAcknowledgements.add(sequenceNumber); + } } } - if (!republishSuccess) { - details.subscription.notifyNotificationDataLost(); + long expectedSequenceNumber = SequenceNumbers.successor(details.lastSequenceNumber); + + if (receivedSequenceNumber != expectedSequenceNumber + && !SequenceNumbers.isAhead(receivedSequenceNumber, expectedSequenceNumber)) { + + long backwardDistance = + SequenceNumbers.forwardDistance(receivedSequenceNumber, expectedSequenceNumber); + + UInteger[] availableSequenceNumbers = response.getAvailableSequenceNumbers(); + + long duplicateWindow = + Math.max( + DEFAULT_MAX_RECOVERABLE_GAP, + availableSequenceNumbers == null ? 0 : availableSequenceNumbers.length); + + if (backwardDistance <= duplicateWindow) { + // Neither the message expected next nor ahead of it, so it has already been accounted + // for: a duplicate, or a message the client recovered via Republish before this copy of + // it arrived. Accounting must only ever move forwards; processing this message again + // would hand it to the application a second time and roll lastSequenceNumber back, + // making every message already received after it look missing and provoking a Republish + // for each one. + logger.debug( + "Discarding NotificationMessage already accounted for, subscriptionId={}, " + + "lastSequenceNumber={}, receivedSequenceNumber={}", + subscriptionId, + details.lastSequenceNumber, + receivedSequenceNumber); + + releasePendingPublish(pendingCount, activation); + return; + } + + // Too far behind to be a duplicate of anything recently recovered or retransmitted: the + // Server's numbering has regressed, e.g. because it restarted and renumbered a restored + // Subscription. Deliver the message and resynchronize the accounting to it — discarding + // instead would silently drop every NotificationMessage until the Server's numbering + // catches back up to where it used to be. + logger.warn( + "NotificationMessage sequence number regressed by {}; resynchronizing to it, " + + "subscriptionId={}, lastSequenceNumber={}, receivedSequenceNumber={}", + backwardDistance, + subscriptionId, + details.lastSequenceNumber, + receivedSequenceNumber); } - details.lastSequenceNumber = expectedSequenceNumber; - } + missingSequenceNumbers = missingSequenceNumbers(details, response, receivedSequenceNumber); - if (receivedSequenceNumber == 1 || !isKeepAlive) { - // Set the last sequence number only if either: - // - this was the first PublishResponse received - // - this *is not* a keep-alive PublishResponse - details.lastSequenceNumber = receivedSequenceNumber; + // Part 4 §5.14.1.1: a keep-alive "contains the sequence number of the next + // NotificationMessage that is to be sent", so it accounts for everything up to that sequence + // number's predecessor and is *not* evidence that the sequence number it carries was + // received. A data message accounts for itself. + details.lastSequenceNumber = + isKeepAlive + ? SequenceNumbers.predecessor(receivedSequenceNumber) + : receivedSequenceNumber; + } else { + // Part 4 §5.14.1.1: "The value 0 is never used for the sequence number." There is no sequence + // arithmetic that can be done with an illegal value, so deliver the message but leave the + // sequence accounting untouched. + logger.warn( + "Received NotificationMessage with illegal sequenceNumber={}, subscriptionId={}", + receivedSequenceNumber, + subscriptionId); } - UInteger[] availableSequenceNumbers = response.getAvailableSequenceNumbers(); - if (availableSequenceNumbers != null && availableSequenceNumbers.length > 0) { - synchronized (details.availableAcknowledgements) { - details.availableAcknowledgements.clear(); - - Collections.addAll(details.availableAcknowledgements, availableSequenceNumbers); - } + if (missingSequenceNumbers.isEmpty()) { + deliverAndReleasePendingPublish(details, notificationMessage, pendingCount, activation); + } else { + // Recovery is a Republish round trip per missing NotificationMessage and must not be waited + // for here. Pausing this Subscription's processing queue is what keeps it ordered: no later + // PublishResponse for this Subscription is processed — and, crucially, none is delivered — + // until every recovered NotificationMessage, and then the one that revealed the gap, has + // been handed to the delivery queue. Other Subscriptions have their own queues and are + // unaffected. pause() is safe to call from here because this task is running on the queue it + // pauses, so no other task for this Subscription can be in flight. + details.processingQueue.pause(); + + republishMissingNotificationMessages(details, subscriptionId, missingSequenceNumbers) + .whenComplete( + (unit, ex) -> { + try { + deliverAndReleasePendingPublish( + details, notificationMessage, pendingCount, activation); + } finally { + details.processingQueue.resume(); + } + }); } + } + + /** + * Deliver {@code notificationMessage} to the application and release the pending-publish permit + * once it has been delivered. + * + * @param details the {@link SubscriptionDetails} for the Subscription the message belongs to. + * @param notificationMessage the {@link NotificationMessage} to deliver. + * @param pendingCount the pending-publish permits held for the Session the request was sent on. + * @param activation the Session activation the request was sent on. + */ + private void deliverAndReleasePendingPublish( + SubscriptionDetails details, + NotificationMessage notificationMessage, + AtomicLong pendingCount, + long activation) { CompletionStage callback = details @@ -314,16 +1415,438 @@ private void processPublishResponse(PublishResponse response, AtomicLong pending "Notification delivery threw an unexpected Exception: {}", ex.getMessage(), ex); } - pendingCount.getAndUpdate(p -> (p > 0) ? p - 1 : 0); - - maybeSendPublishRequests(); + releasePendingPublish(pendingCount, activation); }, client.getTransport().getConfig().getExecutor()); + } else { + // The delivery queue is shut down, so the message will never be delivered — but the permit + // still has to be released, or the pipeline permanently shrinks by one. + releasePendingPublish(pendingCount, activation); + } + } + + /** + * Release the pending-publish permit taken for a PublishRequest whose response has been dealt + * with, and send a replacement PublishRequest if one is wanted. + * + * @param pendingCount the pending-publish permits held for the Session the request was sent on. + * @param activation the Session activation the request was sent on. + */ + private void releasePendingPublish(AtomicLong pendingCount, long activation) { + pendingCount.getAndUpdate(p -> (p > 0) ? p - 1 : 0); + + // This is the one place that knows a PublishRequest went out and came back without being + // refused, so it is where a ceiling learned from Bad_TooManyPublishRequests is paid off. Done + // before the refill below, so a raise takes effect in the very refill this return pays for. + maybeRaisePendingPublishCeiling(activation); + + maybeSendPublishRequests(); + } + + /** + * Count one successful Publish round trip against any ceiling learned from + * Bad_TooManyPublishRequests, and raise that ceiling by one if enough of them have gone by. + * + *

    Part 4 §5.14.5.1 says what a Client must do the moment a Server refuses a PublishRequest for + * holding too many, and nothing about how long it must go on believing it. The condition behind + * the refusal is often momentary — a queue briefly full, a cap raised again a second later — and + * the only two events that clear the ceiling outright, a Subscription being added and a Session + * being activated, need never happen again in the life of a long-lived client. Left alone, one + * transient refusal costs such a client a permanently shallower pipeline, i.e. throughput, for no + * reason. + * + *

    So the ceiling recovers, under three rules that together keep leniency from turning into + * hammering: + * + *

      + *
    • it recovers by one request at a time, never straight back to the target: restoring + * the whole deficit at once would send the Server exactly the burst that drew the fault, + * which is what the ceiling exists to prevent; + *
    • each raise costs a cooldown of successful Publish round trips — {@link + * #PROBE_COOLDOWN_BASE_SUCCESSES} of them to begin with — which is also what keeps a single + * probe in flight: the one extra request a raise adds is answered long before another whole + * cooldown of requests has been; + *
    • a probe the Server refuses lengthens the next cooldown geometrically, up to {@link + * #PROBE_COOLDOWN_MAX_SUCCESSES}. A Server that always refuses therefore sees a number of + * probes growing only logarithmically in the number of responses it delivers. + *
    + * + *

    Counting the Server's own answers, rather than watching a clock, is what makes the schedule + * cost the Server nothing when it has nothing to say: a Subscription that is publishing pays for + * its probes, and one that is silent — or a Session whose Publish pipeline is stalled — does not + * probe at all. + * + *

    Runs on the transport's executor, as part of releasing a pending-publish permit, and + * performs no I/O. + */ + private void maybeRaisePendingPublishCeiling(long activation) { + long natural = getNaturalMaxPendingPublishes(); + + PendingPublishCeiling before = + pendingPublishCeiling.getAndUpdate( + c -> + c.activation() == activation && c.ceiling() != NO_PENDING_PUBLISH_CEILING + ? c.roundTrip(natural) + : c); + + if (before.activation() != activation || before.ceiling() == NO_PENDING_PUBLISH_CEILING) { + // Either this response belongs to a superseded Session or no ceiling has been learned, so + // there is nothing for this round trip to pay down. + return; + } + + // roundTrip() is a function of the state it is applied to, so the transition this call made is + // the one from the state it replaced, whatever any concurrent update has done since. + PendingPublishCeiling after = before.roundTrip(natural); + + if (after.ceiling() > before.ceiling()) { + logger.debug( + "Probing whether the Server will now queue {} PublishRequests, after {} successful " + + "Publish round trips at a ceiling of {}", + after.ceiling(), + before.successes() + 1, + before.ceiling()); + } + } + + /** + * Determine which NotificationMessages are missing between the last sequence number accounted for + * and {@code receivedSequenceNumber}, and which of those the Server can still retransmit. + * + *

    Part 4 §5.14.1.1: "In the case of a retransmission queue overflow, the oldest sent + * NotificationMessage gets deleted." The availableSequenceNumbers of a PublishResponse are what + * is left in that queue, so anything older than the oldest of them is gone for good and + * Republishing it can only be answered Bad_MessageNotAvailable. Recovery therefore starts at the + * oldest sequence number the Server says it still holds, and what precedes it is reported as lost + * data. A gap too large to be recoverable — more than the Server is holding, or more than {@link + * #DEFAULT_MAX_RECOVERABLE_GAP} when it does not say — is reported as lost data in its entirety. + * + *

    Runs on the Subscription's processing queue and performs no I/O. + * + * @param details the {@link SubscriptionDetails} for the Subscription the response belongs to. + * @param response the {@link PublishResponse} being processed. + * @param receivedSequenceNumber the sequence number of the received NotificationMessage. + * @return the sequence numbers to request via Republish, oldest first; empty if there is no gap + * or nothing in it can be recovered. + */ + private List missingSequenceNumbers( + SubscriptionDetails details, PublishResponse response, long receivedSequenceNumber) { + + long expectedSequenceNumber = SequenceNumbers.successor(details.lastSequenceNumber); + + if (!SequenceNumbers.isAhead(receivedSequenceNumber, expectedSequenceNumber)) { + return List.of(); + } + + UInteger subscriptionId = response.getSubscriptionId(); + UInteger[] availableSequenceNumbers = response.getAvailableSequenceNumbers(); + boolean advertised = availableSequenceNumbers != null && availableSequenceNumbers.length > 0; + + long firstRecoverable = expectedSequenceNumber; + long maxRecoverableGap = DEFAULT_MAX_RECOVERABLE_GAP; + boolean dataLost = false; + + if (advertised) { + maxRecoverableGap = availableSequenceNumbers.length; + + long oldestAvailable = oldestAvailable(availableSequenceNumbers, receivedSequenceNumber); + + if (oldestAvailable != SequenceNumbers.NONE + && SequenceNumbers.isAhead(oldestAvailable, expectedSequenceNumber)) { + + logger.warn( + "The oldest NotificationMessage the Server can retransmit is sequenceNumber={}, so " + + "the {} starting at sequenceNumber={} are gone; treating them as lost data, " + + "subscriptionId={}", + oldestAvailable, + SequenceNumbers.forwardDistance(expectedSequenceNumber, oldestAvailable), + expectedSequenceNumber, + subscriptionId); + + firstRecoverable = oldestAvailable; + dataLost = true; + } + } + + long missingCount = SequenceNumbers.forwardDistance(firstRecoverable, receivedSequenceNumber); + + if (missingCount > maxRecoverableGap) { + logger.warn( + "Gap of {} NotificationMessage(s) starting at sequenceNumber={} exceeds the {} the " + + "Server can retransmit; treating it as lost data and resynchronizing to " + + "sequenceNumber={}, subscriptionId={}", + missingCount, + firstRecoverable, + maxRecoverableGap, + receivedSequenceNumber, + subscriptionId); + + if (advertised) { + // Part 4 §5.14.7.1: "The Client should acknowledge all Messages in this list for which it + // will not request retransmission." Nothing in this gap will be requested, and an + // unacknowledged NotificationMessage stays in the Server's retransmission queue — + // re-advertised in every PublishResponse and holding its memory — for the life of the + // Subscription. + acknowledgeAbandonedSequenceNumbers( + details, availableSequenceNumbers, receivedSequenceNumber); + } + + details.subscription.notifyNotificationDataLost(); + + return List.of(); + } + + if (dataLost) { + details.subscription.notifyNotificationDataLost(); + } + + var sequenceNumbers = new ArrayList((int) missingCount); + long sequenceNumber = firstRecoverable; + + for (long i = 0; i < missingCount; i++) { + sequenceNumbers.add(uint(sequenceNumber)); + + sequenceNumber = SequenceNumbers.successor(sequenceNumber); + } + + return sequenceNumbers; + } + + /** + * Queue an acknowledgement for every NotificationMessage the Server advertised as available for + * retransmission but which the client has decided not to recover, so the Server can delete them + * instead of holding them for a retransmission that will never be requested. + * + * @param details the {@link SubscriptionDetails} for the Subscription the messages belong to. + * @param availableSequenceNumbers the availableSequenceNumbers of the PublishResponse whose gap + * is being given up on. + * @param receivedSequenceNumber the sequence number of the received NotificationMessage. + */ + private static void acknowledgeAbandonedSequenceNumbers( + SubscriptionDetails details, + UInteger[] availableSequenceNumbers, + long receivedSequenceNumber) { + + synchronized (details.availableAcknowledgements) { + for (UInteger sequenceNumber : availableSequenceNumbers) { + if (sequenceNumber == null) { + continue; + } + + long value = sequenceNumber.longValue(); + + // A sequence number that is illegal, or that the Server has not sent yet, is not one the + // client is abandoning. + if (!SequenceNumbers.isLegal(value) + || SequenceNumbers.isAhead(value, receivedSequenceNumber)) { + continue; + } + + if (!details.availableAcknowledgements.contains(sequenceNumber)) { + details.availableAcknowledgements.add(sequenceNumber); + } + } + } + } + + /** + * @param availableSequenceNumbers a non-empty availableSequenceNumbers from a PublishResponse. + * @param receivedSequenceNumber the sequence number of the received NotificationMessage. + * @return the oldest sequence number the Server advertised as available for retransmission, or + * {@link SequenceNumbers#NONE} if it advertised none the client can make sense of. + */ + private static long oldestAvailable( + UInteger[] availableSequenceNumbers, long receivedSequenceNumber) { + + long oldest = SequenceNumbers.NONE; + long oldestAge = -1; + + for (UInteger availableSequenceNumber : availableSequenceNumbers) { + if (availableSequenceNumber == null) { + continue; + } + + long sequenceNumber = availableSequenceNumber.longValue(); + + // A sequence number that is illegal, or that the Server has not sent yet, says nothing about + // what its retransmission queue still holds. + if (!SequenceNumbers.isLegal(sequenceNumber) + || SequenceNumbers.isAhead(sequenceNumber, receivedSequenceNumber)) { + continue; + } + + long age = SequenceNumbers.forwardDistance(sequenceNumber, receivedSequenceNumber); + + if (age > oldestAge) { + oldestAge = age; + oldest = sequenceNumber; + } + } + + return oldest; + } + + /** + * Recover, via Republish, the NotificationMessages identified by {@code sequenceNumbers}. + * + *

    The requests are made one at a time and asynchronously: each recovered NotificationMessage + * is handed to the delivery queue as it arrives, so the application sees them in sequence order + * and ahead of the message that revealed the gap, and no thread ever waits for a round trip. If + * any of them cannot be recovered the Subscription is notified that notification data was lost. + * + * @param details the {@link SubscriptionDetails} for the Subscription the messages belong to. + * @param subscriptionId the Server-assigned identifier of that Subscription. + * @param sequenceNumbers the sequence numbers to request, oldest first. + * @return a {@link CompletableFuture} that completes when the last of them has been dealt with. + */ + private CompletableFuture republishMissingNotificationMessages( + SubscriptionDetails details, UInteger subscriptionId, List sequenceNumbers) { + + // Bound to the Session in hand when the gap was found, exactly as the reconnect recovery is: + // a repair that outlives its Session must not have its remaining requests silently re-issued + // on the next one, which has a recovery of its own, and must never park waiting for a Session + // that does not exist yet — the processing queue stays paused until this repair completes, + // and a parked repair would hold the reconnect recovery, and with it every Subscription's + // Publish traffic, shut. + CompletableFuture sessionFuture = client.getSessionAsync(); + + OpcUaSession session = + sessionFuture.isDone() && !sessionFuture.isCompletedExceptionally() + ? sessionFuture.getNow(null) + : null; + + if (session == null) { + logger.warn( + "No Session in hand to repair the gap on; treating it as lost data, subscriptionId={}", + subscriptionId); + + details.subscription.notifyNotificationDataLost(); + + return CompletableFuture.completedFuture(Unit.VALUE); } + + var recovery = new Recovery(); + + CompletableFuture chain = CompletableFuture.completedFuture(Unit.VALUE); + + for (UInteger sequenceNumber : sequenceNumbers) { + chain = + chain.thenCompose( + unit -> + republishNotificationMessage( + session, details, subscriptionId, sequenceNumber, recovery)); + } + + return chain.whenComplete( + (unit, ex) -> { + if (ex != null || recovery.dataLost) { + details.subscription.notifyNotificationDataLost(); + } + }); + } + + /** + * Request one NotificationMessage via Republish and, if it arrives, acknowledge it and hand it to + * the delivery queue. + * + * @param session the Session the repair is bound to. + * @param details the {@link SubscriptionDetails} for the Subscription the message belongs to. + * @param subscriptionId the Server-assigned identifier of that Subscription. + * @param sequenceNumber the sequence number to request. + * @param recovery the state shared by every step of this recovery. + * @return a {@link CompletableFuture} that completes, never exceptionally, once the request has + * been answered one way or the other. + */ + private CompletableFuture republishNotificationMessage( + UaSession session, + SubscriptionDetails details, + UInteger subscriptionId, + UInteger sequenceNumber, + Recovery recovery) { + + if (recovery.abandoned) { + return CompletableFuture.completedFuture(Unit.VALUE); + } + + return republish(session, subscriptionId, sequenceNumber) + .handle( + (republishResponse, ex) -> { + if (ex != null) { + StatusCode statusCode = + UaException.extract(ex).map(UaException::getStatusCode).orElse(StatusCode.BAD); + + recovery.dataLost = true; + + if (statusCode.value() != StatusCodes.Bad_MessageNotAvailable) { + // Bad_MessageNotAvailable is an answer about this one NotificationMessage: the + // Server no longer holds it, but it may well still hold the ones after it, so + // the rest of the recovery is still worth attempting. Any other failure is the + // service call itself failing — a lost Session, a closed connection, a timeout — + // and repeating it for every remaining sequence number can only fail the same + // way. + recovery.abandoned = true; + } + + logger.warn( + "Republish service failure, subscriptionId={}, sequenceNumber={}: {}", + subscriptionId, + sequenceNumber, + statusCode); + } else { + NotificationMessage notificationMessage = + republishResponse.getNotificationMessage(); + + details.availableAcknowledgements.add(sequenceNumber); + + details + .subscription + .getDeliveryQueue() + .execute(() -> deliverNotificationMessage(details, notificationMessage)); + } + + return Unit.VALUE; + }); } + /** + * Deliver {@code notificationMessage} to the application, unless the Subscription it was received + * on is gone. + * + *

    Runs on the Subscription's delivery queue, which is a queue of the {@link OpcUaSubscription} + * object rather than of any one Subscription it has represented: it is neither drained nor + * replaced by {@link OpcUaSubscription#reset()}, so a message can still be waiting here — behind + * an application callback that has not returned — when the Subscription it belongs to is + * discarded and another created in its place. Delivering it then would tell the application that + * a MonitoredItem of the current Subscription has a value that Subscription never reported, and, + * for a Bad_Timeout StatusChangeNotification, would tear down a Subscription that has not timed + * out. The entry's registration is what distinguishes the two: it is dropped the moment its + * Subscription is. + * + * @param details the entry for the Subscription the message was received on. + * @param notificationMessage the {@link NotificationMessage} to deliver. + */ private void deliverNotificationMessage( SubscriptionDetails details, NotificationMessage notificationMessage) { + + if (!details.registered) { + logger.debug( + "Discarding NotificationMessage for a Subscription that no longer exists, " + + "subscriptionId={}, sequenceNumber={}", + details.subscriptionId, + notificationMessage.getSequenceNumber()); + + ExtensionObject[] discardedData = notificationMessage.getNotificationData(); + + if (discardedData != null && discardedData.length > 0) { + // These notifications were received — and may already have been acknowledged, letting the + // Server delete its only retransmission copy — but can no longer be attributed: report + // them as lost data rather than discarding them without a trace. + details.subscription.notifyNotificationDataLost(); + } + + return; + } + ExtensionObject[] notificationData = notificationMessage.getNotificationData(); if (notificationData == null || notificationData.length == 0) { @@ -349,7 +1872,23 @@ private void deliverNotificationMessage( StatusCode status = scn.getStatus(); if (status.value() == StatusCodes.Bad_Timeout) { - details.subscription.getSubscriptionId().ifPresent(subscriptionDetails::remove); + // The Subscription this message was received on no longer exists on the Server, so its + // entry goes with it. Keyed on the entry's own id: the live object's current id belongs + // to whatever Subscription it represents now, which has not timed out. The registered + // check at the top of this method is stale by now — application callbacks of arbitrary + // duration have run since — so the teardown happens only if this entry is still the + // registered one, and the reset only if the object still represents the Subscription + // the entry was registered for. + if (!unregister(details)) { + logger.debug( + "Discarding Bad_Timeout StatusChangeNotification for a Subscription that no " + + "longer exists, subscriptionId={}", + details.subscriptionId); + + continue; + } + + details.subscription.resetIfIncarnation(details.incarnation); } details.subscription.notifyStatusChanged(status); @@ -361,11 +1900,23 @@ private void deliverNotificationMessage( } private long getMaxPendingPublishes() { + return Math.min(getNaturalMaxPendingPublishes(), pendingPublishCeiling.get().ceiling()); + } + + /** + * @return the number of PublishRequests the client would keep outstanding if no Server had ever + * refused one, i.e. {@code min(subscriptionCount + 1, maxPendingPublishRequests)}, or zero if + * there is no Subscription to publish for. It is the target any learned ceiling recovers + * towards and never exceeds. + */ + private long getNaturalMaxPendingPublishes() { + if (subscriptionDetails.isEmpty()) { + return 0; + } + long maxPendingPublishRequests = client.getConfig().getMaxPendingPublishRequests().longValue(); - return subscriptionDetails.isEmpty() - ? 0 - : Math.min(subscriptionDetails.size() + 1, maxPendingPublishRequests); + return Math.min(subscriptionDetails.size() + 1L, maxPendingPublishRequests); } private UInteger getTimeoutHint() { @@ -393,7 +1944,9 @@ private UInteger getTimeoutHint() { double timeoutHint = maxKeepAlive * maxPendingPublishes * 1.5; if (Double.isInfinite(timeoutHint) || timeoutHint > UInteger.MAX_VALUE) { - maxKeepAlive = 0d; + // The timeoutHint is encoded as a UInt32; clamp rather than let an out-of-range value + // reach uint(), which would throw and leave this request unsent. + timeoutHint = UInteger.MAX_VALUE; } logger.debug( @@ -405,17 +1958,198 @@ private UInteger getTimeoutHint() { return uint((long) timeoutHint); } + /** + * The acknowledgements taken from one Subscription's queue for a single PublishRequest. + * + * @param details the Subscription they were taken from. + * @param sequenceNumbers the sequence numbers acknowledged, oldest first. + */ + private record DrainedAcknowledgements( + SubscriptionDetails details, List sequenceNumbers) {} + + /** + * A finished reconnect recovery: the {@link #sessionActivations} value it belonged to and the + * Session it ran on, or {@code null} once that Session has become inactive. + */ + private record RecoveredActivation(long activation, @Nullable UaSession session) {} + + /** + * The available sequence numbers returned while transferring a Subscription to one exact Session. + * Session identity is part of the value so recovery queued for the Session being replaced cannot + * consume or apply the replacement Session's TransferResult. + */ + private record TransferredSequenceNumbers( + UaSession session, UInteger @Nullable [] sequenceNumbers) {} + + /** + * How many PublishRequests the Server has been observed to queue for a Session, and what the + * client has to see before it asks whether it would now queue one more. + * + *

    Immutable, and every transition is a function of the state it is applied to, so the whole of + * it — the ceiling, the cooldown, and whether a probe is outstanding — is read and replaced as + * one value rather than as four fields that a concurrent refusal could interleave with a raise. + * + * @param activation the Session activation this ceiling was learned for. + * @param ceiling the number of PublishRequests the Server was holding when it refused one, or + * {@link #NO_PENDING_PUBLISH_CEILING} if it has never refused one. Never below one: a ceiling + * of zero is a pipeline that can never refill. + * @param successes successful Publish round trips counted towards the next probe. + * @param cooldown successful Publish round trips a probe currently costs. + * @param probing {@code true} while the newest raise is still answerable, i.e. from the raise + * until either a Bad_TooManyPublishRequests is charged to it or the ceiling it installed has + * stood for a whole cooldown. Deliberately not cleared by the first successful round trip + * after the raise: a refused PublishRequest and a delivered NotificationMessage are handled + * on paths of different lengths, so which the client finishes with first is a race, and a + * delivery that could clear this flag would leave the refusal behind it uncharged — which is + * precisely the case a Server that always refuses produces, and would have it probed at a + * constant rate. + */ + private record PendingPublishCeiling( + long activation, long ceiling, long successes, long cooldown, boolean probing) { + + /** No Server has refused a PublishRequest, so only the client's own target applies. */ + static PendingPublishCeiling none(long activation) { + return new PendingPublishCeiling( + activation, NO_PENDING_PUBLISH_CEILING, 0L, PROBE_COOLDOWN_BASE_SUCCESSES, false); + } + + /** + * @param observed the number of PublishRequests the Server was in fact holding. + * @return the state after a Bad_TooManyPublishRequests. The ceiling only ever descends, since a + * Server that refused at {@code observed} has said nothing to withdraw what it refused at + * less; a refusal that answers a probe also lengthens the next cooldown geometrically, + * which is what stops a Server that always refuses from being asked at a constant rate. + * Only the first refusal after a raise is charged for it, so a raise a Server answers with + * more than one refusal costs one lengthening rather than several. + */ + PendingPublishCeiling refused(long observed) { + return new PendingPublishCeiling( + activation, + Math.min(ceiling, observed), + 0L, + probing + ? Math.min(PROBE_COOLDOWN_MAX_SUCCESSES, cooldown * PROBE_COOLDOWN_GROWTH) + : cooldown, + false); + } + + /** + * @param natural the number of PublishRequests the client would keep outstanding if nothing had + * ever been refused. The ceiling never rises above it: it is what recovery aims at, not a + * value to overshoot. + * @return the state after one successful Publish round trip, which is either a payment towards + * the next probe or — when it completes the cooldown — the raise that is that probe. + */ + PendingPublishCeiling roundTrip(long natural) { + if (ceiling >= natural) { + // Fully recovered: there is nothing left to ask for. The raise that got here stops being + // answerable once the ceiling it installed has stood for a whole cooldown, so that a + // Bad_TooManyPublishRequests long afterwards is treated as a new condition on the Server + // rather than as the answer to this probe. Charging it as one would let a client that lives + // through many unrelated transient conditions ratchet its cooldown up to the cap. + return new PendingPublishCeiling( + activation, ceiling, successes + 1, cooldown, probing && successes + 1 < cooldown); + } + + if (successes + 1 < cooldown) { + // Still paying for the next probe. + return new PendingPublishCeiling(activation, ceiling, successes + 1, cooldown, probing); + } + + // One more request than the Server last refused, and no more than that: the question is + // whether it will now take one more, and one extra PublishRequest asks it. No second raise + // follows until a whole further cooldown of round trips has been answered, which is what + // keeps a single probe in flight. + return new PendingPublishCeiling(activation, ceiling + 1, 0L, cooldown, true); + } + } + + /** State shared by the steps of a single Republish recovery. */ + private static class Recovery { + + /** {@code true} if at least one NotificationMessage could not be recovered. */ + private volatile boolean dataLost = false; + + /** {@code true} if the sequence numbers not yet requested are not worth requesting. */ + private volatile boolean abandoned = false; + } + + /** + * One Subscription's registration with this manager. + * + *

    An entry represents a single Subscription, not the {@link OpcUaSubscription} object that + * currently stands for it: the object outlives the Subscription and can be made to stand for + * another, so an entry is bound to the SubscriptionId it was registered under and is discarded, + * never re-bound, when that Subscription goes away. + */ private static class SubscriptionDetails { + /** + * The Server-assigned identifier of the Subscription this entry represents, and the key it is + * registered under. Immutable, unlike {@link OpcUaSubscription#getSubscriptionId()}. + */ + private final UInteger subscriptionId; + + /** + * The incarnation of the {@link OpcUaSubscription} object at the moment this entry was + * registered, i.e. while it still represented the Subscription this entry represents. A reset + * changes it, so comparing it again later asks whether the object still does. + */ + private final long incarnation; + + /** + * {@code false} once this entry has been unregistered, i.e. once the Subscription it represents + * is known to be gone. Work queued for it before then is discarded rather than applied to the + * Subscription the {@link #subscription} object represents now. + */ + private volatile boolean registered = true; + + /** Sequence numbers of received NotificationMessages awaiting acknowledgement. */ private final List availableAcknowledgements = Collections.synchronizedList(new ArrayList<>()); - private volatile long lastSequenceNumber = 0L; + /** + * The availableSequenceNumbers of the TransferResult that last transferred this Subscription to + * an exact Session, or {@code null} if the client has not been told what the Server holds since + * the last time it asked for it. + * + *

    Set while the Session the transfer was part of is on its way to Active and consumed by the + * recovery that runs when it gets there. Session identity is stored so a task queued for the + * Session being replaced cannot consume the replacement Session's result. No future activation + * number is predicted here: Session activity callbacks are asynchronous and may be reordered. + */ + private final AtomicReference<@Nullable TransferredSequenceNumbers> transferredSequenceNumbers = + new AtomicReference<>(); + + /** + * Serial queue on which this Subscription's PublishResponses are processed, in the order the + * Server sent them. + * + *

    One queue per Subscription rather than one for the client: a Subscription that is + * recovering a gap pauses its own queue for the duration, and a Subscription with nothing + * missing must not have to wait behind it. + */ + private final TaskQueue processingQueue; + + /** + * The sequence number of the last NotificationMessage accounted for, i.e. received, recovered + * via Republish, or given up on. {@link SequenceNumbers#NONE} until the first PublishResponse + * has been processed, at which point the next NotificationMessage expected is {@link + * SequenceNumbers#FIRST}. + */ + private volatile long lastSequenceNumber = SequenceNumbers.NONE; private final OpcUaSubscription subscription; - private SubscriptionDetails(OpcUaSubscription subscription) { + private SubscriptionDetails( + OpcUaSubscription subscription, UInteger subscriptionId, Executor executor) { + this.subscription = subscription; + this.subscriptionId = subscriptionId; + + incarnation = subscription.getIncarnation(); + + processingQueue = new TaskQueue(executor); } } } diff --git a/opc-ua-sdk/sdk-client/src/main/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SequenceNumbers.java b/opc-ua-sdk/sdk-client/src/main/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SequenceNumbers.java new file mode 100644 index 0000000000..9e6fc3e654 --- /dev/null +++ b/opc-ua-sdk/sdk-client/src/main/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SequenceNumbers.java @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2026 the Eclipse Milo Authors + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.milo.opcua.sdk.client.subscriptions; + +/** + * Arithmetic on NotificationMessage sequence numbers. + * + *

    OPC UA Part 4 §5.14.1.1: "The value 0 is never used for the sequence number. The first + * NotificationMessage sent on a Subscription has a sequence number of 1. If the sequence number + * rolls over, it rolls over to 1." + * + *

    Sequence numbers are therefore a cycle over the values {@code 1..0xFFFFFFFF} — not the + * plain UInt32 Counter of Part 4 §7.8, whose successor of {@code 0xFFFFFFFF} is 0. Arithmetic that + * uses the §7.8 rule, or that does not wrap at all, is wrong by one at every rollover. + */ +final class SequenceNumbers { + + /** The sequence number of the first NotificationMessage sent on a Subscription. */ + static final long FIRST = 1L; + + /** The largest legal sequence number. Its successor is {@link #FIRST}. */ + static final long LAST = 0xFFFF_FFFFL; + + /** + * Not a legal sequence number, and therefore usable as a distinguished "no NotificationMessage + * has been accounted for yet" marker. {@link #successor(long)} maps it to {@link #FIRST}, which + * is the sequence number of the first NotificationMessage a Subscription sends; no other + * operation accepts it. + */ + static final long NONE = 0L; + + /** The number of distinct legal sequence numbers, i.e. the length of the cycle. */ + private static final long CYCLE = LAST - FIRST + 1; + + /** + * The largest forward distance still interpreted as "ahead". Beyond the half-way point of the + * cycle a sequence number is nearer going backwards, and is treated as stale rather than as the + * far end of an enormous gap. + */ + private static final long AHEAD_LIMIT = CYCLE / 2; + + private SequenceNumbers() {} + + /** + * @param sequenceNumber the value to test. + * @return {@code true} if {@code sequenceNumber} is a legal NotificationMessage sequence number. + */ + static boolean isLegal(long sequenceNumber) { + return sequenceNumber >= FIRST && sequenceNumber <= LAST; + } + + /** + * Returns the sequence number that follows {@code sequenceNumber}, wrapping to {@link #FIRST} and + * skipping {@link #NONE}. + * + * @param sequenceNumber a legal sequence number, or {@link #NONE} for "nothing accounted for + * yet". + * @return the next sequence number; never {@link #NONE}. + */ + static long successor(long sequenceNumber) { + if (sequenceNumber == NONE || sequenceNumber == LAST) { + return FIRST; + } + checkLegal(sequenceNumber); + + return sequenceNumber + 1; + } + + /** + * Returns the sequence number that precedes {@code sequenceNumber}, wrapping to {@link #LAST} and + * skipping {@link #NONE}. + * + * @param sequenceNumber a legal sequence number. {@link #NONE} is rejected: nothing precedes "no + * messages yet". + * @return the previous sequence number; never {@link #NONE}. + */ + static long predecessor(long sequenceNumber) { + checkLegal(sequenceNumber); + + return sequenceNumber == FIRST ? LAST : sequenceNumber - 1; + } + + /** + * Returns the number of steps from {@code from} to {@code to} in the direction sequence numbers + * advance, i.e. the number of NotificationMessages in the half-open range {@code [from, to)}. + * + * @param from a legal sequence number. + * @param to a legal sequence number. + * @return the forward distance, in {@code [0, CYCLE)}. + */ + static long forwardDistance(long from, long to) { + checkLegal(from); + checkLegal(to); + + return Math.floorMod(to - from, CYCLE); + } + + /** + * Returns whether {@code received} is ahead of {@code expected}, i.e. whether + * NotificationMessages are missing between them. + * + *

    A sequence number that is neither {@code expected} nor ahead of it is behind it: a stale, + * duplicated, or reordered message rather than a gap. + * + * @param received the sequence number of the NotificationMessage that arrived. + * @param expected the sequence number of the NotificationMessage that was expected. + * @return {@code true} if at least one NotificationMessage is missing between them. + */ + static boolean isAhead(long received, long expected) { + long distance = forwardDistance(expected, received); + + return distance > 0 && distance <= AHEAD_LIMIT; + } + + private static void checkLegal(long sequenceNumber) { + if (!isLegal(sequenceNumber)) { + throw new IllegalArgumentException("illegal sequenceNumber: " + sequenceNumber); + } + } +} diff --git a/opc-ua-sdk/sdk-client/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SequenceNumbersTest.java b/opc-ua-sdk/sdk-client/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SequenceNumbersTest.java new file mode 100644 index 0000000000..92877a3fb2 --- /dev/null +++ b/opc-ua-sdk/sdk-client/src/test/java/org/eclipse/milo/opcua/sdk/client/subscriptions/SequenceNumbersTest.java @@ -0,0 +1,180 @@ +/* + * Copyright (c) 2026 the Eclipse Milo Authors + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.milo.opcua.sdk.client.subscriptions; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; + +/** + * NotificationMessage sequence number arithmetic. + * + *

    Part 4 §5.14.1.1: "The value 0 is never used for the sequence number. The first + * NotificationMessage sent on a Subscription has a sequence number of 1. If the sequence number + * rolls over, it rolls over to 1." Sequence numbers are therefore a cycle over {@code + * 1..0xFFFFFFFF}, and not the plain UInt32 Counter of §7.8 whose successor of {@code + * 0xFFFFFFFF} is 0 — an implementation that uses the §7.8 rule is wrong by one at every rollover, + * and one that does not wrap at all stops detecting gaps entirely once it reaches the boundary. + * + *

    The rollover cannot be reached over the wire (a sequence number advances one message at a + * time), so these unit tests are the only place the boundary behavior can be pinned down. + */ +class SequenceNumbersTest { + + private static final long LAST = 0xFFFF_FFFFL; + + /** The rollover rule of §5.14.1.1: the successor of the largest sequence number is 1, not 0. */ + @Test + void successorOfTheLargestSequenceNumberIsOne() { + assertEquals(1L, SequenceNumbers.successor(LAST)); + } + + /** The mirror image of the rollover rule, used to derive the message before a keep-alive. */ + @Test + void predecessorOfOneIsTheLargestSequenceNumber() { + assertEquals(LAST, SequenceNumbers.predecessor(1L)); + } + + /** + * 0 is not a legal sequence number, so it is used as the "nothing accounted for yet" marker; the + * first NotificationMessage a Subscription sends has sequence number 1, so that is what follows + * it. + */ + @Test + void successorOfNoneIsTheFirstSequenceNumber() { + assertEquals(SequenceNumbers.FIRST, SequenceNumbers.successor(SequenceNumbers.NONE)); + } + + /** + * 0 doubles as the "nothing accounted for yet" marker, so producing it from real arithmetic would + * silently reset a Subscription's gap detection. + */ + @ParameterizedTest + @ValueSource(longs = {1L, 2L, 0x8000_0000L, 0xFFFF_FFFEL, LAST}) + void neitherSuccessorNorPredecessorEverProducesZero(long sequenceNumber) { + assertNotEquals(0L, SequenceNumbers.successor(sequenceNumber)); + assertNotEquals(0L, SequenceNumbers.predecessor(sequenceNumber)); + } + + @ParameterizedTest + @ValueSource(longs = {1L, 2L, 0x8000_0000L, 0xFFFF_FFFEL, LAST}) + void predecessorUndoesSuccessor(long sequenceNumber) { + assertEquals( + sequenceNumber, SequenceNumbers.predecessor(SequenceNumbers.successor(sequenceNumber))); + assertEquals( + sequenceNumber, SequenceNumbers.successor(SequenceNumbers.predecessor(sequenceNumber))); + } + + /** + * The number of NotificationMessages between two sequence numbers is what bounds Republish + * recovery; counting one too many at the rollover would request a message that cannot exist. + */ + @ParameterizedTest(name = "forwardDistance({0}, {1}) == {2}") + @CsvSource({ + "1, 1, 0", + "1, 2, 1", + "5, 10, 5", + "4294967295, 1, 1", // the rollover is a single step + "4294967295, 2, 2", + "4294967294, 1, 2", + "1, 4294967295, 4294967294", // all the way around, one short of the full cycle + "2, 1, 4294967294" // backwards by one is almost a full cycle forwards + }) + void forwardDistanceCountsStepsInTransmissionOrder(long from, long to, long expected) { + assertEquals(expected, SequenceNumbers.forwardDistance(from, to)); + } + + /** + * The gap-detection predicate. The wrapped sequence number 1 arriving when {@code 0xFFFFFFFF} was + * expected means {@code 0xFFFFFFFF} is missing — with unwrapped arithmetic this compares as + * "behind" and the loss is never noticed. + */ + @Test + void aWrappedSequenceNumberIsAheadOfTheExpectedOne() { + assertTrue(SequenceNumbers.isAhead(1L, LAST)); + } + + @ParameterizedTest(name = "isAhead({0}, {1}) == {2}") + @CsvSource({ + "1, 1, false", // in sequence + "2, 1, true", // one message missing + "10, 6, true", // four messages missing + "1, 4294967295, true", // the message at the rollover boundary is missing + "4294967295, 1, false", // stale: 1 was expected, 0xFFFFFFFF is a full cycle behind it + "6, 10, false", // a duplicate of an already-accounted-for message + "2147483648, 1, true", // half the cycle ahead is still a gap + "2147483649, 1, false" // more than half the cycle ahead is nearer going backwards: stale + }) + void isAheadDistinguishesGapsFromStaleSequenceNumbers( + long received, long expected, boolean ahead) { + + assertEquals(ahead, SequenceNumbers.isAhead(received, expected)); + } + + /** + * The composition the Republish recovery loop relies on: {@code forwardDistance} says how many + * NotificationMessages are missing and {@code successor} enumerates them, so a gap that spans the + * rollover must yield {@code 0xFFFFFFFF} and 1 — never 0, and never 4294967296. + */ + @Test + void aGapSpanningTheRolloverEnumeratesEveryMissingSequenceNumber() { + long expected = LAST; + long received = 2L; + + long missingCount = SequenceNumbers.forwardDistance(expected, received); + var missing = new ArrayList(); + + long sequenceNumber = expected; + for (long i = 0; i < missingCount; i++) { + missing.add(sequenceNumber); + sequenceNumber = SequenceNumbers.successor(sequenceNumber); + } + + assertEquals(List.of(LAST, 1L), missing); + assertEquals( + received, + sequenceNumber, + "walking forwardDistance() successors must land exactly on the received sequence number"); + } + + @ParameterizedTest + @ValueSource(longs = {0L, 1L, 2L, LAST, 0x1_0000_0000L}) + void onlyValuesInOneThroughTheLargestSequenceNumberAreLegal(long sequenceNumber) { + assertEquals( + sequenceNumber >= 1L && sequenceNumber <= LAST, SequenceNumbers.isLegal(sequenceNumber)); + } + + /** + * The "nothing accounted for yet" marker and out-of-range values have no defined arithmetic; + * failing loudly keeps a bad value from being laundered into a plausible-looking sequence number. + */ + @Test + void illegalSequenceNumbersAreRejected() { + assertThrows(IllegalArgumentException.class, () -> SequenceNumbers.predecessor(0L)); + assertThrows(IllegalArgumentException.class, () -> SequenceNumbers.successor(0x1_0000_0000L)); + assertThrows(IllegalArgumentException.class, () -> SequenceNumbers.forwardDistance(0L, 1L)); + assertThrows(IllegalArgumentException.class, () -> SequenceNumbers.isAhead(1L, 0L)); + } + + @Test + void theNothingAccountedForYetMarkerIsNotALegalSequenceNumber() { + assertFalse(SequenceNumbers.isLegal(SequenceNumbers.NONE)); + } +}