ci: migrate conformance to restatedev/e2e sdk-tests + CI job (closes #1)

.github/workflows/conformance.yml

@@ -0,0 +1,53 @@
+name: Conformance
+
+# The cross-SDK conformance suite (restatedev/e2e sdk-tests) is heavy — it boots a real
+# Restate runtime + the PHP service image via Testcontainers — so it does NOT run on every
+# PR. It runs on pushes to main, on demand, and weekly. GitHub runners have AVX2, so the
+# new suite's Restate >= 1.6 runtime works here (unlike AVX2-free dev hosts, which use the
+# archived v3.4 fallback documented in conformance/README.md).
+on:
+  push:
+    branches: [main]
+  pull_request:
+    types: [labeled]
+  workflow_dispatch:
+  schedule:
+    - cron: '0 6 * * 1' # Mondays 06:00 UTC
+
+permissions:
+  contents: read
+
+concurrency:
+  group: conformance-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  conformance:
+    name: e2e/sdk-tests (default, bidi V7)
+    # On PRs, only run when explicitly labelled `conformance` (the suite is too slow to
+    # gate every PR); always run on push/dispatch/schedule.
+    if: github.event_name != 'pull_request' || github.event.label.name == 'conformance'
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Set up JDK 21
+        uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: '21'
+
+      - name: Run conformance suite (downloads jar, builds runtime + service images)
+        run: make conformance
+        env:
+          TEST_SUITE: default
+
+      - name: Upload conformance report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: conformance-report
+          path: build/conformance-report/
+          if-no-files-found: ignore
+          retention-days: 14

Makefile

@@ -89,37 +89,46 @@ examples:
 		-d '{"uri":"http://examples-endpoint:9080","force":true}' >/dev/null \
 		&& echo "Examples registered. Try: curl $(INGRESS_URL)/FanOut/fanOut"
 
-# --- Cross-SDK conformance (official restatedev/sdk-test-suite) -----------
-# Pins Restate 1.5.2 (last AVX2-free runtime); the PHP image is tagged localhost/
-# so the suite uses it from local cache instead of pulling.
-SUITE_VERSION   ?= v3.4
-SUITE_JAR       ?= build/restate-sdk-test-suite.jar
-SUITE_URL       := https://github.com/restatedev/sdk-test-suite/releases/download/$(SUITE_VERSION)/restate-sdk-test-suite.jar
-RESTATE_IMAGE   ?= docker.io/restatedev/restate:1.5.2
-PHP_TS_IMAGE    ?= localhost/restatedev/php-test-services:latest
-TEST_SUITE      ?= default
-EXCLUSIONS      ?= conformance/exclusions.yaml
-REPORT_DIR      ?= build/conformance-report
+# --- Cross-SDK conformance (official restatedev/e2e sdk-tests) ------------
+# The primary path runs the actively-maintained restatedev/e2e suite against the
+# bidi (amphp) transport on a V7-enabled runtime — the SDK's default transport,
+# where Cancellation / KillInvocation / Signals pass. Needs JDK >= 21 and an AVX2
+# host (Restate >= 1.6). The archived sdk-test-suite v3.4 + Restate 1.5.2 offline
+# fallback for AVX2-free hosts is documented in conformance/README.md.
+SUITE_VERSION    ?= v2.2
+SUITE_JAR        ?= build/sdk-tests.jar
+SUITE_URL        := https://github.com/restatedev/e2e/releases/download/$(SUITE_VERSION)/sdk-tests.jar
+JAVA             ?= java
+RESTATE_V7_IMAGE ?= localhost/restatedev/restate-v7:latest
+PHP_TS_IMAGE     ?= localhost/restatedev/php-amp-test-services:latest
+TEST_SUITE       ?= default
+EXCLUSIONS       ?= conformance/exclusions.yaml
+REPORT_DIR       ?= build/conformance-report
 
 .PHONY: conformance conformance-image conformance-jar conformance-run
 
 conformance-jar:
 	@test -f $(SUITE_JAR) || (mkdir -p $(dir $(SUITE_JAR)) && curl -fSL -o $(SUITE_JAR) $(SUITE_URL))
 
+# Build the V7-enabled runtime (stock Restate + the experimental-protocol-v7 flag) and the
+# bidi (amphp) test-services image. Both are tagged localhost/ so the suite uses them from
+# cache (--image-pull-policy=CACHED) instead of pulling.
 conformance-image:
-	docker build -f conformance/Dockerfile -t $(PHP_TS_IMAGE) .
+	docker build -f conformance/Dockerfile.restate-v7 -t $(RESTATE_V7_IMAGE) .
+	docker build -f conformance/Dockerfile.amp -t $(PHP_TS_IMAGE) .
 
-# Full run: download suite, build image, run the chosen config(s).
+# Full run: download suite, build the runtime + service images, run the chosen config(s).
 conformance: conformance-jar conformance-image conformance-run
 
 conformance-run:
-	java -jar $(SUITE_JAR) run \
-		--restate-container-image=$(RESTATE_IMAGE) \
+	$(JAVA) -jar $(SUITE_JAR) run \
+		--restate-container-image=$(RESTATE_V7_IMAGE) \
+		--service-container-image=$(PHP_TS_IMAGE) \
 		--test-suite=$(TEST_SUITE) \
 		--sequential \
 		$(if $(wildcard $(EXCLUSIONS)),--exclusions-file=$(EXCLUSIONS),) \
-		--report-dir=$(REPORT_DIR) \
-		$(PHP_TS_IMAGE)
+		--image-pull-policy=CACHED \
+		--report-dir=$(REPORT_DIR)
 	@echo "Report + exclusions.new.yaml under $(REPORT_DIR)/"
 
 logs:

README.md

@@ -291,17 +291,19 @@ composer test          # vendor/bin/phpunit --testsuite unit
 ```
 
 End-to-end verification is the **official cross-SDK conformance suite**
-([`restatedev/sdk-test-suite`](https://github.com/restatedev/sdk-test-suite)) — the
-same battery every Restate SDK runs. It boots a real Restate runtime + a PHP image of
-the standard test-services and drives them:
+([`restatedev/e2e`](https://github.com/restatedev/e2e/tree/main/sdk-tests)) — the same
+battery every Restate SDK runs. It boots a real Restate runtime + a PHP image of the
+standard test-services and drives them (needs JDK ≥ 21 + an AVX2 host):
 
 ```bash
-make conformance               # downloads the suite, builds the image, runs `default`
+make conformance               # downloads the suite, builds the images, runs `default`
 make conformance TEST_SUITE=all
 ```
 
-The `default` config passes **30/30** (8 documented exclusions). See
-[`conformance/README.md`](conformance/README.md).
+The `default` config passes **48 / 49** over the bidi (amphp) transport on a V7-enabled
+runtime — `Cancellation` 6/6, `KillInvocation` 1/1, `Signals` 2/2 included. It also runs
+in CI ([`conformance.yml`](.github/workflows/conformance.yml)); an AVX2-free offline
+fallback is documented in [`conformance/README.md`](conformance/README.md).
 
 To try the example services live by hand:
 

conformance/README.md

@@ -1,75 +1,84 @@
 # Cross-SDK conformance
 
 This SDK is verified against the **official Restate conformance suite**
-([`restatedev/sdk-test-suite`](https://github.com/restatedev/sdk-test-suite)) — the
-same battery of tests every Restate SDK (TypeScript, Java, Python, Go, Rust) runs.
+([`restatedev/e2e`](https://github.com/restatedev/e2e/tree/main/sdk-tests) — the actively
+maintained successor to the now-archived `restatedev/sdk-test-suite`) — the same battery
+of tests every Restate SDK (TypeScript, Java, Python, Go, Rust) runs.
 
-`conformance/Services/` ports the standard contract test-services to PHP; the suite
-boots a real Restate runtime + this image (via Testcontainers) and drives them.
+`conformance/Services/` ports the standard contract test-services to PHP; the suite boots
+a real Restate runtime + this image (via Testcontainers) and drives them.
 
 ## Result
 
-`default` configuration — **30 / 30 passing, 0 failures** (Restate 1.5.2):
-
-State · ServiceToServiceCommunication (calls, idempotency keys, delayed sends) ·
-WorkflowAPI (durable promises) · Sleep · RunRetry (per-run retry policy) ·
-UserErrors (terminal vs retryable propagation across calls and side effects) ·
-ProxyRequestSigning (Ed25519 request identity) · Combinators (awakeable-or-timeout) ·
-SleepWithFailures · StopRuntime / KillRuntime (durability across restarts) ·
-UpgradeWithNewInvocation · KafkaIngress · Ingress (header pass-through).
+The primary configuration drives the **bidirectional (amphp) transport on service
+protocol V7** — the SDK's default server — against a V7-enabled runtime. The `default`
+suite passes **48 / 49**, including `Cancellation` 6/6, `KillInvocation` 1/1, `Signals`
+2/2, `Combinators` 9/9, `RunRetry` 3/3, `UserErrors` 10/10, and
+`ServiceToServiceCommunication` 5/5.
 
 A few tests are excluded with documented reasons in `exclusions.yaml` (`awaitAny`
 combinator edge cases the Rust SDK also excludes; per-handler raw serde; one fan-out
-ordering case; in-flight deployment upgrade; V7 scoped concurrency).
-
-### Bidirectional (amp) transport on service protocol V7
-
-The `AmpStreamingServer` transport speaks service protocol **V7** (signals, signal-backed
-awakeables, named signals, the Future-based suspension/`AwaitingOn`). Against a V7-enabled
-runtime (`Dockerfile.restate-v7`) the `default` suite passes **48 / 49**, including
-`Cancellation` 6/6, `KillInvocation` 1/1, `Signals` 2/2, `Combinators` 9/9,
-`RunRetry` 3/3, `UserErrors` 10/10, and `ServiceToServiceCommunication` 5/5. The remaining
-exclusions are the same documented gaps as above plus `ServiceToServiceScopeConcurrency`
-(V7 scoped concurrency / virtual queues — not yet implemented). See the run instructions
-below and `../docs/adr/0001-cancellation-over-bidirectional-streaming.md`.
+ordering case; in-flight deployment upgrade; V7 scoped concurrency / virtual queues). See
+`../docs/adr/0001-cancellation-over-bidirectional-streaming.md`.
 
 ## Run it
 
-Requires Java 21 + Docker. On this host the Restate runtime is pinned to **1.5.2**
-(the `:latest` image needs AVX2 and SIGILLs on older CPUs).
+Requires **JDK ≥ 21** + Docker, and an **AVX2** host (the suite's Restate ≥ 1.6 runtime
+SIGILLs on older CPUs — see the fallback below). `make conformance` downloads the suite
+jar, builds the V7-enabled runtime (`Dockerfile.restate-v7`) and the bidi service image
+(`Dockerfile.amp`), and runs the `default` config:
 
 ```bash
-make conformance                       # downloads the suite jar, builds the image, runs `default`
+make conformance                       # the default config, bidi V7
 make conformance TEST_SUITE=all        # every configuration
-# or a single class while debugging:
-java -jar build/restate-sdk-test-suite.jar run \
-  --restate-container-image=docker.io/restatedev/restate:1.5.2 \
-  --test-suite=default --sequential --test-name=State \
-  localhost/restatedev/php-test-services:latest
-```
-
-`--sequential` is used because parallel container startup makes the runtime's h2c
-discovery handshake flaky on a single host. The PHP server speaks HTTP/2 cleartext
-(h2c), which the runtime uses for discovery + invocation.
-
-### Bidirectional (HTTP/2) streaming + service protocol V7
-
-The `AmpStreamingServer` transport (`conformance/Dockerfile.amp` →
-`localhost/restatedev/php-amp-test-services`) serves true bidi h2c and speaks service
-protocol **V7** (signals, signal-backed awakeables, `AwaitingOnMessage`). Restate 1.7.0
-supports V7 but negotiates V6 by default, so build a V7-enabled runtime image first:
-
-```bash
-docker build -f conformance/Dockerfile.restate-v7 -t localhost/restatedev/restate-v7:latest .
-docker build -f conformance/Dockerfile.amp -t localhost/restatedev/php-amp-test-services:latest .
+JAVA=/path/to/jdk21/bin/java make conformance   # if `java` is < 21
 
+# or a single class while debugging:
 java -jar build/sdk-tests.jar run \
   --restate-container-image=localhost/restatedev/restate-v7:latest \
   --service-container-image=localhost/restatedev/php-amp-test-services:latest \
   --test-suite=default --test-name=Cancellation \
   --exclusions-file=conformance/exclusions.yaml \
-  --image-pull-policy=CACHED --report-dir=build/conformance-amp-report --sequential
+  --image-pull-policy=CACHED --report-dir=build/conformance-report --sequential
 ```
 
+`--sequential` is used because parallel container startup makes the runtime's h2c
+discovery handshake flaky on a single host. The PHP server speaks HTTP/2 cleartext (h2c),
+which the runtime uses for discovery + bidirectional invocation.
+
+The runtime is built from `Dockerfile.restate-v7` (stock Restate + the
+`experimental-enable-protocol-v7` flag): Restate 1.7.0 supports V7 but negotiates V6 by
+default, on which the SDK's signal/awakeable model is invalid.
+
 After a run, `build/conformance-report/<ts>/exclusions.new.yaml` lists everything that
 failed/was skipped — copy entries into `exclusions.yaml` to baseline new gaps.
+
+## CI
+
+`.github/workflows/conformance.yml` runs the same `make conformance` on pushes to `main`,
+weekly, on demand (`workflow_dispatch`), and on PRs labelled `conformance`. GitHub runners
+have AVX2, so the ≥ 1.6 runtime works there. The suite is too heavy to gate every PR, so it
+is not in the main `CI` workflow.
+
+## Offline fallback (AVX2-free hosts)
+
+The new suite's runtime needs AVX2. On an AVX2-free host, run the **archived**
+[`restatedev/sdk-test-suite`](https://github.com/restatedev/sdk-test-suite) `v4.1` jar
+against the **request/response Swoole** image (`conformance/Dockerfile` →
+`php-test-services`) on the last AVX2-free runtime, **Restate 1.5.2**:
+
+```bash
+curl -fSL -o build/restate-sdk-test-suite.jar \
+  https://github.com/restatedev/sdk-test-suite/releases/download/v4.1/restate-sdk-test-suite.jar
+docker build -f conformance/Dockerfile -t localhost/restatedev/php-test-services:latest .
+java -jar build/restate-sdk-test-suite.jar run \
+  --restate-container-image=docker.io/restatedev/restate:1.5.2 \
+  --test-suite=default --sequential \
+  --report-dir=build/conformance-report \
+  localhost/restatedev/php-test-services:latest
+```
+
+This path serves request/response, so the V7-only cases (`Cancellation`, `KillInvocation`,
+`Signals`) cannot pass and must be excluded — generate a fallback exclusions file from the
+run's `exclusions.new.yaml` rather than reusing the bidi `exclusions.yaml`. It is frozen
+(no new-runtime compatibility), but runs where the maintained suite's runtime will not.