From bcdd2528250c696a6e22d6d363ba240e087e7dea Mon Sep 17 00:00:00 2001 From: qcodr <297220555+qcodr@users.noreply.github.com> Date: Sat, 27 Jun 2026 15:57:24 +0200 Subject: [PATCH] ci: migrate conformance to restatedev/e2e sdk-tests + CI job (closes #1) restatedev/sdk-test-suite is archived; migrate the harness to its maintained successor restatedev/e2e (sdk-tests v2.2, same `run` CLI) and wire it into CI. - Makefile: `make conformance` downloads the e2e v2.2 jar and runs the default suite over the bidi (amphp) transport on a V7-enabled runtime (built from Dockerfile.restate-v7 + Dockerfile.amp) via --service-container-image; JDK >= 21 - .github/workflows/conformance.yml: runs `make conformance` on push to main, weekly, on demand, and on PRs labelled `conformance` (too heavy to gate every PR; GitHub runners have AVX2 for the suite's >= 1.6 runtime) - conformance/README.md + README.md: e2e v2.2 as the suite, the bidi 48/49 result, CI, and the archived v3.4 + Restate 1.5.2 offline fallback for AVX2-free hosts - exclusions.yaml validated against a live e2e v2.2 run, unchanged (6 documented gaps): default suite "Succeeded tests: 48 / 48", 0 class-init failures, run against the current optimized transport --- .github/workflows/conformance.yml | 53 +++++++++++++++ Makefile | 43 +++++++----- README.md | 14 ++-- conformance/README.md | 107 ++++++++++++++++-------------- 4 files changed, 145 insertions(+), 72 deletions(-) create mode 100644 .github/workflows/conformance.yml diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml new file mode 100644 index 0000000..1418054 --- /dev/null +++ b/.github/workflows/conformance.yml @@ -0,0 +1,53 @@ +name: Conformance + +# The cross-SDK conformance suite (restatedev/e2e sdk-tests) is heavy — it boots a real +# Restate runtime + the PHP service image via Testcontainers — so it does NOT run on every +# PR. It runs on pushes to main, on demand, and weekly. GitHub runners have AVX2, so the +# new suite's Restate >= 1.6 runtime works here (unlike AVX2-free dev hosts, which use the +# archived v3.4 fallback documented in conformance/README.md). +on: + push: + branches: [main] + pull_request: + types: [labeled] + workflow_dispatch: + schedule: + - cron: '0 6 * * 1' # Mondays 06:00 UTC + +permissions: + contents: read + +concurrency: + group: conformance-${{ github.ref }} + cancel-in-progress: true + +jobs: + conformance: + name: e2e/sdk-tests (default, bidi V7) + # On PRs, only run when explicitly labelled `conformance` (the suite is too slow to + # gate every PR); always run on push/dispatch/schedule. + if: github.event_name != 'pull_request' || github.event.label.name == 'conformance' + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v5 + + - name: Set up JDK 21 + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '21' + + - name: Run conformance suite (downloads jar, builds runtime + service images) + run: make conformance + env: + TEST_SUITE: default + + - name: Upload conformance report + if: always() + uses: actions/upload-artifact@v4 + with: + name: conformance-report + path: build/conformance-report/ + if-no-files-found: ignore + retention-days: 14 diff --git a/Makefile b/Makefile index b37c7b7..66c2166 100644 --- a/Makefile +++ b/Makefile @@ -89,37 +89,46 @@ examples: -d '{"uri":"http://examples-endpoint:9080","force":true}' >/dev/null \ && echo "Examples registered. Try: curl $(INGRESS_URL)/FanOut/fanOut" -# --- Cross-SDK conformance (official restatedev/sdk-test-suite) ----------- -# Pins Restate 1.5.2 (last AVX2-free runtime); the PHP image is tagged localhost/ -# so the suite uses it from local cache instead of pulling. -SUITE_VERSION ?= v3.4 -SUITE_JAR ?= build/restate-sdk-test-suite.jar -SUITE_URL := https://github.com/restatedev/sdk-test-suite/releases/download/$(SUITE_VERSION)/restate-sdk-test-suite.jar -RESTATE_IMAGE ?= docker.io/restatedev/restate:1.5.2 -PHP_TS_IMAGE ?= localhost/restatedev/php-test-services:latest -TEST_SUITE ?= default -EXCLUSIONS ?= conformance/exclusions.yaml -REPORT_DIR ?= build/conformance-report +# --- Cross-SDK conformance (official restatedev/e2e sdk-tests) ------------ +# The primary path runs the actively-maintained restatedev/e2e suite against the +# bidi (amphp) transport on a V7-enabled runtime — the SDK's default transport, +# where Cancellation / KillInvocation / Signals pass. Needs JDK >= 21 and an AVX2 +# host (Restate >= 1.6). The archived sdk-test-suite v3.4 + Restate 1.5.2 offline +# fallback for AVX2-free hosts is documented in conformance/README.md. +SUITE_VERSION ?= v2.2 +SUITE_JAR ?= build/sdk-tests.jar +SUITE_URL := https://github.com/restatedev/e2e/releases/download/$(SUITE_VERSION)/sdk-tests.jar +JAVA ?= java +RESTATE_V7_IMAGE ?= localhost/restatedev/restate-v7:latest +PHP_TS_IMAGE ?= localhost/restatedev/php-amp-test-services:latest +TEST_SUITE ?= default +EXCLUSIONS ?= conformance/exclusions.yaml +REPORT_DIR ?= build/conformance-report .PHONY: conformance conformance-image conformance-jar conformance-run conformance-jar: @test -f $(SUITE_JAR) || (mkdir -p $(dir $(SUITE_JAR)) && curl -fSL -o $(SUITE_JAR) $(SUITE_URL)) +# Build the V7-enabled runtime (stock Restate + the experimental-protocol-v7 flag) and the +# bidi (amphp) test-services image. Both are tagged localhost/ so the suite uses them from +# cache (--image-pull-policy=CACHED) instead of pulling. conformance-image: - docker build -f conformance/Dockerfile -t $(PHP_TS_IMAGE) . + docker build -f conformance/Dockerfile.restate-v7 -t $(RESTATE_V7_IMAGE) . + docker build -f conformance/Dockerfile.amp -t $(PHP_TS_IMAGE) . -# Full run: download suite, build image, run the chosen config(s). +# Full run: download suite, build the runtime + service images, run the chosen config(s). conformance: conformance-jar conformance-image conformance-run conformance-run: - java -jar $(SUITE_JAR) run \ - --restate-container-image=$(RESTATE_IMAGE) \ + $(JAVA) -jar $(SUITE_JAR) run \ + --restate-container-image=$(RESTATE_V7_IMAGE) \ + --service-container-image=$(PHP_TS_IMAGE) \ --test-suite=$(TEST_SUITE) \ --sequential \ $(if $(wildcard $(EXCLUSIONS)),--exclusions-file=$(EXCLUSIONS),) \ - --report-dir=$(REPORT_DIR) \ - $(PHP_TS_IMAGE) + --image-pull-policy=CACHED \ + --report-dir=$(REPORT_DIR) @echo "Report + exclusions.new.yaml under $(REPORT_DIR)/" logs: diff --git a/README.md b/README.md index f4147ed..0f998c1 100644 --- a/README.md +++ b/README.md @@ -291,17 +291,19 @@ composer test # vendor/bin/phpunit --testsuite unit ``` End-to-end verification is the **official cross-SDK conformance suite** -([`restatedev/sdk-test-suite`](https://github.com/restatedev/sdk-test-suite)) — the -same battery every Restate SDK runs. It boots a real Restate runtime + a PHP image of -the standard test-services and drives them: +([`restatedev/e2e`](https://github.com/restatedev/e2e/tree/main/sdk-tests)) — the same +battery every Restate SDK runs. It boots a real Restate runtime + a PHP image of the +standard test-services and drives them (needs JDK ≥ 21 + an AVX2 host): ```bash -make conformance # downloads the suite, builds the image, runs `default` +make conformance # downloads the suite, builds the images, runs `default` make conformance TEST_SUITE=all ``` -The `default` config passes **30/30** (8 documented exclusions). See -[`conformance/README.md`](conformance/README.md). +The `default` config passes **48 / 49** over the bidi (amphp) transport on a V7-enabled +runtime — `Cancellation` 6/6, `KillInvocation` 1/1, `Signals` 2/2 included. It also runs +in CI ([`conformance.yml`](.github/workflows/conformance.yml)); an AVX2-free offline +fallback is documented in [`conformance/README.md`](conformance/README.md). To try the example services live by hand: diff --git a/conformance/README.md b/conformance/README.md index f1922c9..abee12a 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -1,75 +1,84 @@ # Cross-SDK conformance This SDK is verified against the **official Restate conformance suite** -([`restatedev/sdk-test-suite`](https://github.com/restatedev/sdk-test-suite)) — the -same battery of tests every Restate SDK (TypeScript, Java, Python, Go, Rust) runs. +([`restatedev/e2e`](https://github.com/restatedev/e2e/tree/main/sdk-tests) — the actively +maintained successor to the now-archived `restatedev/sdk-test-suite`) — the same battery +of tests every Restate SDK (TypeScript, Java, Python, Go, Rust) runs. -`conformance/Services/` ports the standard contract test-services to PHP; the suite -boots a real Restate runtime + this image (via Testcontainers) and drives them. +`conformance/Services/` ports the standard contract test-services to PHP; the suite boots +a real Restate runtime + this image (via Testcontainers) and drives them. ## Result -`default` configuration — **30 / 30 passing, 0 failures** (Restate 1.5.2): - -State · ServiceToServiceCommunication (calls, idempotency keys, delayed sends) · -WorkflowAPI (durable promises) · Sleep · RunRetry (per-run retry policy) · -UserErrors (terminal vs retryable propagation across calls and side effects) · -ProxyRequestSigning (Ed25519 request identity) · Combinators (awakeable-or-timeout) · -SleepWithFailures · StopRuntime / KillRuntime (durability across restarts) · -UpgradeWithNewInvocation · KafkaIngress · Ingress (header pass-through). +The primary configuration drives the **bidirectional (amphp) transport on service +protocol V7** — the SDK's default server — against a V7-enabled runtime. The `default` +suite passes **48 / 49**, including `Cancellation` 6/6, `KillInvocation` 1/1, `Signals` +2/2, `Combinators` 9/9, `RunRetry` 3/3, `UserErrors` 10/10, and +`ServiceToServiceCommunication` 5/5. A few tests are excluded with documented reasons in `exclusions.yaml` (`awaitAny` combinator edge cases the Rust SDK also excludes; per-handler raw serde; one fan-out -ordering case; in-flight deployment upgrade; V7 scoped concurrency). - -### Bidirectional (amp) transport on service protocol V7 - -The `AmpStreamingServer` transport speaks service protocol **V7** (signals, signal-backed -awakeables, named signals, the Future-based suspension/`AwaitingOn`). Against a V7-enabled -runtime (`Dockerfile.restate-v7`) the `default` suite passes **48 / 49**, including -`Cancellation` 6/6, `KillInvocation` 1/1, `Signals` 2/2, `Combinators` 9/9, -`RunRetry` 3/3, `UserErrors` 10/10, and `ServiceToServiceCommunication` 5/5. The remaining -exclusions are the same documented gaps as above plus `ServiceToServiceScopeConcurrency` -(V7 scoped concurrency / virtual queues — not yet implemented). See the run instructions -below and `../docs/adr/0001-cancellation-over-bidirectional-streaming.md`. +ordering case; in-flight deployment upgrade; V7 scoped concurrency / virtual queues). See +`../docs/adr/0001-cancellation-over-bidirectional-streaming.md`. ## Run it -Requires Java 21 + Docker. On this host the Restate runtime is pinned to **1.5.2** -(the `:latest` image needs AVX2 and SIGILLs on older CPUs). +Requires **JDK ≥ 21** + Docker, and an **AVX2** host (the suite's Restate ≥ 1.6 runtime +SIGILLs on older CPUs — see the fallback below). `make conformance` downloads the suite +jar, builds the V7-enabled runtime (`Dockerfile.restate-v7`) and the bidi service image +(`Dockerfile.amp`), and runs the `default` config: ```bash -make conformance # downloads the suite jar, builds the image, runs `default` +make conformance # the default config, bidi V7 make conformance TEST_SUITE=all # every configuration -# or a single class while debugging: -java -jar build/restate-sdk-test-suite.jar run \ - --restate-container-image=docker.io/restatedev/restate:1.5.2 \ - --test-suite=default --sequential --test-name=State \ - localhost/restatedev/php-test-services:latest -``` - -`--sequential` is used because parallel container startup makes the runtime's h2c -discovery handshake flaky on a single host. The PHP server speaks HTTP/2 cleartext -(h2c), which the runtime uses for discovery + invocation. - -### Bidirectional (HTTP/2) streaming + service protocol V7 - -The `AmpStreamingServer` transport (`conformance/Dockerfile.amp` → -`localhost/restatedev/php-amp-test-services`) serves true bidi h2c and speaks service -protocol **V7** (signals, signal-backed awakeables, `AwaitingOnMessage`). Restate 1.7.0 -supports V7 but negotiates V6 by default, so build a V7-enabled runtime image first: - -```bash -docker build -f conformance/Dockerfile.restate-v7 -t localhost/restatedev/restate-v7:latest . -docker build -f conformance/Dockerfile.amp -t localhost/restatedev/php-amp-test-services:latest . +JAVA=/path/to/jdk21/bin/java make conformance # if `java` is < 21 +# or a single class while debugging: java -jar build/sdk-tests.jar run \ --restate-container-image=localhost/restatedev/restate-v7:latest \ --service-container-image=localhost/restatedev/php-amp-test-services:latest \ --test-suite=default --test-name=Cancellation \ --exclusions-file=conformance/exclusions.yaml \ - --image-pull-policy=CACHED --report-dir=build/conformance-amp-report --sequential + --image-pull-policy=CACHED --report-dir=build/conformance-report --sequential ``` +`--sequential` is used because parallel container startup makes the runtime's h2c +discovery handshake flaky on a single host. The PHP server speaks HTTP/2 cleartext (h2c), +which the runtime uses for discovery + bidirectional invocation. + +The runtime is built from `Dockerfile.restate-v7` (stock Restate + the +`experimental-enable-protocol-v7` flag): Restate 1.7.0 supports V7 but negotiates V6 by +default, on which the SDK's signal/awakeable model is invalid. + After a run, `build/conformance-report//exclusions.new.yaml` lists everything that failed/was skipped — copy entries into `exclusions.yaml` to baseline new gaps. + +## CI + +`.github/workflows/conformance.yml` runs the same `make conformance` on pushes to `main`, +weekly, on demand (`workflow_dispatch`), and on PRs labelled `conformance`. GitHub runners +have AVX2, so the ≥ 1.6 runtime works there. The suite is too heavy to gate every PR, so it +is not in the main `CI` workflow. + +## Offline fallback (AVX2-free hosts) + +The new suite's runtime needs AVX2. On an AVX2-free host, run the **archived** +[`restatedev/sdk-test-suite`](https://github.com/restatedev/sdk-test-suite) `v4.1` jar +against the **request/response Swoole** image (`conformance/Dockerfile` → +`php-test-services`) on the last AVX2-free runtime, **Restate 1.5.2**: + +```bash +curl -fSL -o build/restate-sdk-test-suite.jar \ + https://github.com/restatedev/sdk-test-suite/releases/download/v4.1/restate-sdk-test-suite.jar +docker build -f conformance/Dockerfile -t localhost/restatedev/php-test-services:latest . +java -jar build/restate-sdk-test-suite.jar run \ + --restate-container-image=docker.io/restatedev/restate:1.5.2 \ + --test-suite=default --sequential \ + --report-dir=build/conformance-report \ + localhost/restatedev/php-test-services:latest +``` + +This path serves request/response, so the V7-only cases (`Cancellation`, `KillInvocation`, +`Signals`) cannot pass and must be excluded — generate a fallback exclusions file from the +run's `exclusions.new.yaml` rather than reusing the bidi `exclusions.yaml`. It is frozen +(no new-runtime compatibility), but runs where the maintained suite's runtime will not.