From e071d270c879fb408c7c35899828ed92441b3f77 Mon Sep 17 00:00:00 2001 From: Luke Bond Date: Thu, 13 Aug 2026 11:50:28 +0100 Subject: [PATCH] Release 3.0.0 Bump all version references to 3.0.0 (Cargo.toml/lock, both charts and the restate-operator-crds dependency pin) and consolidate the unreleased notes into release-notes/v3.0.0.md. Major bump: upgrading from <= 2.8.1 requires a one-time CRD ownership hand-off now that the CRDs ship as templated resources. Also: - Add release notes for #158 (nested worker.invoker request-identity key) and #151 (Restate image references bumped to 1.7), which were missing. - Fix the release checklist in release-notes/README.md: bump the CRD subchart Chart.yaml and the dependency pin (previously listed only three files), and add the accept-draft-release and announce steps. --- Cargo.lock | 2 +- Cargo.toml | 2 +- charts/restate-operator-crds/Chart.yaml | 2 +- charts/restate-operator-helm/Chart.yaml | 4 +- release-notes/README.md | 11 +- .../166-poll-for-crds-on-startup.md | 50 ------- .../unreleased/167-standalone-crds-chart.md | 20 --- .../172-restatedeployment-deletion-hang.md | 86 ------------ .../unreleased/174-rollback-promotion.md | 127 ------------------ .../unreleased/iampolicymember-apiversion.md | 42 ------ release-notes/v3.0.0.md | 116 ++++++++++++++++ 11 files changed, 129 insertions(+), 333 deletions(-) delete mode 100644 release-notes/unreleased/166-poll-for-crds-on-startup.md delete mode 100644 release-notes/unreleased/167-standalone-crds-chart.md delete mode 100644 release-notes/unreleased/172-restatedeployment-deletion-hang.md delete mode 100644 release-notes/unreleased/174-rollback-promotion.md delete mode 100644 release-notes/unreleased/iampolicymember-apiversion.md create mode 100644 release-notes/v3.0.0.md diff --git a/Cargo.lock b/Cargo.lock index 5ae27d0..9bd5d0b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2425,7 +2425,7 @@ dependencies = [ [[package]] name = "restate-operator" -version = "2.8.1" +version = "3.0.0" dependencies = [ "actix-web", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index d7ebbbf..519fac6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "restate-operator" -version = "2.8.1" +version = "3.0.0" authors = ["restate.dev"] edition = "2024" rust-version = "1.92" diff --git a/charts/restate-operator-crds/Chart.yaml b/charts/restate-operator-crds/Chart.yaml index c3148c3..07ed04e 100644 --- a/charts/restate-operator-crds/Chart.yaml +++ b/charts/restate-operator-crds/Chart.yaml @@ -2,4 +2,4 @@ apiVersion: v2 name: restate-operator-crds description: Restate operator CustomResourceDefinitions (RestateCluster, RestateDeployment, RestateCloudEnvironment) type: application -version: "2.8.1" +version: "3.0.0" diff --git a/charts/restate-operator-helm/Chart.yaml b/charts/restate-operator-helm/Chart.yaml index cde2dae..6a55f22 100644 --- a/charts/restate-operator-helm/Chart.yaml +++ b/charts/restate-operator-helm/Chart.yaml @@ -2,10 +2,10 @@ apiVersion: v2 name: restate-operator-helm description: An operator for Restate clusters type: application -version: "2.9.0" +version: "3.0.0" dependencies: - name: restate-operator-crds - version: "2.8.1" + version: "3.0.0" repository: "file://../restate-operator-crds" condition: installCrds diff --git a/release-notes/README.md b/release-notes/README.md index 46bcc05..a57064b 100644 --- a/release-notes/README.md +++ b/release-notes/README.md @@ -66,17 +66,22 @@ When making a significant change that affects users, create a release note file Release checklist (in order): -1. **Bump the version in all three files, together, to the new version:** +1. **Bump the version in all these places, together, to the new version:** - `Cargo.toml` — `version = ""` - `Cargo.lock` — the `restate-operator` package entry. Do **not** hand-edit it; run `cargo check` after bumping `Cargo.toml` so the lockfile updates, then confirm the `restate-operator` entry reads `version = ""`. - - `charts/restate-operator-helm/Chart.yaml` — `version: ""` + - `charts/restate-operator-helm/Chart.yaml` — the chart `version:` **and** the + `restate-operator-crds` dependency `version:` (the two charts release in lockstep). + - `charts/restate-operator-crds/Chart.yaml` — `version: ""` 2. **Consolidate release notes** into `v.md` (see the detailed steps below). 3. **Delete the individual `unreleased/` files** after consolidation (keep `.gitkeep`). 4. **Merge the release PR** into `main`. 5. **Tag `v` on the merge commit** and push it to trigger the release - workflow, which builds and publishes the docker image and helm chart. + workflow, which builds and publishes the docker image and both helm charts + (`restate-operator-helm` and `restate-operator-crds`). +6. **Accept the draft GitHub release** once the workflow finishes. +7. **Announce the release** in the community Slack and Discord. The rest of this section details the consolidation step. diff --git a/release-notes/unreleased/166-poll-for-crds-on-startup.md b/release-notes/unreleased/166-poll-for-crds-on-startup.md deleted file mode 100644 index 6927495..0000000 --- a/release-notes/unreleased/166-poll-for-crds-on-startup.md +++ /dev/null @@ -1,50 +0,0 @@ -# Release Notes for Issue #166: Poll for CRDs on startup instead of exiting - -## Behavioral Change - -### What Changed - -Previously, if a required CRD (`RestateCluster`, `RestateDeployment`, or `RestateCloudEnvironment`) was not installed in the cluster when the operator started, the operator would log an error and exit immediately. The same applied to the optional `PodIdentityAssociation` CRD when an AWS pod identity association cluster was configured. - -Now, on startup the operator polls the apiserver's discovery endpoint for each required CRD and loops until the resource appears, emitting a warning every 10 seconds while it is missing. The operator no longer exits when a required CRD is absent — it waits for it to be applied and then begins reconciling. - -Discovery is used rather than listing the resource or reading the `CustomResourceDefinition` object: it needs no RBAC beyond what every authenticated client has, and a resource only shows up there once its CRD is established and the version is served. - -All of these now mean "not ready yet, keep waiting": a missing group/version (`404`), a missing resource within a served group/version, the responses that mean the apiserver cannot answer right now (`429` and `5xx`), and a failure to reach the apiserver at all (connection refused, timeouts). Every one of those routinely happens while a cluster is coming up, and exiting fixes none of them — it just turns a wait into a crashloop. This also covers the API-group listing the `RestateCluster` controller uses to detect which *optional* CRDs are installed, which previously exited on any failure and so ran before (and defeated) the CRD wait. - -Any other failure — a `401` or `403`, a bad kubeconfig — still logs an error and exits, since those need someone to change something. The corollary is that a permanently unreachable apiserver (a wrong CA, say) is now waited on rather than crashlooped; that is deliberate, because waiting is no longer silent (see below). - -Because a CRD that never arrives would otherwise leave the operator idle but healthy, the wait is now surfaced in three ways beyond the log line: - -- **A readiness endpoint.** The new `/ready` endpoint returns `200` only once every controller has its CRD and has started reconciling, and `503` otherwise, with the controllers still waiting listed in the body: - - ```json - {"ready":false,"pendingControllers":["RestateCluster","RestateDeployment"]} - ``` - - `/health` keeps its old always-`200` behaviour and is now used as a liveness probe: restarting the operator does not make a missing CRD appear, so waiting for a CRD must not get the pod killed. -- **A metric.** `restate_operator_crd_missing{crd="."}` is `1` while the operator is waiting for that CRD and `0` once it is available, so a CRD that never arrives can be alerted on (e.g. `restate_operator_crd_missing > 0 for 5m`). -- **A Kubernetes event.** A single `Warning` event with reason `WaitingForCRD` is recorded against the operator's own pod, naming every CRD being waited for at once: - - > Waiting for 3 CRDs to be installed before reconciling them: restateclusters.restate.dev, restatecloudenvironments.restate.dev, restatedeployments.restate.dev. - - The controllers wait independently, so this is reported centrally rather than from each of them — one event series for the operator instead of one per CRD. It is re-published every 10 seconds, which the API folds into that series' count, and a `Normal` `CRDsAvailable` event closes it out once everything is reconciling. Nothing is emitted at all if the CRDs land within the first 10 seconds, which is the common case for the GitOps race this change exists to handle. - -While waiting, the operator remains responsive to `SIGTERM`/`SIGINT`: the wait is aborted and the process shuts down promptly rather than running until the kubelet's grace period expires. The signal handlers are installed once, before the poll loop, so there is no window in which a signal would terminate the process outright. - -The `PodIdentityAssociation` check (when `aws-pod-identity-association-cluster` is set) still exits, since that is a configuration error rather than a missing-CRD race. - -### Why This Matters - -In GitOps and automated install flows the operator and the CRDs are often applied together, and the operator can come up before the CRDs are admitted. The old behaviour required careful ordering (or restarts) to get the operator running. With this change the operator simply waits for the CRDs to arrive, which matches how most Kubernetes operators behave and removes the need to restart the operator after installing the CRDs. - -### Impact on Users - -- **New deployments:** The operator can be installed in any order relative to its CRDs. It will start and begin reconciling as soon as the CRDs are available. Until then the pod is `NotReady`, so `kubectl get pods` and `kubectl describe pod` both say what is being waited on. -- **Existing deployments:** No change once the CRDs are present. If the CRDs are removed while the operator is running, the controller for that resource will keep retrying its watches (existing kube-rs behaviour) rather than exiting; readiness is not revoked once a controller has started. -- **Observability:** While waiting for a CRD, the operator logs a `WARN` line every 10 seconds naming the missing CRD, serves `503` on `/ready`, sets `restate_operator_crd_missing` to `1` for that CRD, and records one `WaitingForCRD` event on its own pod listing all of them (`kubectl describe pod -n `). -- **RBAC:** No new permissions are required. The discovery endpoint is readable by any authenticated client, and the operator already holds `create`/`patch` on `events.k8s.io` for the events it emits on the resources it manages. -- **Chart:** - - The readiness probe now targets `/ready` and a liveness probe targeting `/health` has been added. **Upgrade note:** the chart's `version` defaults to the chart version, so chart and image move together. If you pin `version` to an operator image older than this release, pin the chart too — an older image has no `/ready`, and the resulting `404` would keep the pod `NotReady` forever. - - The Deployment passes `OPERATOR_POD_NAME` and `OPERATOR_POD_UID` via the downward API so that operator-level events can be attached to the right pod. If you deploy the operator without the chart, set both (or neither — without them the operator logs and reports the metric as usual, but emits no events). - - The operator `Service` sets `publishNotReadyAddresses: true`. It exists only to serve `/metrics` and `/`, and a `NotReady` operator is exactly when `restate_operator_crd_missing` needs scraping, so its endpoint must not drop out of the `ServiceMonitor`'s targets. \ No newline at end of file diff --git a/release-notes/unreleased/167-standalone-crds-chart.md b/release-notes/unreleased/167-standalone-crds-chart.md deleted file mode 100644 index e245cdc..0000000 --- a/release-notes/unreleased/167-standalone-crds-chart.md +++ /dev/null @@ -1,20 +0,0 @@ -# Standalone `restate-operator-crds` chart - -2.8.1 shipped the CRDs through Helm's native `crds/` directory, which is install-only: once the CRDs are in the cluster, `helm upgrade` never touches them again. That bit us — after an operator upgrade a customer's new schema fields were silently pruned at admission until the CRDs were reapplied by hand. - -So we've pulled the CRDs into their own chart and gone back to templating them, which means upgrading the chart actually updates the schema: - -```bash -helm upgrade --install restate-operator-crds \ - oci://ghcr.io/restatedev/restate-operator-crds --version -``` - -They still carry `helm.sh/resource-policy: keep`, so `helm uninstall` won't take your CRDs (or the custom resources under them) down with it. - -`restate-operator-helm` keeps bundling this chart behind `installCrds` (default `true`), so nothing changes if you install the operator on its own. If you'd rather own the CRD lifecycle yourself — the nicer path for ArgoCD/Flux — install `restate-operator-crds` directly and set `installCrds=false` on the operator. - -## Upgrading - -If your CRDs came in through the old `crds/` directory (2.8.1 or earlier), Helm doesn't consider them managed yet, so the first upgrade needs a one-time ownership hand-off: either pass `--take-ownership` to the CRD chart (Helm >= 3.17.0), or, on older Helm, label/annotate the three CRDs with `app.kubernetes.io/managed-by=Helm` plus the `meta.helm.sh/release-name`/`-namespace` metadata. ArgoCD does this for you on sync. Either way, no custom resources are deleted. - -This doesn't touch #166 (the operator still exits if an optional CRD is missing at startup) — that's a separate binary-side fix. diff --git a/release-notes/unreleased/172-restatedeployment-deletion-hang.md b/release-notes/unreleased/172-restatedeployment-deletion-hang.md deleted file mode 100644 index 65fe6cb..0000000 --- a/release-notes/unreleased/172-restatedeployment-deletion-hang.md +++ /dev/null @@ -1,86 +0,0 @@ -# Release Notes for Issue #172: RestateDeployment deletion no longer hangs forever - -## Bug Fix - -### What Changed - -Deleting a `RestateDeployment` no longer blocks indefinitely on its own latest version. - -Cleanup previously asked Restate a single question — "is this deployment active?" — where -"active" meant *either* that a service still pointed at the deployment *or* that it had -unfinished invocations. During a rollout that conflation is harmless: a newer version -eventually takes over as the service's endpoint and the old one goes inactive. During a -deletion it was fatal. Nothing is coming to supersede the endpoint, so the latest version -stayed "active" forever, cleanup never deregistered it, `active_count` never reached zero, -and the finalizer requeued every 30 seconds without end. - -The operator now tracks the two facts separately and weighs them by why cleanup is -running. Being a service's current endpoint holds a version through a rollout but is -ignored during a deletion; only unfinished invocations — which drain on their own — can -hold a deletion. The version then goes through the normal drain, deregistration and -teardown path. - -Alongside that: - -- **Unfinished invocations not yet bound to a deployment now count.** Paused, queued and - scheduled work carries no `pinned_deployment_id`, so the old query scored it as zero and - a deletion could tear the endpoint out from under it. It is now attributed through the - target service. Only a deletion asks: that attribution costs a second scan of - `sys_invocation_status` and a per-row decode of the invocation target, and during a - rollout it can only ever name the deployment that is already the service's endpoint. The - reconcile path's query is unchanged. -- **A blocked deletion says what is blocking it.** The `DeploymentInUse` event now names - each version and its pinned/unpinned invocation counts instead of a generic message. -- **A blocked deletion backs off.** Retries start at the usual 30 seconds and stretch to - five minutes the longer the wait runs, so a deletion parked behind a scheduled invocation - days out stops re-running that query twice a minute for the duration. -- **Drain deadlines are now honoured.** The requeue interval derived from a version's - remove-at time was being discarded, because errors reach the controller's error policy - wrapped by the finalizer machinery; a short `drainDelaySeconds` cost up to 30 seconds per - version regardless. It is now unwrapped, and floored at one second so a sub-second - deadline cannot spin the reconciler. -- **A draining version keeps its autoscaler.** Removal of an inactive version's - operator-managed HPA moved to the point where the version is actually scaled to zero. - Previously a deletion stripped the HPA from every version on the first reconcile, while - those versions were still serving traffic for the whole drain window. - -### Why This Matters - -Before this fix, `kubectl delete restatedeployment` never returned for a -`RestateDeployment` whose services were still registered — which is the normal state of any -healthy deployment. The only workaround was to remove the finalizer by hand, which left the -deployment registered in Restate with no pods behind it. Namespace deletion inherited the -same hang. - -### Impact on Users - -- **Existing deployments:** no configuration change. Deletions that were previously wedged - will proceed the next time the operator reconciles them; ones whose finalizer was removed - by hand may have left a stale registration behind in Restate. -- **Deletion now takes at least `spec.restate.drainDelaySeconds` (default 300s).** The - latest version is put through the same drain as any superseded version, so teardown waits - out the drain window even when the deployment never served an invocation. This is the - interval that was previously unbounded. -- **Deletion still waits on unfinished invocations, and that wait has no upper bound.** - Scheduled invocations are the sharp edge: a delayed call whose execution time is days out - counts as unfinished and holds the deletion until it fires. The `DeploymentInUse` event - reports the counts so this is diagnosable. Bounding or skipping the wait is the job of the - planned `deletionPolicy` field, not of manual intervention. -- **New deployments:** no impact. - -### Migration Guidance - -None required. If a `RestateDeployment` is currently stuck deleting, upgrading the operator -is sufficient — no manual finalizer edits. - -To check what is holding a deletion: - -```bash -kubectl describe restatedeployment -n -# Warning FailedReconcile ... This RestateDeployment is backing active versions in -# Restate: greeter-7f9c4d (0 pinned, 3 unpinned invocations). ... -``` - -### Related Issues - -- Issue #172: RestateDeployment finalizer never completes diff --git a/release-notes/unreleased/174-rollback-promotion.md b/release-notes/unreleased/174-rollback-promotion.md deleted file mode 100644 index a8f7c8c..0000000 --- a/release-notes/unreleased/174-rollback-promotion.md +++ /dev/null @@ -1,127 +0,0 @@ -# Release Notes for Issue #174: Rolling back now moves Restate's routing - -## Bug Fix - -### What Changed - -Rolling a `RestateDeployment` back to a previously registered revision now makes that -revision the one Restate sends new invocations to, and `Ready=True` no longer claims -otherwise. - -Because ReplicaSets and Services are named by a content hash of the pod template, -`v1 -> v2 -> v1` re-adopts the original ReplicaSet, Service URL and Restate deployment id -rather than creating new ones. Restoring the Kubernetes revision was therefore never enough -on its own — Restate also has to be told that the restored deployment is latest again — and -two separate faults stopped that from happening: - -- **The decision asked the wrong question.** The operator skipped registration whenever the - recorded deployment was "active", where active meant *either* that a service pointed at it - *or* that it had unfinished invocations. A rolled-back version holding a single pinned - invocation satisfied the second half, so the operator did nothing at all and the newer - version kept serving. -- **When it did register, it could not promote.** Registration is a `POST /deployments` - against the unversioned admin path, which resolves to `AdminApiVersion::Unknown`, where - `force` defaults to *false* (changed in Restate v1.6.0). Re-registering an unchanged - endpoint on such a server returns `200` with the existing deployment id and changes - nothing. The operator read that as success. - -The operator now decides purely on whether Restate routes new invocations to the recorded -deployment, and re-registers with overwrite when it does not — which bumps that deployment's -service revisions past the current latest while keeping its deployment id, so invocations -already pinned to it are undisturbed. It then asks Restate what it actually routes before -reporting `Ready`. - -Alongside that: - -- **`force` is now always sent explicitly.** Its default depends on the admin API version a - request resolves to, and that default has already changed once underneath the operator. -- **Registration is verified, not assumed.** After registering, the operator checks - `GET /services` and confirms every service discovered at the endpoint resolves to this - deployment. A registration that made no routing change is no longer treated as a success. -- **The deployment id is recorded before the routing is confirmed.** The id is true either - way — it is what Restate holds for that endpoint — and recording it is what lets the next - reconcile plan a promotion. Confirming first would strand a registration that landed on an - endpoint Restate already knew but was not routing to, with nothing written down and every - subsequent reconcile planning the same plain registration again. -- **A promotion emits a `Promoted` event.** Overwriting is how Restate is asked to move - routing, and it also permits breaking schema changes and resets the deployment's - registration timestamp, so it leaves an audit trail rather than only a log line. -- **A foreign owner is reported, not fought.** If the desired deployment is superseded and - no version of this `RestateDeployment` holds the service either, something outside it has - registered those services. The operator refuses to force in that case and reports - `Ready=False` with reason `ForeignDeployment`. Forcing would start a promotion war, with - two controllers bumping revisions to take the service back indefinitely. -- **The ReplicaSet cache is prewarmed at startup.** That cache is how a rollback is told - apart from a foreign owner, and an unsynced one would answer "nothing of ours is latest", - which would park a healthy rollback at `Ready=False` until the cache filled. -- **Knative mode promotes too.** It used to return on the recorded deployment id alone, so - it could never promote on a rollback; it now plans against Restate's routing exactly as - ReplicaSet mode does, using its Configurations as the owned-version evidence. - -### Known limitation - -`latest_for_service`, the flag this decides on, is true when a deployment is latest for *any* -of its services. A deployment that is latest for some and superseded for others therefore -reads as current, and a rollback onto it does nothing. Reaching that state needs one version -to expose a strict subset of another's services. Resolving routing per service needs admin -reads this change deliberately does not add; it is tracked separately. - -### Why This Matters - -A rollback that restored the pods but not the routing looked entirely healthy: the -ReplicaSet scaled back up, `Ready` went `True`, and the deployment id in status was the -expected one. Only the traffic was still going to the version being rolled back *from* — -which, during an incident, is the version being rolled back away from. - -### Impact on Users - -- **`Ready=True` is stricter.** It now means Restate routes new invocations to the desired - revision, not merely that the pods are up. Deployments sitting in an inconsistent state — - including ones that have been inconsistent for a while — will report `Ready=False` with a - reason until it resolves. -- **On upgrade, inconsistent deployments are promoted on the next reconcile.** If a - `RestateDeployment`'s desired revision is not currently latest in Restate, the operator - will move routing to it. This is the fix working, but it is a routing change that happens - without being asked for, so upgrade at a time when that is acceptable. -- **An endpoint registered outside its `RestateDeployment` now stalls rather than being - silently tolerated.** Services that were pointed at a manually registered deployment, or at - one left behind by a decommissioned controller, report `Ready=False` with reason - `ForeignDeployment` on the first reconcile after the upgrade — the operator declines to - take them by force. Nothing is written and no routing changes; resolve it by removing the - other registrant so the `RestateDeployment`'s own endpoint becomes latest again. -- **Rollback requires the restored version's pods to serve discovery.** Overwriting - re-runs discovery against the endpoint, so a rollback to a revision whose image can no - longer be pulled now blocks at `Ready=False` instead of silently half-completing. -- **A promoted deployment's registration time is reset in Restate.** Overwriting replaces - `sys_deployment.created_at`, so a rolled-back deployment appears freshly registered in the - UI and in `sys_deployment`. The operator keeps no separate record of the original time. -- **`force: false` is now sent explicitly where the field was previously omitted.** No - behaviour change. Restate defaults the omitted flag by admin API version — *true* on `/v1` - and `/v2`, *false* on the unversioned router and `/v3` — but the operator only ever reaches - the unversioned router, where the default was already false: it builds each request as - `admin_url.join("/deployments")`, and a leading-slash path replaces the base path outright, - so a version prefix on `restate.register` could not survive into the request even if one - were configured. Sending the flag makes the operator's intent independent of that default - rather than changing what any current deployment does. -- **New deployments:** no impact beyond the above. - -### Migration Guidance - -None required. No CRD change, and no new annotation or status field. - -To check whether a deployment is in the inconsistent state this fixes, compare the id the -operator recorded against the one Restate routes to: - -```bash -kubectl get restatedeployment -n -o jsonpath='{.status.deploymentId}' -curl -s "$RESTATE_ADMIN/services/" | jq -r .deployment_id -``` - -If they differ, upgrading the operator resolves it on the next reconcile. - -### Related Issues - -- Issue #174: Support rolling RestateDeployments back to a previously registered revision -- Issue #146: content-hash RS naming doesn't handle niche rollback edge case -- restatedev/restate#5157: decouple deployment registration from selecting the latest - revision — the intended replacement for forced re-registration diff --git a/release-notes/unreleased/iampolicymember-apiversion.md b/release-notes/unreleased/iampolicymember-apiversion.md deleted file mode 100644 index 822b79b..0000000 --- a/release-notes/unreleased/iampolicymember-apiversion.md +++ /dev/null @@ -1,42 +0,0 @@ -# Release Notes: Fix IAMPolicyMember server-side apply wedge - -## Bug Fix - -### What Changed -The `restate-workload-identity` IAMPolicyMember CR is now applied with -`spec.resourceRef.apiVersion` set to `iam.cnrm.cloud.google.com/v1beta1`, -matching the value Config Connector defaults at creation time. - -### Why This Matters -The operator previously omitted `resourceRef.apiVersion` from its -server-side apply request. Config Connector defaults the field on the -initial create, but SSA's atomic-struct semantics then stripped that -defaulted value on every subsequent reconcile. Config Connector's -`deny-immutable-field-updates` admission webhook rejected the change -with: - -``` -admission webhook "deny-immutable-field-updates.cnrm.cloud.google.com" -denied the request: the IAMPolicyMember's spec is immutable -``` - -Reconcile looped on the failure and blocked unrelated `RestateCluster` -updates (image overrides, resource changes, etc.) from being applied. - -### Impact on Users -- Affects clusters using GCP workload identity binding via the operator - (BYOC installs on GKE / Config Connector). -- Existing installs already wedged on this error are unblocked once the - upgraded operator runs a reconcile. -- The webhook only fires on changes to immutable fields, and the value - the operator now emits matches what Config Connector defaulted at - create time, so the apply becomes a no-op for that field. - -### Migration Guidance -No user action required beyond upgrading the operator. Existing BYOC -installs need an operator chart version bump and a Nuon roll to pick -up the fix. - -### Related Issues -- restatedev/restate-cloud#833 (per-environment GCP service account - identity, where this bug was surfaced during dev canary validation) diff --git a/release-notes/v3.0.0.md b/release-notes/v3.0.0.md new file mode 100644 index 0000000..59af2d0 --- /dev/null +++ b/release-notes/v3.0.0.md @@ -0,0 +1,116 @@ +# Restate Operator v3.0.0 Release Notes + +See the [v2.8.1 release notes](https://github.com/restatedev/restate-operator/blob/main/release-notes/v2.8.1.md) for changes prior to this release. + +This is a major release: upgrading an existing install requires a **one-time CRD ownership hand-off** (see [Breaking Changes](#breaking-changes)), and several deployment-lifecycle fixes change behavior on the first reconcile after upgrade (see [Upgrade Notes](#upgrade-notes)). + +## Highlights + +- **CRDs now upgrade with the chart.** The three CRDs ship as a templated, standalone `restate-operator-crds` chart, so `helm upgrade` applies schema changes instead of silently skipping them. The first upgrade from ≤ 2.8.1 needs a one-time ownership hand-off. +- **The operator waits for its CRDs on startup** instead of exiting/crashlooping, and surfaces the wait via `/ready`, a metric, and a Kubernetes event. +- **`RestateDeployment` deletion no longer hangs forever** on its own latest version. +- **Rolling back now moves Restate's routing**, and `Ready=True` is stricter — inconsistent deployments are promoted on the next reconcile after upgrade. +- **GCP workload-identity** `IAMPolicyMember` server-side-apply wedge is fixed. + +## Table of Contents + +- [Upgrade Notes](#upgrade-notes) +- [Breaking Changes](#breaking-changes) +- [Improvements](#improvements) +- [Bug Fixes](#bug-fixes) + +## Upgrade Notes + +Read these before upgrading — each is expanded below. + +- **CRDs: one-time ownership hand-off.** The first upgrade to templated CRDs fails with `invalid ownership metadata` until you hand ownership to Helm. See [Breaking Changes](#breaking-changes). Nothing is deleted if you skip it — the upgrade just aborts. +- **Chart and image move in lockstep.** The readiness probe now targets `/ready`, which older operator images do not serve. If you pin the chart `version` to an image older than 3.0.0, pin the chart to a matching older version too — otherwise the `/ready` `404` leaves the pod `NotReady` forever. +- **Inconsistent `RestateDeployment`s are promoted on the next reconcile.** If a deployment's desired revision is not the one Restate currently routes to, the operator will move routing to it — a routing change that happens without being asked for. Upgrade when that is acceptable. Endpoints registered *outside* their `RestateDeployment` now stall at `Ready=False` with reason `ForeignDeployment` instead of being silently tolerated. +- **Deletion now waits out the drain window.** Deleting a `RestateDeployment` now takes at least `spec.restate.drainDelaySeconds` (default 300s) and still blocks on unfinished invocations (including scheduled ones, which have no upper bound). + +## Breaking Changes + +### CRD installation now upgrades with the chart — action required on first upgrade + +Through 2.8.1 the CRDs shipped through Helm's native `crds/` directory, which is install-only: once the CRDs were in the cluster, `helm upgrade` never touched them again. That bit a customer — after an operator upgrade, new schema fields were silently pruned at admission until the CRDs were reapplied by hand. + +The CRDs now live in their own `restate-operator-crds` chart, rendered as ordinary templates, so upgrading the chart applies the new schema. They carry `helm.sh/resource-policy: keep`, so `helm uninstall` will not take the CRDs (or the custom resources under them) down with it. `restate-operator-helm` bundles this chart behind `installCrds` (default `true`), so a standalone operator install is unchanged. The CRD chart is also published on its own: + +```bash +helm upgrade --install restate-operator-crds \ + oci://ghcr.io/restatedev/restate-operator-crds --version 3.0.0 +``` + +**One-time ownership hand-off.** CRDs installed through the old `crds/` directory (2.8.1 or earlier) carry no Helm ownership metadata, so the first upgrade to the templated CRDs fails with `invalid ownership metadata` until ownership is handed over. Pick one: + +- **Helm ≥ 3.17** — add `--take-ownership` to the upgrade. Where you pass it depends on how you run the CRDs: + - **Bundled (default, `installCrds=true`):** `helm upgrade restate-operator … --take-ownership` + - **Standalone CRD chart (`installCrds=false`):** `helm upgrade restate-operator-crds … --take-ownership` +- **Older Helm** — set the `app.kubernetes.io/managed-by=Helm` label plus the `meta.helm.sh/release-name` and `meta.helm.sh/release-namespace` annotations on the three CRDs, for the release that owns them. +- **ArgoCD / Flux** — no action needed; they adopt the resources on sync and never hit Helm's ownership gate. + +Either way **no custom resources are deleted** — the failure aborts the upgrade, it does not cascade. + +**If you don't want Helm upgrading your CRDs.** CRDs are cluster-scoped, so on a shared or multi-tenant cluster a `helm upgrade` now mutates a cluster-wide object that can affect other teams. To gate CRD changes behind manual review — or if you already manage CRDs via GitOps — set `installCrds=false` on the operator and own the CRD lifecycle yourself: apply `crd/*.yaml` directly, or install the standalone `restate-operator-crds` chart on your own cadence. This is the recommended path for ArgoCD/Flux. + +## Improvements + +### Startup: wait for CRDs instead of exiting (#166, #169) + +If a required CRD (`RestateCluster`, `RestateDeployment`, `RestateCloudEnvironment`) is not yet installed when the operator starts, it no longer logs an error and exits. It now polls the apiserver's discovery endpoint and waits for each CRD to appear, then begins reconciling. This matches how most Kubernetes operators behave and removes the ordering/restart dance in GitOps flows where the operator and its CRDs are applied together. + +Missing group/versions, `404`/`429`/`5xx` responses, and an unreachable apiserver are all treated as "not ready yet, keep waiting". Genuine configuration errors (`401`/`403`, bad kubeconfig) still log and exit. The `PodIdentityAssociation` check (when `aws-pod-identity-association-cluster` is set) still exits, since that is a configuration error rather than a race. + +The wait is observable three ways: + +- **Readiness endpoint.** `/ready` returns `200` only once every controller has its CRD and has started reconciling, otherwise `503` with the pending controllers listed: + + ```json + {"ready":false,"pendingControllers":["RestateCluster","RestateDeployment"]} + ``` + + `/health` keeps its always-`200` behavior and is now the liveness probe. +- **Metric.** `restate_operator_crd_missing{crd="."}` is `1` while waiting and `0` once available — alert with e.g. `restate_operator_crd_missing > 0 for 5m`. +- **Event.** A single `Warning`/`WaitingForCRD` event is recorded against the operator's own pod naming every CRD being waited on, closed out by a `Normal`/`CRDsAvailable` event. Nothing is emitted if the CRDs land within the first 10 seconds (the common case). + +No new RBAC is required. + +### Helm Chart + +- The CRDs are delivered by the templated, standalone `restate-operator-crds` chart — see [Breaking Changes](#breaking-changes). +- The readiness probe now targets `/ready`; a liveness probe targeting `/health` was added. **See the [chart/image lockstep note](#upgrade-notes).** +- The Deployment passes `OPERATOR_POD_NAME` and `OPERATOR_POD_UID` via the downward API so operator-level events attach to the right pod. If you deploy the operator without the chart, set both (or neither — without them the operator still logs and reports the metric, but emits no events). +- The operator `Service` sets `publishNotReadyAddresses: true`, so a `NotReady` operator — exactly when `restate_operator_crd_missing` needs scraping — stays in the `ServiceMonitor`'s targets. + +### Examples & Docs + +- Example manifests, Pkl templates, and docs now reference Restate server 1.7 (#151). No forced default: the operator does not hardcode a server image, so existing clusters keep whatever version their manifests pin. + +## Bug Fixes + +### RestateDeployment + +**Deletion no longer hangs forever (#172).** Cleanup previously conflated "a service still points at this deployment" with "it has unfinished invocations", so during a deletion the latest version stayed "active" forever, the finalizer never completed, and `kubectl delete restatedeployment` never returned (namespace deletion inherited the hang). The operator now tracks the two facts separately: being a service's current endpoint holds a version through a rollout but is ignored during a deletion; only unfinished invocations can hold a deletion, and they drain on their own. + +- Deletion now takes at least `spec.restate.drainDelaySeconds` (default 300s), and still blocks on unfinished invocations — scheduled invocations days out have no upper bound. A blocked deletion now backs off (30s → 5m) and the `DeploymentInUse` event names each version and its pinned/unpinned invocation counts. +- No migration needed: a currently-stuck deletion proceeds on the next reconcile after upgrade (no manual finalizer edits). To see what is holding a deletion: + + ```bash + kubectl describe restatedeployment -n + ``` + +**Rolling back now moves Restate's routing (#174).** Because ReplicaSets/Services are named by a content hash, `v1 → v2 → v1` re-adopts the original Restate deployment id, so restoring the Kubernetes revision was never enough on its own — Restate also has to be told the restored deployment is latest again. The operator now decides on whether Restate routes new invocations to the recorded deployment, re-registers with overwrite when it does not (preserving the deployment id and pinned invocations), verifies routing via `GET /services` before reporting `Ready`, and emits a `Promoted` event. + +- `Ready=True` is stricter: it now means Restate routes new invocations to the desired revision, not merely that the pods are up. See the [routing-promotion upgrade note](#upgrade-notes) for the on-upgrade impact (unrequested promotions, `ForeignDeployment` stalls, and that a promoted deployment's registration timestamp is reset in Restate). +- To check for the inconsistent state this fixes: + + ```bash + kubectl get restatedeployment -n -o jsonpath='{.status.deploymentId}' + curl -s "$RESTATE_ADMIN/services/" | jq -r .deployment_id + ``` + +### RestateCluster + +**Request-identity config emitted under the nested `worker.invoker` key (#158).** With request signing enabled (`spec.security.requestSigningPrivateKey`), the operator now emits `RESTATE_WORKER__INVOKER__REQUEST_IDENTITY_PRIVATE_KEY_PEM_FILE` (→ `worker.invoker.request-identity-private-key-pem-file`) instead of the flat alias restate-server deprecated. Only the env var name changes; signing behavior is unchanged. No action required — this removes the startup deprecation warning and keeps working once restate-server drops the alias. + +**GCP workload-identity `IAMPolicyMember` SSA wedge fixed (#136).** The `restate-workload-identity` `IAMPolicyMember` CR is now applied with `spec.resourceRef.apiVersion` set, matching what Config Connector defaults at creation. Previously SSA stripped the defaulted value on every reconcile, Config Connector's `deny-immutable-field-updates` webhook rejected it, and the reconcile loop blocked unrelated `RestateCluster` updates. Affects BYOC installs on GKE / Config Connector; existing wedged installs unblock once the upgraded operator reconciles.