From 83c25cca6a5e9d2205c102410b452eb78fc50a00 Mon Sep 17 00:00:00 2001 From: Leif Battermann Date: Mon, 11 May 2026 08:30:15 +0200 Subject: [PATCH 1/5] WPB-22963 migrate domain registration data to postgres (#5195) --- changelog.d/0-release-notes/WPB-22963 | 5 + .../background-worker/configmap.yaml | 5 +- .../wire-server/templates/brig/configmap.yaml | 1 + charts/wire-server/values.yaml | 11 +- .../src/developer/reference/config-options.md | 123 ++++++------ hack/helm_vars/common.yaml.gotmpl | 1 + hack/helm_vars/wire-server/values.yaml.gotmpl | 5 +- integration/integration.cabal | 1 + .../test/Test/Migration/DomainRegistration.hs | 186 +++++++++++++++++ integration/test/Testlib/ModService.hs | 3 +- .../wire-api/src/Wire/API/PostgresMarshall.hs | 59 ++++++ .../20260420134603-domain_registration.sql | 27 +++ .../src/Wire/DomainRegistrationStore.hs | 74 +++++++ .../Wire/DomainRegistrationStore/DualWrite.hs | 59 ++++++ .../Wire/DomainRegistrationStore/Migration.hs | 189 ++++++++++++++++++ .../Wire/DomainRegistrationStore/Postgres.hs | 125 ++++++++++++ .../Cassandra.hs | 45 ++--- .../DualWrite.hs | 49 +++++ .../Postgres.hs | 100 +++++++++ .../wire-subsystems/src/Wire/MigrationLock.hs | 11 +- .../src/Wire/PostgresMigrationOpts.hs | 4 +- libs/wire-subsystems/wire-subsystems.cabal | 5 + postgres-schema.sql | 69 ++++++- .../background-worker.integration.yaml | 2 + .../src/Wire/BackgroundWorker.hs | 10 +- .../src/Wire/BackgroundWorker/Options.hs | 1 + .../src/Wire/PostgresMigrations.hs | 18 ++ .../Wire/BackendNotificationPusherSpec.hs | 6 +- .../background-worker/test/Test/Wire/Util.hs | 3 +- services/brig/brig.integration.yaml | 6 + services/brig/src/Brig/App.hs | 8 +- .../brig/src/Brig/CanonicalInterpreter.hs | 27 ++- services/brig/src/Brig/Options.hs | 2 + services/galley/galley.integration.yaml | 1 + 34 files changed, 1132 insertions(+), 109 deletions(-) create mode 100644 changelog.d/0-release-notes/WPB-22963 create mode 100644 integration/test/Test/Migration/DomainRegistration.hs create mode 100644 libs/wire-subsystems/postgres-migrations/20260420134603-domain_registration.sql create mode 100644 libs/wire-subsystems/src/Wire/DomainRegistrationStore/DualWrite.hs create mode 100644 libs/wire-subsystems/src/Wire/DomainRegistrationStore/Migration.hs create mode 100644 libs/wire-subsystems/src/Wire/DomainRegistrationStore/Postgres.hs create mode 100644 libs/wire-subsystems/src/Wire/DomainVerificationChallengeStore/DualWrite.hs create mode 100644 libs/wire-subsystems/src/Wire/DomainVerificationChallengeStore/Postgres.hs diff --git a/changelog.d/0-release-notes/WPB-22963 b/changelog.d/0-release-notes/WPB-22963 new file mode 100644 index 00000000000..db32715b668 --- /dev/null +++ b/changelog.d/0-release-notes/WPB-22963 @@ -0,0 +1,5 @@ +- `postgresMigration` now has a single source of truth in the Galley chart values. Galley, Brig, and background-worker all read their PostgreSQL migration settings from there. +- If your deployment overrides the full `postgresMigration` object, add the new `domainRegistration` field to that override. Otherwise services may fail to start because the config is incomplete. +- To migrate domain registration data to PostgreSQL, set `postgresMigration.domainRegistration` to `migration-to-postgresql`, run the background-worker migration with `migrateDomainRegistration: true`, and switch the setting to `postgresql` after completion. +- The domain registration migration covers these Cassandra tables: + `domain_registration`, `domain_registration_by_team`, and `domain_registration_challenge`. diff --git a/charts/wire-server/templates/background-worker/configmap.yaml b/charts/wire-server/templates/background-worker/configmap.yaml index 7c8ef9aee43..594bd5d1a92 100644 --- a/charts/wire-server/templates/background-worker/configmap.yaml +++ b/charts/wire-server/templates/background-worker/configmap.yaml @@ -83,6 +83,7 @@ data: migrateConversations: {{ .migrateConversations }} migrateConversationCodes: {{ .migrateConversationCodes }} migrateTeamFeatures: {{ .migrateTeamFeatures }} + migrateDomainRegistration: {{ .migrateDomainRegistration }} migrateConversationsOptions: {{toYaml .migrateConversationsOptions | indent 6 }} @@ -92,7 +93,7 @@ data: backgroundJobs: {{ toYaml . | indent 6 }} {{- end }} - {{- if .postgresMigration }} - postgresMigration: {{- toYaml .postgresMigration | nindent 6 }} + {{- if $.Values.galley.config.postgresMigration }} + postgresMigration: {{- toYaml $.Values.galley.config.postgresMigration | nindent 6 }} {{- end }} {{- end }} diff --git a/charts/wire-server/templates/brig/configmap.yaml b/charts/wire-server/templates/brig/configmap.yaml index 66435c8d799..ad9ee08bf8f 100644 --- a/charts/wire-server/templates/brig/configmap.yaml +++ b/charts/wire-server/templates/brig/configmap.yaml @@ -37,6 +37,7 @@ data: {{- if hasKey $.Values.brig.secrets "pgPassword" }} postgresqlPassword: /etc/wire/brig/secrets/pgPassword {{- end }} + postgresMigration: {{- toYaml $.Values.galley.config.postgresMigration | nindent 6 }} elasticsearch: url: {{ .elasticsearch.scheme }}://{{ .elasticsearch.host }}:{{ .elasticsearch.port }} diff --git a/charts/wire-server/values.yaml b/charts/wire-server/values.yaml index 77dd4b6953d..fc50c17dfad 100644 --- a/charts/wire-server/values.yaml +++ b/charts/wire-server/values.yaml @@ -89,6 +89,7 @@ galley: conversation: cassandra conversationCodes: cassandra teamFeatures: cassandra + domainRegistration: cassandra settings: httpPoolSize: 128 maxTeamSize: 10000 @@ -962,6 +963,10 @@ background-worker: # It's important to set `settings.postgresMigration.teamFeatures` to `migration-to-postgresql` # before starting the migration. migrateTeamFeatures: false + # This will start the migration of domain registration data. + # It's important to set `settings.postgresMigration.domainRegistration` to `migration-to-postgresql` + # before starting the migration. + migrateDomainRegistration: false backendNotificationPusher: pushBackoffMinWait: 10000 # in microseconds, so 10ms @@ -977,12 +982,6 @@ background-worker: # Total attempts, including the first try maxAttempts: 3 - # Controls where conversation data is stored/accessed - postgresMigration: - conversation: cassandra - conversationCodes: cassandra - teamFeatures: cassandra - secrets: {} diff --git a/docs/src/developer/reference/config-options.md b/docs/src/developer/reference/config-options.md index edda1f74f53..4a269a7fe45 100644 --- a/docs/src/developer/reference/config-options.md +++ b/docs/src/developer/reference/config-options.md @@ -1877,12 +1877,13 @@ parameters](https://www.postgresql.org/docs/17/libpq-connect.html#LIBPQ-PARAMKEY The `postgresqlPassword` file is read by `brig`, `galley`, and `background-worker`. Its content is used as `password` field. -### Using PostgreSQL for storing conversation data +### Using PostgreSQL for storing Cassandra-backed data #### New Installations -For new installations, configure both `galley` and `background-worker` to use -PostgreSQL for conversation data: +For new installations, configure `galley.config.postgresMigration` to use +PostgreSQL for migrated Cassandra-backed data. In the Helm charts, this is the single source +of truth and is consumed by `galley`, `brig`, and `background-worker`: ```yaml galley: @@ -1891,35 +1892,43 @@ galley: conversation: postgresql conversationCodes: postgresql teamFeatures: postgresql + domainRegistration: postgresql background-worker: config: - postgresMigration: - conversation: postgresql - conversationCodes: postgresql - teamFeatures: postgresql migrateConversations: false + migrateConversationCodes: false + migrateTeamFeatures: false + migrateDomainRegistration: false ``` #### Migration for existing installations -Existing installations should migrate conversation data to PostgreSQL from -Cassandra. This is necessary for channel search and management of channels from -the team-management UI. It is highly recommended to take a backup of the Galley -Cassandra before triggering the migration. +Existing installations should migrate Cassandra-backed data to PostgreSQL over +time. For conversations, this is necessary for channel search and management of +channels from the team-management UI. It is highly recommended to take a backup +of the affected Cassandra data before triggering a migration. Migrations are independent and can be run separately, in batches, or all at once. This is expected, because migrations will be released over time. The -pattern below applies per store. Use it for `conversation` and -`conversationCodes` now, and for future stores as they are added. +pattern below applies per `postgresMigration` setting. A single setting may +cover multiple Cassandra tables, depending on the store. -**Migration pattern per store(s)** +The current settings and their background-worker flags are: -1. Prepare the selected store(s) for migration by setting - `postgresMigration.` to `migration-to-postgresql`. This enables the - migration interpreter for that store, which ensures data is written to +- `conversation` -> `migrateConversations` +- `conversationCodes` -> `migrateConversationCodes` +- `teamFeatures` -> `migrateTeamFeatures` +- `domainRegistration` -> `migrateDomainRegistration` + +**Migration pattern per migration setting** + +1. Prepare the selected migration setting(s) for migration by setting + `postgresMigration.` to `migration-to-postgresql`. This enables the + migration interpreter for that setting, which ensures data is written to PostgreSQL (store-specific details are handled internally). - The configuration must be consistent across `galley` and - `background-worker`. + In the Helm charts, configure this only under `galley.config.postgresMigration`. + `brig` and `background-worker` consume the same settings from there, so the + migration configuration remains consistent across services. ```yaml galley: @@ -1928,21 +1937,19 @@ pattern below applies per store. Use it for `conversation` and conversation: migration-to-postgresql conversationCodes: migration-to-postgresql teamFeatures: migration-to-postgresql + domainRegistration: cassandra background-worker: config: - postgresMigration: - conversation: migration-to-postgresql - conversationCodes: migration-to-postgresql - teamFeatures: migration-to-postgresql - migrateConversations: false - migrateConversationCodes: false - migrateTeamFeatures: false + migrateConversations: false + migrateConversationCodes: false + migrateTeamFeatures: false + migrateDomainRegistration: false ``` - This change should restart all the galley pods, and new writes will follow - the migration interpreter. + This change should restart the affected pods, and new writes will follow the + migration interpreter. -2. Run the backfill for the selected store(s) via background-worker. +2. Run the backfill for the selected migration setting(s) via background-worker. ```yaml background-worker: @@ -1950,6 +1957,7 @@ pattern below applies per store. Use it for `conversation` and migrateConversations: true migrateConversationCodes: true migrateTeamFeatures: true + migrateDomainRegistration: true ``` During migration, Cassandra rows are not deleted. Writes and migration share @@ -1957,13 +1965,18 @@ pattern below applies per store. Use it for `conversation` and deferred to keep rollback options and to remove Cassandra only after a full cutover to PostgreSQL-only. - Wait for the store-specific migration metrics to reach `1.0`. For - conversations: `wire_local_convs_migration_finished` and - `wire_user_remote_convs_migration_finished`. For conversation codes: - `wire_conv_codes_migration_finished`. + Wait for the setting-specific migration metrics to reach `1.0`. Metric names + are store-specific. Current examples are: -3. Cut over reads and writes to PostgreSQL for the selected store(s). This - configuration must be used from now on for every new release. + - `conversation`: `wire_local_convs_migration_finished` and + `wire_user_remote_convs_migration_finished` + - `conversationCodes`: `wire_conv_codes_migration_finished` + - `teamFeatures`: `wire_team_features_migration_finished` + - `domainRegistration`: `wire_domain_registration_migration_finished` + +3. Cut over reads and writes to PostgreSQL for the selected migration + setting(s). This configuration must be used from now on for every new + release. ```yaml galley: @@ -1972,24 +1985,26 @@ pattern below applies per store. Use it for `conversation` and conversation: postgresql conversationCodes: postgresql teamFeatures: postgresql + domainRegistration: cassandra background-worker: config: - postgresMigration: - conversation: postgresql - conversationCodes: postgresql - teamFeatures: postgresql - migrateConversations: false - migrateConversationCodes: false - migrateTeamFeatures: false + migrateConversations: false + migrateConversationCodes: false + migrateTeamFeatures: false + migrateDomainRegistration: false ``` **How to run migrations independently or in batches** -- To migrate a single store, set only that store’s `postgresMigration.` - and `migrate` flags; leave others unchanged. -- To migrate a batch, set multiple stores to `migration-to-postgresql` and - enable only the matching `migrate` flags together. +- To migrate a single setting, set only that setting’s + `postgresMigration.` and matching `migrate<...>` flag; leave + others unchanged. +- To migrate a batch, set multiple settings to `migration-to-postgresql` and + enable only the matching `migrate<...>` flags together. - To reduce load, run large stores alone and group small stores together. +- Some settings cover multiple Cassandra tables. For example, + `postgresMigration.domainRegistration` covers `domain_registration`, + `domain_registration_by_team`, and `domain_registration_challenge`. ## Configure Cells @@ -2061,15 +2076,11 @@ postgresqlPool: agingTimeout: 1d idlenessTimeout: 10m -# Controls where conversation data is read/written -postgresMigration: - # Valid: cassandra | migration-to-postgresql | postgresql - conversation: postgresql - conversationCodes: postgresql - teamFeatures: postgresql - -# Start the migration worker when true +# Start migration workers when true migrateConversations: false +migrateConversationCodes: false +migrateTeamFeatures: false +migrateDomainRegistration: false # Background jobs consumer backgroundJobs: @@ -2089,7 +2100,7 @@ Notes - `postgresql` values follow libpq keywords; password is sourced via `secrets.pgPassword`. - RabbitMQ admin fields (`adminHost`, `adminPort`) are templated only when `config.enableFederation` is true. -- `postgresMigration.` must match between `galley` and `background-worker` during migration phases. -- `migrateConversations: true` triggers the conversation migration job; leave it `false` for new installs and after migration. +- In the Helm charts, `background-worker` reads `postgresMigration` from `galley.config.postgresMigration`. +- The `migrate...` flags control the corresponding PostgreSQL backfill jobs for the current migration settings; leave them `false` for new installs and after migration. - `concurrency`, `jobTimeout`, and `maxAttempts` control parallelism and retry behavior of the consumer. - `brig` and `gundeck` endpoints default to in-cluster services; override via `background-worker.config.brig` and `.gundeck` if your service DNS/ports differ. diff --git a/hack/helm_vars/common.yaml.gotmpl b/hack/helm_vars/common.yaml.gotmpl index 7cd9bb5fac5..17b0dbd6005 100644 --- a/hack/helm_vars/common.yaml.gotmpl +++ b/hack/helm_vars/common.yaml.gotmpl @@ -17,6 +17,7 @@ dynBackendDomain3: dynamic-backend-3.{{ requiredEnv "NAMESPACE_1" }}.svc.cluster conversationStore: {{ $preferredStore }} conversationCodesStore: {{ $preferredStore }} teamFeaturesStore: {{ $preferredStore }} +domainRegistration: {{ $preferredStore }} {{- if (eq (env "UPLOAD_XML_S3_BASE_URL") "") }} uploadXml: {} diff --git a/hack/helm_vars/wire-server/values.yaml.gotmpl b/hack/helm_vars/wire-server/values.yaml.gotmpl index fe0a19d0682..8ed568c72ca 100644 --- a/hack/helm_vars/wire-server/values.yaml.gotmpl +++ b/hack/helm_vars/wire-server/values.yaml.gotmpl @@ -304,6 +304,7 @@ galley: conversation: {{ .Values.conversationStore }} conversationCodes: {{ .Values.conversationCodesStore }} teamFeatures: {{ .Values.teamFeaturesStore }} + domainRegistration: {{ .Values.domainRegistration }} settings: maxConvAndTeamSize: 16 maxTeamSize: 32 @@ -661,10 +662,6 @@ background-worker: name: "cassandra-jks-keystore" key: "ca.crt" {{- end }} - postgresMigration: - conversation: {{ .Values.conversationStore }} - conversationCodes: {{ .Values.conversationCodesStore }} - teamFeatures: {{ .Values.teamFeaturesStore }} rabbitmq: port: 5671 adminPort: 15671 diff --git a/integration/integration.cabal b/integration/integration.cabal index 2a4fb71b60d..49fc43bcf5a 100644 --- a/integration/integration.cabal +++ b/integration/integration.cabal @@ -176,6 +176,7 @@ library Test.MessageTimer Test.Migration.Conversation Test.Migration.ConversationCodes + Test.Migration.DomainRegistration Test.Migration.TeamFeatures Test.Migration.Util Test.MLS diff --git a/integration/test/Test/Migration/DomainRegistration.hs b/integration/test/Test/Migration/DomainRegistration.hs new file mode 100644 index 00000000000..3a7915b3242 --- /dev/null +++ b/integration/test/Test/Migration/DomainRegistration.hs @@ -0,0 +1,186 @@ +module Test.Migration.DomainRegistration (testDomainRegistrationMigration) where + +import qualified API.Brig as Brig +import qualified API.BrigInternal as BrigInternal +import API.Common +import qualified API.GalleyInternal as GalleyInternal +import Control.Error (MaybeT (..)) +import Control.Monad.Codensity +import Control.Monad.Reader +import SetupHelpers +import Test.DNSMock +import Test.Migration.Util (waitForMigration) +import Testlib.Prelude +import Testlib.ResourcePool + +data DomainRegistrationTestCase = TeamFlow TeamStep | OnPremFlow OnPremStep + +type EmailDomain = String + +type AuthToken = String + +type TeamId = String + +type Owner = Value + +type Config = Value + +type OwnershipToken = String + +data OnPremStep + = PreAuthorization EmailDomain + | SetupChallenge EmailDomain + | VerifyDomain EmailDomain ChallengeSetup + | PostConfig EmailDomain AuthToken Config + | OnPremVerify EmailDomain Config + | OnPremSuccess EmailDomain Config + +data TeamStep + = TeamSetupChallenge (Owner, TeamId) EmailDomain + | TeamVerifyDomain (Owner, TeamId) EmailDomain ChallengeSetup + | TeamAuthorizeTeam (Owner, TeamId) EmailDomain OwnershipToken + | TeamUpdateConfig (Owner, TeamId) EmailDomain + | TeamSuccess (Owner, TeamId) EmailDomain + +testDomainRegistrationMigration :: (HasCallStack) => App () +testDomainRegistrationMigration = do + resourcePool <- asks (.resourcePool) + runCodensity (acquireResources 1 resourcePool) $ \[backend] -> do + let domain = backend.berDomain + let initTestCases = do + [t1, t2, t3, t4] <- replicateM 4 $ OnPremFlow . PreAuthorization <$> randomDomain + [t5, t6, t7, t8] <- replicateM 4 $ do + (owner, tid, _) <- createTeam domain 1 + GalleyInternal.setTeamFeatureLockStatus owner tid "domainRegistration" "unlocked" + GalleyInternal.setTeamFeatureStatus owner tid "domainRegistration" "enabled" >>= assertSuccess + TeamFlow . TeamSetupChallenge (owner, tid) <$> randomDomain + + sequence + [ pure t1, + runStep domain t2, + runStep domain t3 >>= runStep domain, + runStep domain t4 >>= runStep domain >>= runStep domain, + pure t5, + runStep domain t6, + runStep domain t7 >>= runStep domain, + runStep domain t8 >>= runStep domain >>= runStep domain + ] + + testCases1 <- runCodensity (startDynamicBackend backend (conf "cassandra" False)) . const $ do + testCases0 <- initTestCases + nextStepCases <- for testCases0 (runStep domain) + newCases <- initTestCases + pure $ nextStepCases <> newCases + + testCases2 <- runCodensity (startDynamicBackend backend (conf "migration-to-postgresql" False)) . const $ do + nextStepCases <- for testCases1 (runStep domain) + newCases <- initTestCases + pure $ nextStepCases <> newCases + + testCases3 <- runCodensity (startDynamicBackend backend (conf "migration-to-postgresql" True)) . const $ do + nextStepCases <- for testCases2 (runStep domain) + newCases <- initTestCases + waitForMigration domain counterName + + nextStepCases' <- for (nextStepCases <> newCases) (runStep domain) + newCases' <- initTestCases + pure $ nextStepCases' <> newCases' + + runCodensity (startDynamicBackend backend (conf "postgresql" False)) . const $ do + for_ testCases3 (runAll domain) + where + runStep :: (HasCallStack) => String -> DomainRegistrationTestCase -> App DomainRegistrationTestCase + -- TEAM FLOW + runStep domain (TeamFlow (TeamSetupChallenge team emailDomain)) = do + challenge <- setupChallenge domain emailDomain + registerTechnitiumRecord challenge.technitiumToken emailDomain ("wire-domain." <> emailDomain) "TXT" challenge.dnsToken + pure $ TeamFlow $ TeamVerifyDomain team emailDomain challenge + runStep _ (TeamFlow (TeamVerifyDomain team@(owner, _) emailDomain challenge)) = do + token <- bindResponse (Brig.verifyDomainForTeam owner emailDomain challenge.challengeId challenge.challengeToken) $ \resp -> do + resp.status `shouldMatchInt` 200 + resp.json %. "domain_ownership_token" & asString + pure $ TeamFlow $ TeamAuthorizeTeam team emailDomain token + runStep _ (TeamFlow (TeamAuthorizeTeam team@(owner, _) emailDomain token)) = do + Brig.authorizeTeam owner emailDomain token >>= assertStatus 200 + pure $ TeamFlow $ TeamUpdateConfig team emailDomain + runStep domain (TeamFlow (TeamUpdateConfig team@(owner, tid) emailDomain)) = do + bindResponse (Brig.updateTeamInvite owner emailDomain (object ["team_invite" .= "team", "team" .= tid])) $ \res -> do + res.status `shouldMatchInt` 200 + verifyTeamConfig domain tid emailDomain + pure $ TeamFlow $ TeamSuccess team emailDomain + runStep domain (TeamFlow (TeamSuccess team@(_, tid) emailDomain)) = do + verifyTeamConfig domain tid emailDomain + pure $ TeamFlow $ TeamSuccess team emailDomain + -- ON PREM FLOW + runStep domain (OnPremFlow (PreAuthorization emailDomain)) = do + BrigInternal.domainRegistrationPreAuthorize domain emailDomain >>= assertStatus 204 + pure $ OnPremFlow $ SetupChallenge emailDomain + runStep domain (OnPremFlow (SetupChallenge emailDomain)) = do + challenge <- setupChallenge domain emailDomain + registerTechnitiumRecord challenge.technitiumToken emailDomain ("wire-domain." <> emailDomain) "TXT" challenge.dnsToken + pure $ OnPremFlow $ VerifyDomain emailDomain challenge + runStep domain (OnPremFlow (VerifyDomain emailDomain challenge)) = do + bindResponse (BrigInternal.getDomainRegistration domain emailDomain) $ \res -> do + res.status `shouldMatchInt` 200 + token <- bindResponse (Brig.verifyDomain domain emailDomain challenge.challengeId challenge.challengeToken) $ \resp -> do + resp.status `shouldMatchInt` 200 + resp.json %. "domain_ownership_token" & asString + let config = mkDomainRedirectBackend "https://wire.example.com" "https://webapp.wire.example.com" + pure $ OnPremFlow $ PostConfig emailDomain token config + runStep domain (OnPremFlow (PostConfig emailDomain token config)) = do + Brig.updateDomainRedirect domain Versioned emailDomain (Just token) config + >>= assertStatus 200 + pure $ OnPremFlow (OnPremVerify emailDomain config) + runStep domain (OnPremFlow (OnPremVerify emailDomain config)) = do + verifyOnPremConfig domain emailDomain config + pure $ OnPremFlow $ OnPremSuccess emailDomain config + runStep domain success@(OnPremFlow (OnPremSuccess emailDomain config)) = do + verifyOnPremConfig domain emailDomain config + pure success + + verifyOnPremConfig :: (HasCallStack) => String -> String -> Value -> App () + verifyOnPremConfig domain emailDomain config = + bindResponse (Brig.getDomainRegistrationFromEmail domain Versioned ("ruffy@" ++ emailDomain)) \resp -> do + resp.status `shouldMatchInt` 200 + resp.json %. "domain_redirect" `shouldMatch` (config %. "domain_redirect") + let backendUrl v = runMaybeT $ lookupFieldM v "backend" >>= flip lookupFieldM "config_url" + webappUrl v = runMaybeT $ lookupFieldM v "backend" >>= flip lookupFieldM "webapp_url" + backendUrl resp.json `shouldMatch` backendUrl config + webappUrl resp.json `shouldMatch` webappUrl config + + verifyTeamConfig :: (HasCallStack) => String -> String -> String -> App () + verifyTeamConfig domain tid emailDomain = do + bindResponse (BrigInternal.getDomainRegistration domain emailDomain) $ \resp -> do + resp.status `shouldMatchInt` 200 + resp.json %. "domain" `shouldMatch` emailDomain + resp.json %. "domain_redirect" `shouldMatch` "none" + resp.json %. "team_invite" `shouldMatch` "team" + resp.json %. "team" `shouldMatch` tid + + bindResponse (Brig.getDomainRegistrationFromEmail domain Versioned ("ruffy@" ++ emailDomain)) \resp -> do + resp.status `shouldMatchInt` 200 + resp.json %. "domain_redirect" `shouldMatch` "none" + + runAll :: (HasCallStack) => String -> DomainRegistrationTestCase -> App () + runAll domain success@(OnPremFlow (OnPremSuccess _ _)) = void $ runStep domain success + runAll domain success@(TeamFlow (TeamSuccess _ _)) = void $ runStep domain success + runAll domain inProgress = runAll domain =<< runStep domain inProgress + + mkDomainRedirectBackend :: String -> String -> Value + mkDomainRedirectBackend configUrl webappUrl = + object + [ "domain_redirect" .= "backend", + "backend" .= object ["config_url" .= configUrl, "webapp_url" .= webappUrl] + ] + + conf :: String -> Bool -> ServiceOverrides + conf db runMigration = + def + { brigCfg = setField "postgresMigration.domainRegistration" db, + backgroundWorkerCfg = + setField "postgresMigration.domainRegistration" db + >=> setField "migrateDomainRegistration" runMigration + } + + counterName :: String + counterName = "^wire_domain_registration_migration_finished" diff --git a/integration/test/Testlib/ModService.hs b/integration/test/Testlib/ModService.hs index a28f3505365..5dddcb56c4f 100644 --- a/integration/test/Testlib/ModService.hs +++ b/integration/test/Testlib/ModService.hs @@ -242,7 +242,8 @@ startDynamicBackend resource beOverrides = do gundeckCfg = setField "cassandra.keyspace" resource.berGundeckKeyspace, backgroundWorkerCfg = setField "cassandra.keyspace" resource.berGundeckKeyspace - >=> setField "cassandraGalley.keyspace" resource.berGalleyKeyspace, + >=> setField "cassandraGalley.keyspace" resource.berGalleyKeyspace + >=> setField "cassandraBrig.keyspace" resource.berBrigKeyspace, cannonCfg = setField "cassandra.keyspace" resource.berGundeckKeyspace } diff --git a/libs/wire-api/src/Wire/API/PostgresMarshall.hs b/libs/wire-api/src/Wire/API/PostgresMarshall.hs index 666b5b78c40..46a806c7fc7 100644 --- a/libs/wire-api/src/Wire/API/PostgresMarshall.hs +++ b/libs/wire-api/src/Wire/API/PostgresMarshall.hs @@ -36,12 +36,15 @@ import Data.Misc import Data.Profunctor import Data.Set qualified as Set import Data.Text qualified as Text +import Data.Text.Ascii qualified as Ascii import Data.Text.Encoding qualified as Text import Data.UUID import Data.Vector (Vector) import Data.Vector qualified as V import Hasql.Statement import Imports +import SAML2.WebSSO qualified as SAML +import Wire.API.EnterpriseLogin class PostgresMarshall db domain where postgresMarshall :: domain -> db @@ -538,6 +541,33 @@ instance PostgresMarshall Text Code.Key where instance PostgresMarshall Text Code.Value where postgresMarshall = Text.decodeUtf8 . toByteString' +instance PostgresMarshall ByteString HttpsUrl where + postgresMarshall = toByteString' + +instance PostgresMarshall ByteString Token where + postgresMarshall = (.unToken) + +instance PostgresMarshall Text DnsVerificationToken where + postgresMarshall = Ascii.toText . (.unDnsVerificationToken) + +instance PostgresMarshall Int32 DomainRedirectTag where + postgresMarshall = \case + NoneTag -> 1 + LockedTag -> 2 + SSOTag -> 3 + BackendTag -> 4 + NoRegistrationTag -> 5 + PreAuthorizedTag -> 6 + +instance PostgresMarshall Int32 TeamInviteTag where + postgresMarshall = \case + AllowedTag -> 1 + NotAllowedTag -> 2 + TeamTag -> 3 + +instance PostgresMarshall UUID SAML.IdPId where + postgresMarshall = SAML.fromIdPId + --- class PostgresUnmarshall db domain where @@ -869,6 +899,35 @@ instance PostgresUnmarshall Text Code.Key where instance PostgresUnmarshall Text Code.Value where postgresUnmarshall = mapLeft Text.pack . BSC.runParser BSC.parser . Text.encodeUtf8 +instance PostgresUnmarshall ByteString HttpsUrl where + postgresUnmarshall = first Text.pack . BSC.runParser BSC.parser + +instance PostgresUnmarshall ByteString Token where + postgresUnmarshall = Right . Token + +instance PostgresUnmarshall Text DnsVerificationToken where + postgresUnmarshall = first Text.pack . fmap DnsVerificationToken . Ascii.validate + +instance PostgresUnmarshall Int32 DomainRedirectTag where + postgresUnmarshall = \case + 1 -> Right NoneTag + 2 -> Right LockedTag + 3 -> Right SSOTag + 4 -> Right BackendTag + 5 -> Right NoRegistrationTag + 6 -> Right PreAuthorizedTag + n -> Left $ "Unexpected DomainRedirectTag value: " <> Text.pack (show n) + +instance PostgresUnmarshall Int32 TeamInviteTag where + postgresUnmarshall = \case + 1 -> Right AllowedTag + 2 -> Right NotAllowedTag + 3 -> Right TeamTag + n -> Left $ "Unexpected TeamInviteTag value: " <> Text.pack (show n) + +instance PostgresUnmarshall UUID SAML.IdPId where + postgresUnmarshall = Right . SAML.IdPId + --- lmapPG :: (PostgresMarshall db domain, Profunctor p) => p db x -> p domain x diff --git a/libs/wire-subsystems/postgres-migrations/20260420134603-domain_registration.sql b/libs/wire-subsystems/postgres-migrations/20260420134603-domain_registration.sql new file mode 100644 index 00000000000..9fd2e05998f --- /dev/null +++ b/libs/wire-subsystems/postgres-migrations/20260420134603-domain_registration.sql @@ -0,0 +1,27 @@ +CREATE TABLE domain_registration ( + domain text PRIMARY KEY, + authorized_team uuid, + domain_redirect integer, + team_invite integer, + idp_id uuid, + backend_url bytea, + team uuid, + dns_verification_token text, + ownership_token_hash bytea, + webapp_url bytea +); + +CREATE INDEX domain_registration_authorized_team_idx + ON domain_registration (authorized_team); + +CREATE TABLE domain_registration_challenge ( + id uuid PRIMARY KEY, + domain text NOT NULL, + challenge_token_hash bytea NOT NULL, + dns_verification_token text NOT NULL, + expires_at timestamptz NOT NULL +); + +-- index for deletes like `DELETE ... WHERE expires_at <= now()` +CREATE INDEX domain_registration_challenge_expires_at_idx + ON domain_registration_challenge (expires_at); diff --git a/libs/wire-subsystems/src/Wire/DomainRegistrationStore.hs b/libs/wire-subsystems/src/Wire/DomainRegistrationStore.hs index 489b4bc1ca3..ec6697dded8 100644 --- a/libs/wire-subsystems/src/Wire/DomainRegistrationStore.hs +++ b/libs/wire-subsystems/src/Wire/DomainRegistrationStore.hs @@ -1,3 +1,4 @@ +{-# LANGUAGE RecordWildCards #-} {-# LANGUAGE TemplateHaskell #-} -- This file is part of the Wire Server implementation. @@ -25,6 +26,11 @@ module Wire.DomainRegistrationStore lookup, lookupByTeam, delete, + DomainRegistrationRow, + upsertInternal, + lookupInternal, + lookupByTeamInternal, + deleteInternal, ) where @@ -33,9 +39,11 @@ import Data.ByteString.Conversion import Data.CaseInsensitive import Data.CaseInsensitive qualified as CI import Data.Domain as Domain +import Data.Hashable (hash) import Data.Id import Data.Misc import Data.Text as T +import Data.UUID (UUID) import Database.CQL.Protocol (Record (..), TupleType, recordInstance) import Imports hiding (lookup) import Polysemy @@ -45,6 +53,8 @@ import Polysemy.TinyLog qualified as Log import SAML2.WebSSO qualified as SAML import System.Logger.Message qualified as Log import Wire.API.EnterpriseLogin +import Wire.API.PostgresMarshall +import Wire.MigrationLock newtype DomainKey = DomainKey {unDomainKey :: CI Text} deriving stock (Eq, Ord, Show) @@ -61,6 +71,68 @@ instance Cql DomainKey where fromCql (CqlText txt) = pure . DomainKey . CI.mk $ txt fromCql _ = Left "DomainKey: Text expected" +instance PostgresMarshall Text DomainKey where + postgresMarshall = CI.foldedCase . unDomainKey + +instance PostgresUnmarshall Text DomainKey where + postgresUnmarshall = Right . DomainKey . CI.mk + +instance MigrationLockable DomainKey where + lockKey = fromIntegral . hash . CI.foldedCase . unDomainKey + lockScope = "domain_registration" + +type DomainRegistrationRow = + ( Text, + Maybe Int32, + Maybe Int32, + Maybe UUID, + Maybe ByteString, + Maybe UUID, + Maybe Text, + Maybe ByteString, + Maybe UUID, + Maybe ByteString + ) + +instance PostgresMarshall DomainRegistrationRow StoredDomainRegistration where + postgresMarshall StoredDomainRegistration {..} = + ( postgresMarshall domain, + postgresMarshall domainRedirect, + postgresMarshall teamInvite, + postgresMarshall idpId, + postgresMarshall backendUrl, + postgresMarshall team, + postgresMarshall dnsVerificationToken, + postgresMarshall authTokenHash, + postgresMarshall authorizedTeam, + postgresMarshall webappUrl + ) + +instance PostgresUnmarshall DomainRegistrationRow StoredDomainRegistration where + postgresUnmarshall + ( domain, + domainRedirect, + teamInvite, + idpId, + backendUrl, + team, + dnsVerificationToken, + authTokenHash, + authorizedTeam, + webappUrl + ) = + StoredDomainRegistration + <$> postgresUnmarshall domain + <*> postgresUnmarshall domainRedirect + <*> postgresUnmarshall teamInvite + <*> postgresUnmarshall idpId + <*> postgresUnmarshall backendUrl + <*> postgresUnmarshall team + <*> postgresUnmarshall dnsVerificationToken + <*> postgresUnmarshall authTokenHash + <*> postgresUnmarshall authorizedTeam + <*> postgresUnmarshall webappUrl + data StoredDomainRegistration = StoredDomainRegistration { domain :: DomainKey, domainRedirect :: Maybe DomainRedirectTag, @@ -83,6 +155,8 @@ data DomainRegistrationStore m a where LookupByTeamInternal :: TeamId -> DomainRegistrationStore m [StoredDomainRegistration] DeleteInternal :: DomainKey -> DomainRegistrationStore m () +makeSem ''DomainRegistrationStore + upsert :: (Member DomainRegistrationStore r) => DomainRegistration -> Sem r () upsert = send . UpsertInternal . toStored diff --git a/libs/wire-subsystems/src/Wire/DomainRegistrationStore/DualWrite.hs b/libs/wire-subsystems/src/Wire/DomainRegistrationStore/DualWrite.hs new file mode 100644 index 00000000000..435a4c418f8 --- /dev/null +++ b/libs/wire-subsystems/src/Wire/DomainRegistrationStore/DualWrite.hs @@ -0,0 +1,59 @@ +-- This file is part of the Wire Server implementation. +-- +-- Copyright (C) 2026 Wire Swiss GmbH +-- +-- This program is free software: you can redistribute it and/or modify it under +-- the terms of the GNU Affero General Public License as published by the Free +-- Software Foundation, either version 3 of the License, or (at your option) any +-- later version. +-- +-- This program is distributed in the hope that it will be useful, but WITHOUT +-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +-- details. +-- +-- You should have received a copy of the GNU Affero General Public License along +-- with this program. If not, see . + +module Wire.DomainRegistrationStore.DualWrite + ( interpretDomainRegistrationStoreToCassandraAndPostgres, + ) +where + +import Cassandra (ClientState) +import Imports +import Polysemy +import Polysemy.Async +import Polysemy.Conc.Effect.Race +import Polysemy.Error +import Polysemy.Time +import Polysemy.TinyLog +import Wire.DomainRegistrationStore +import Wire.DomainRegistrationStore qualified as DomainRegistrationStore +import Wire.DomainRegistrationStore.Cassandra qualified as Cassandra +import Wire.DomainRegistrationStore.Postgres qualified as Postgres +import Wire.MigrationLock +import Wire.Postgres + +interpretDomainRegistrationStoreToCassandraAndPostgres :: + ( PGConstraints r, + Member TinyLog r, + Member Async r, + Member Race r, + Member (Error MigrationLockError) r + ) => + ClientState -> + InterpreterFor DomainRegistrationStore r +interpretDomainRegistrationStoreToCassandraAndPostgres cs = interpret $ \case + UpsertInternal dr -> + withMigrationLocks LockShared (MilliSeconds 500) [dr.domain] $ do + Cassandra.interpretDomainRegistrationStoreToCassandra cs $ DomainRegistrationStore.upsertInternal dr + Postgres.interpretDomainRegistrationStoreToPostgres $ DomainRegistrationStore.upsertInternal dr + LookupInternal domain -> + Cassandra.interpretDomainRegistrationStoreToCassandra cs $ DomainRegistrationStore.lookupInternal domain + LookupByTeamInternal tid -> + Cassandra.interpretDomainRegistrationStoreToCassandra cs $ DomainRegistrationStore.lookupByTeamInternal tid + DeleteInternal domain -> + withMigrationLocks LockShared (MilliSeconds 500) [domain] $ do + Cassandra.interpretDomainRegistrationStoreToCassandra cs $ DomainRegistrationStore.deleteInternal domain + Postgres.interpretDomainRegistrationStoreToPostgres $ DomainRegistrationStore.deleteInternal domain diff --git a/libs/wire-subsystems/src/Wire/DomainRegistrationStore/Migration.hs b/libs/wire-subsystems/src/Wire/DomainRegistrationStore/Migration.hs new file mode 100644 index 00000000000..8bebc7aa2e0 --- /dev/null +++ b/libs/wire-subsystems/src/Wire/DomainRegistrationStore/Migration.hs @@ -0,0 +1,189 @@ +-- This file is part of the Wire Server implementation. +-- +-- Copyright (C) 2026 Wire Swiss GmbH +-- +-- This program is free software: you can redistribute it and/or modify it under +-- the terms of the GNU Affero General Public License as published by the Free +-- Software Foundation, either version 3 of the License, or (at your option) any +-- later version. +-- +-- This program is distributed in the hope that it will be useful, but WITHOUT +-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +-- details. +-- +-- You should have received a copy of the GNU Affero General Public License along +-- with this program. If not, see . + +module Wire.DomainRegistrationStore.Migration + ( migrateDomainRegistrationsLoop, + ) +where + +import Cassandra +import Data.ByteString.Conversion +import Data.Conduit +import Data.Conduit.List qualified as C +import Data.Domain +import Data.Id +import Database.CQL.Protocol (Record (asRecord), TupleType) +import Hasql.Pool qualified as Hasql +import Imports hiding (lookup) +import Polysemy +import Polysemy.Async +import Polysemy.Conc (interpretRace) +import Polysemy.Conc.Effect.Race hiding (Timeout) +import Polysemy.Error +import Polysemy.Input +import Polysemy.State +import Polysemy.Time +import Polysemy.TinyLog +import Prometheus qualified +import System.Logger qualified as Log +import Util.Timeout +import Wire.API.EnterpriseLogin +import Wire.DomainRegistrationStore +import Wire.DomainRegistrationStore.Cassandra qualified as DomainRegistrationCassandra +import Wire.DomainRegistrationStore.Postgres qualified as DomainRegistrationPostgres +import Wire.DomainVerificationChallengeStore +import Wire.DomainVerificationChallengeStore.Postgres qualified as ChallengePostgres +import Wire.Migration +import Wire.MigrationLock +import Wire.Postgres +import Wire.Sem.Logger (mapLogger) +import Wire.Sem.Logger.TinyLog (loggerToTinyLog) + +type EffectStack = + [ State Int, + Input ClientState, + Input Hasql.Pool, + Async, + Race, + TinyLog, + Embed IO, + Final IO + ] + +migrateDomainRegistrationsLoop :: + MigrationOptions -> + ClientState -> + Hasql.Pool -> + Log.Logger -> + Prometheus.Counter -> + Prometheus.Counter -> + Prometheus.Counter -> + IO () +migrateDomainRegistrationsLoop migOpts cassClient pgPool logger migCounter migFinished migFailed = + migrationLoop + logger + "domain registrations" + migFinished + migFailed + (interpreter cassClient pgPool logger "domain registrations") + (migrateAllDomainRegistrations migOpts migCounter) + +interpreter :: ClientState -> Hasql.Pool -> Log.Logger -> ByteString -> Sem EffectStack a -> IO (Int, a) +interpreter cassClient pgPool logger name = + runFinal + . embedToFinal + . loggerToTinyLog logger + . mapLogger (Log.field "migration" (Log.val name) .) + . raiseUnder + . interpretRace + . asyncToIOFinal + . runInputConst pgPool + . runInputConst cassClient + . runState 0 + +migrateAllDomainRegistrations :: + ( Member (Input Hasql.Pool) r, + Member (Embed IO) r, + Member (Input ClientState) r, + Member TinyLog r, + Member (State Int) r, + Member Async r, + Member Race r + ) => + MigrationOptions -> + Prometheus.Counter -> + ConduitM () Void (Sem r) () +migrateAllDomainRegistrations migOpts migCounter = do + lift $ info $ Log.msg (Log.val "migrateAllDomainRegistrationChallenges") + withCount (paginateSem selectAllChallenges (paramsP LocalQuorum () migOpts.pageSize) x5) + .| logRetrievedPage migOpts.pageSize id + .| C.mapM_ (traverse_ (\row@(cid, _, _, _, _) -> handleErrors (toByteString' cid) (migrateDomainVerificationChallengeRow migCounter row))) + + lift $ info $ Log.msg (Log.val "migrateAllDomainRegistrations") + withCount (paginateSem selectAllRegistrations (paramsP LocalQuorum () migOpts.pageSize) x5) + .| logRetrievedPage migOpts.pageSize asRecord + .| C.mapM_ (traverse_ (\row -> handleRegistrationErrors (toByteString' (show row.domain)) (migrateDomainRegistrationRow migCounter row))) + +migrateDomainRegistrationRow :: + ( PGConstraints r, + Member (Input ClientState) r, + Member TinyLog r, + Member Async r, + Member (Error MigrationLockError) r, + Member Race r + ) => + Prometheus.Counter -> + StoredDomainRegistration -> + Sem r () +migrateDomainRegistrationRow migCounter row = do + void . withMigrationLocks LockExclusive (Seconds 10) [row.domain] $ do + isMigrated <- DomainRegistrationPostgres.exists row.domain + unless isMigrated $ do + cassClient <- input @ClientState + mCurrentRow <- + DomainRegistrationCassandra.interpretDomainRegistrationStoreToCassandra cassClient $ + lookupInternal row.domain + for_ mCurrentRow $ \currentRow -> do + DomainRegistrationPostgres.interpretDomainRegistrationStoreToPostgres $ upsertInternal currentRow + liftIO $ Prometheus.incCounter migCounter + +migrateDomainVerificationChallengeRow :: + (PGConstraints r) => + Prometheus.Counter -> + (ChallengeId, Domain, Token, DnsVerificationToken, Int32) -> + Sem r () +migrateDomainVerificationChallengeRow migCounter (cid, domain, challengeTokenHash, dnsVerificationToken, ttlSecs) = + when (ttlSecs > 0) $ do + let ttl = Timeout (fromIntegral ttlSecs) + row = + StoredDomainVerificationChallenge + { challengeId = cid, + domain = domain, + challengeTokenHash = challengeTokenHash, + dnsVerificationToken = dnsVerificationToken + } + ChallengePostgres.interpretDomainVerificationChallengeStoreToPostgres ttl $ insert row + liftIO $ Prometheus.incCounter migCounter + +selectAllRegistrations :: PrepQuery R () (TupleType StoredDomainRegistration) +selectAllRegistrations = + "SELECT domain, domain_redirect, team_invite, idp_id, backend_url, team, dns_verification_token, ownership_token_hash, authorized_team, webapp_url FROM domain_registration" + +selectAllChallenges :: PrepQuery R () (ChallengeId, Domain, Token, DnsVerificationToken, Int32) +selectAllChallenges = + "SELECT id, domain, challenge_token_hash, dns_verification_token, ttl(challenge_token_hash) FROM domain_registration_challenge" + +handleRegistrationErrors :: + ( Member (State Int) r, + Member TinyLog r + ) => + ByteString -> + (Sem (Error MigrationLockError : Error Hasql.UsageError : r) ()) -> + Sem r () +handleRegistrationErrors key action = do + eithErr <- runError (runError action) + case eithErr of + Right (Right _) -> pure () + Right (Left e) -> logError (show e) + Left e -> logError (show e) + where + logError e = do + warn $ + Log.msg (Log.val "error occurred during migration") + . Log.field "key" (show key) + . Log.field "error" e + modify (+ 1) diff --git a/libs/wire-subsystems/src/Wire/DomainRegistrationStore/Postgres.hs b/libs/wire-subsystems/src/Wire/DomainRegistrationStore/Postgres.hs new file mode 100644 index 00000000000..179ac74b41a --- /dev/null +++ b/libs/wire-subsystems/src/Wire/DomainRegistrationStore/Postgres.hs @@ -0,0 +1,125 @@ +{-# LANGUAGE RecordWildCards #-} + +-- This file is part of the Wire Server implementation. +-- +-- Copyright (C) 2026 Wire Swiss GmbH +-- +-- This program is free software: you can redistribute it and/or modify it under +-- the terms of the GNU Affero General Public License as published by the Free +-- Software Foundation, either version 3 of the License, or (at your option) any +-- later version. +-- +-- This program is distributed in the hope that it will be useful, but WITHOUT +-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +-- details. +-- +-- You should have received a copy of the GNU Affero General Public License along +-- with this program. If not, see . + +module Wire.DomainRegistrationStore.Postgres + ( interpretDomainRegistrationStoreToPostgres, + exists, + ) +where + +import Data.Id (TeamId) +import Data.UUID (UUID) +import Data.Vector (Vector) +import Data.Vector qualified as Vector +import Hasql.Statement qualified as Hasql +import Hasql.TH +import Imports hiding (lookup) +import Polysemy +import Wire.API.PostgresMarshall +import Wire.DomainRegistrationStore +import Wire.Postgres + +interpretDomainRegistrationStoreToPostgres :: + (PGConstraints r) => + InterpreterFor DomainRegistrationStore r +interpretDomainRegistrationStoreToPostgres = interpret $ \case + UpsertInternal dr -> upsertImpl dr + LookupInternal domain -> lookupImpl domain + LookupByTeamInternal tid -> lookupByTeamInternalImpl tid + DeleteInternal domain -> deleteImpl domain + +upsertImpl :: (PGConstraints r) => StoredDomainRegistration -> Sem r () +upsertImpl dr = + runStatement dr upsertStatement + where + upsertStatement :: Hasql.Statement StoredDomainRegistration () + upsertStatement = + lmapPG + [resultlessStatement|INSERT INTO domain_registration + (domain, domain_redirect, team_invite, idp_id, backend_url, + team, dns_verification_token, ownership_token_hash, authorized_team, webapp_url) + VALUES + ($1 :: text, $2 :: int?, $3 :: int?, $4 :: uuid?, $5 :: bytea?, + $6 :: uuid?, $7 :: text?, $8 :: bytea?, $9 :: uuid?, $10 :: bytea?) + ON CONFLICT (domain) DO UPDATE + SET domain_redirect = ($2 :: int?), + team_invite = ($3 :: int?), + idp_id = ($4 :: uuid?), + backend_url = ($5 :: bytea?), + team = ($6 :: uuid?), + dns_verification_token = ($7 :: text?), + ownership_token_hash = ($8 :: bytea?), + authorized_team = ($9 :: uuid?), + webapp_url = ($10 :: bytea?) + |] + +lookupImpl :: (PGConstraints r) => DomainKey -> Sem r (Maybe StoredDomainRegistration) +lookupImpl domain = + runStatement domain selectStatement + where + selectStatement :: Hasql.Statement DomainKey (Maybe StoredDomainRegistration) + selectStatement = + dimapPG @Text @DomainKey @(Maybe DomainRegistrationRow) @(Maybe StoredDomainRegistration) $ + [maybeStatement|SELECT (domain :: text), (domain_redirect :: int?), (team_invite :: int?), + (idp_id :: uuid?), (backend_url :: bytea?), (team :: uuid?), + (dns_verification_token :: text?), (ownership_token_hash :: bytea?), + (authorized_team :: uuid?), (webapp_url :: bytea?) + FROM domain_registration + WHERE domain = ($1 :: text) + |] + +lookupByTeamInternalImpl :: (PGConstraints r) => TeamId -> Sem r [StoredDomainRegistration] +lookupByTeamInternalImpl tid = do + rows <- runStatement tid selectByTeamStatement + pure $ Vector.toList rows + where + selectByTeamStatement :: Hasql.Statement TeamId (Vector StoredDomainRegistration) + selectByTeamStatement = + dimapPG @UUID @TeamId @(Vector DomainRegistrationRow) @(Vector StoredDomainRegistration) $ + [vectorStatement|SELECT (domain :: text), (domain_redirect :: int?), (team_invite :: int?), + (idp_id :: uuid?), (backend_url :: bytea?), (team :: uuid?), + (dns_verification_token :: text?), (ownership_token_hash :: bytea?), + (authorized_team :: uuid?), (webapp_url :: bytea?) + FROM domain_registration + WHERE authorized_team = ($1 :: uuid) + |] + +deleteImpl :: (PGConstraints r) => DomainKey -> Sem r () +deleteImpl domain = + runStatement domain deleteStatement + where + deleteStatement :: Hasql.Statement DomainKey () + deleteStatement = + lmapPG + [resultlessStatement|DELETE FROM domain_registration + WHERE domain = ($1 :: text) + |] + +exists :: (PGConstraints r) => DomainKey -> Sem r Bool +exists domain = + runStatement domain existsStatement + where + existsStatement :: Hasql.Statement DomainKey Bool + existsStatement = + lmapPG @Text @DomainKey + [singletonStatement|SELECT EXISTS ( + SELECT 1 + FROM domain_registration + WHERE domain = ($1 :: text) + ) :: bool|] diff --git a/libs/wire-subsystems/src/Wire/DomainVerificationChallengeStore/Cassandra.hs b/libs/wire-subsystems/src/Wire/DomainVerificationChallengeStore/Cassandra.hs index 44ed929a560..bf32c931417 100644 --- a/libs/wire-subsystems/src/Wire/DomainVerificationChallengeStore/Cassandra.hs +++ b/libs/wire-subsystems/src/Wire/DomainVerificationChallengeStore/Cassandra.hs @@ -27,34 +27,31 @@ import Data.Id import Database.CQL.Protocol (Record (..), TupleType, asTuple) import Imports hiding (lookup) import Polysemy -import Polysemy.Embed import Polysemy.Input import Util.Timeout import Wire.DomainVerificationChallengeStore +import Wire.Util (embedClientInput) interpretDomainVerificationChallengeStoreToCassandra :: forall r. - (Member (Embed IO) r) => - ClientState -> + ( Member (Embed IO) r, + Member (Input ClientState) r + ) => Timeout -> InterpreterFor DomainVerificationChallengeStore r -interpretDomainVerificationChallengeStoreToCassandra casClient ttl = - runInputConst ttl - . runEmbedded (runClient casClient) - . interpret - ( \case - Insert challenge -> insertImpl challenge - Lookup challengeId -> lookupImpl challengeId - Delete challengeId -> deleteImpl challengeId - ) - . raiseUnder2 +interpretDomainVerificationChallengeStoreToCassandra ttl = + interpret + ( \case + Insert challenge -> embedClientInput $ insertImpl ttl challenge + Lookup challengeId -> embedClientInput $ lookupImpl challengeId + Delete challengeId -> embedClientInput $ deleteImpl challengeId + ) insertImpl :: - (Member (Embed Client) r, Member (Input Timeout) r) => + Timeout -> StoredDomainVerificationChallenge -> - Sem r () -insertImpl challenge = do - ttl <- input + Client () +insertImpl ttl challenge = do let q :: PrepQuery W (TupleType StoredDomainVerificationChallenge) () q = fromString $ @@ -62,22 +59,20 @@ insertImpl challenge = do \ (id, domain, challenge_token_hash, dns_verification_token)\ \ VALUES (?,?,?,?) using ttl " <> show (round (nominalDiffTimeToSeconds (timeoutDiff ttl)) :: Integer) - embed $ retry x5 $ write q (params LocalQuorum (asTuple challenge)) + retry x5 $ write q (params LocalQuorum (asTuple challenge)) lookupImpl :: - (Member (Embed Client) r) => ChallengeId -> - Sem r (Maybe StoredDomainVerificationChallenge) + Client (Maybe StoredDomainVerificationChallenge) lookupImpl challengeId = - embed $ - fmap asRecord - <$> retry x1 (query1 cqlSelect (params LocalQuorum (Identity challengeId))) + fmap asRecord + <$> retry x1 (query1 cqlSelect (params LocalQuorum (Identity challengeId))) cqlSelect :: PrepQuery R (Identity ChallengeId) (TupleType StoredDomainVerificationChallenge) cqlSelect = "SELECT id, domain, challenge_token_hash, dns_verification_token FROM domain_registration_challenge WHERE id = ?" -deleteImpl :: (Member (Embed Client) r) => ChallengeId -> Sem r () -deleteImpl challengeId = embed $ retry x5 $ write cqlDelete (params LocalQuorum (Identity challengeId)) +deleteImpl :: ChallengeId -> Client () +deleteImpl challengeId = retry x5 $ write cqlDelete (params LocalQuorum (Identity challengeId)) cqlDelete :: PrepQuery W (Identity ChallengeId) () cqlDelete = "DELETE FROM domain_registration_challenge WHERE id = ?" diff --git a/libs/wire-subsystems/src/Wire/DomainVerificationChallengeStore/DualWrite.hs b/libs/wire-subsystems/src/Wire/DomainVerificationChallengeStore/DualWrite.hs new file mode 100644 index 00000000000..333ab4d9601 --- /dev/null +++ b/libs/wire-subsystems/src/Wire/DomainVerificationChallengeStore/DualWrite.hs @@ -0,0 +1,49 @@ +-- This file is part of the Wire Server implementation. +-- +-- Copyright (C) 2026 Wire Swiss GmbH +-- +-- This program is free software: you can redistribute it and/or modify it under +-- the terms of the GNU Affero General Public License as published by the Free +-- Software Foundation, either version 3 of the License, or (at your option) any +-- later version. +-- +-- This program is distributed in the hope that it will be useful, but WITHOUT +-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +-- details. +-- +-- You should have received a copy of the GNU Affero General Public License along +-- with this program. If not, see . + +module Wire.DomainVerificationChallengeStore.DualWrite + ( interpretDomainVerificationChallengeStoreToCassandraAndPostgres, + ) +where + +import Cassandra +import Imports +import Polysemy +import Polysemy.Input +import Util.Timeout +import Wire.DomainVerificationChallengeStore +import Wire.DomainVerificationChallengeStore qualified as DomainVerificationChallengeStore +import Wire.DomainVerificationChallengeStore.Cassandra qualified as Cassandra +import Wire.DomainVerificationChallengeStore.Postgres qualified as Postgres +import Wire.Postgres + +-- | Cassandra is the source of truth during migration; writes are mirrored to Postgres. +interpretDomainVerificationChallengeStoreToCassandraAndPostgres :: + ( Member (Input ClientState) r, + PGConstraints r + ) => + Timeout -> + InterpreterFor DomainVerificationChallengeStore r +interpretDomainVerificationChallengeStoreToCassandraAndPostgres to = interpret $ \case + Insert challenge -> do + Cassandra.interpretDomainVerificationChallengeStoreToCassandra to $ DomainVerificationChallengeStore.insert challenge + Postgres.interpretDomainVerificationChallengeStoreToPostgres to $ DomainVerificationChallengeStore.insert challenge + Lookup challengeId -> + Cassandra.interpretDomainVerificationChallengeStoreToCassandra to $ DomainVerificationChallengeStore.lookup challengeId + Delete challengeId -> do + Cassandra.interpretDomainVerificationChallengeStoreToCassandra to $ DomainVerificationChallengeStore.delete challengeId + Postgres.interpretDomainVerificationChallengeStoreToPostgres to $ DomainVerificationChallengeStore.delete challengeId diff --git a/libs/wire-subsystems/src/Wire/DomainVerificationChallengeStore/Postgres.hs b/libs/wire-subsystems/src/Wire/DomainVerificationChallengeStore/Postgres.hs new file mode 100644 index 00000000000..c06d1f9c227 --- /dev/null +++ b/libs/wire-subsystems/src/Wire/DomainVerificationChallengeStore/Postgres.hs @@ -0,0 +1,100 @@ +-- This file is part of the Wire Server implementation. +-- +-- Copyright (C) 2026 Wire Swiss GmbH +-- +-- This program is free software: you can redistribute it and/or modify it under +-- the terms of the GNU Affero General Public License as published by the Free +-- Software Foundation, either version 3 of the License, or (at your option) any +-- later version. +-- +-- This program is distributed in the hope that it will be useful, but WITHOUT +-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +-- details. +-- +-- You should have received a copy of the GNU Affero General Public License along +-- with this program. If not, see . + +module Wire.DomainVerificationChallengeStore.Postgres + ( interpretDomainVerificationChallengeStoreToPostgres, + ) +where + +import Data.Domain +import Data.Id +import Hasql.Statement qualified as Hasql +import Hasql.TH +import Imports hiding (lookup) +import Polysemy +import Util.Timeout +import Wire.API.EnterpriseLogin +import Wire.API.PostgresMarshall +import Wire.DomainVerificationChallengeStore +import Wire.Postgres + +interpretDomainVerificationChallengeStoreToPostgres :: + forall r. + (PGConstraints r) => + Timeout -> + InterpreterFor DomainVerificationChallengeStore r +interpretDomainVerificationChallengeStoreToPostgres ttl = + interpret $ + \case + Insert challenge -> insertImpl ttl challenge + Lookup challengeId -> lookupImpl challengeId + Delete challengeId -> deleteImpl challengeId + +deleteImpl :: (PGConstraints r) => ChallengeId -> Sem r () +deleteImpl cid = + runStatement cid deleteStmt + where + deleteStmt :: Hasql.Statement ChallengeId () + deleteStmt = + lmapPG + [resultlessStatement|DELETE FROM domain_registration_challenge + WHERE id = ($1 :: uuid) + |] + +lookupImpl :: (PGConstraints r) => ChallengeId -> Sem r (Maybe StoredDomainVerificationChallenge) +lookupImpl cid = do + mRow <- runStatement cid select + pure $ mk <$> mRow + where + mk :: (Token, DnsVerificationToken, Domain) -> StoredDomainVerificationChallenge + mk (hash, token, domain) = + StoredDomainVerificationChallenge + { challengeId = cid, + domain = domain, + challengeTokenHash = hash, + dnsVerificationToken = token + } + + select :: Hasql.Statement ChallengeId (Maybe (Token, DnsVerificationToken, Domain)) + select = + dimapPG + [maybeStatement|SELECT + (challenge_token_hash :: bytea), + (dns_verification_token :: text), + (domain :: text) + FROM domain_registration_challenge + WHERE id = ($1 :: uuid) AND expires_at > now () + |] + +insertImpl :: (PGConstraints r) => Timeout -> StoredDomainVerificationChallenge -> Sem r () +insertImpl ttl ch = + runStatement (ch.challengeId, ch.domain, ch.challengeTokenHash, ch.dnsVerificationToken, ttlSecs) insertStmt + where + ttlSecs = round (nominalDiffTimeToSeconds (timeoutDiff ttl)) :: Int32 + insertStmt :: Hasql.Statement (ChallengeId, Domain, Token, DnsVerificationToken, Int32) () + insertStmt = + lmapPG + [resultlessStatement|INSERT INTO domain_registration_challenge + (id, domain, challenge_token_hash, dns_verification_token, expires_at) + VALUES + ($1 :: uuid, $2 :: text, $3 :: bytea, $4 :: text, now() + make_interval(secs => $5 :: int)) + ON CONFLICT (id) DO UPDATE + SET domain = ($2 :: text), + challenge_token_hash = ($3 :: bytea), + dns_verification_token = ($4 :: text), + expires_at = now() + make_interval(secs => $5 :: int) + |] diff --git a/libs/wire-subsystems/src/Wire/MigrationLock.hs b/libs/wire-subsystems/src/Wire/MigrationLock.hs index 140d7342bba..a1e18b5099d 100644 --- a/libs/wire-subsystems/src/Wire/MigrationLock.hs +++ b/libs/wire-subsystems/src/Wire/MigrationLock.hs @@ -30,7 +30,7 @@ import Hasql.Statement qualified as Hasql import Hasql.TH import Imports import Network.HTTP.Types.Status (status500) -import Network.Wai.Utilities.Error qualified as WaiError +import Network.Wai.Utilities.Error qualified as Wai import Network.Wai.Utilities.JSONResponse import Polysemy import Polysemy.Async @@ -43,6 +43,7 @@ import Polysemy.TinyLog qualified as TinyLog import System.Logger.Message qualified as Log import Wire.API.Error import Wire.API.PostgresMarshall +import Wire.Error import Wire.Postgres class MigrationLockable a where @@ -62,7 +63,13 @@ data MigrationLockError = TimedOutAcquiringLock deriving (Show) instance APIError MigrationLockError where - toResponse _ = waiErrorToJSONResponse $ WaiError.mkError status500 "internal-server-error" "Internal Server Error" + toResponse = waiErrorToJSONResponse . migrationLockErrorToWai + +migrationLockErrorToHttpError :: MigrationLockError -> HttpError +migrationLockErrorToHttpError = StdError . migrationLockErrorToWai + +migrationLockErrorToWai :: MigrationLockError -> Wai.Error +migrationLockErrorToWai _ = Wai.mkError status500 "internal-server-error" "Internal Server Error" withMigrationLocks :: forall x a u r. diff --git a/libs/wire-subsystems/src/Wire/PostgresMigrationOpts.hs b/libs/wire-subsystems/src/Wire/PostgresMigrationOpts.hs index df635d14530..f02ade14b9b 100644 --- a/libs/wire-subsystems/src/Wire/PostgresMigrationOpts.hs +++ b/libs/wire-subsystems/src/Wire/PostgresMigrationOpts.hs @@ -43,7 +43,8 @@ instance FromJSON StorageLocation where data PostgresMigrationOpts = PostgresMigrationOpts { conversation :: StorageLocation, conversationCodes :: StorageLocation, - teamFeatures :: StorageLocation + teamFeatures :: StorageLocation, + domainRegistration :: StorageLocation } deriving (Show) @@ -53,3 +54,4 @@ instance FromJSON PostgresMigrationOpts where <$> o .: "conversation" <*> o .: "conversationCodes" <*> o .: "teamFeatures" + <*> o .: "domainRegistration" diff --git a/libs/wire-subsystems/wire-subsystems.cabal b/libs/wire-subsystems/wire-subsystems.cabal index 6aa8541f47a..0f4c739c004 100644 --- a/libs/wire-subsystems/wire-subsystems.cabal +++ b/libs/wire-subsystems/wire-subsystems.cabal @@ -306,8 +306,13 @@ library Wire.DeleteQueue.InMemory Wire.DomainRegistrationStore Wire.DomainRegistrationStore.Cassandra + Wire.DomainRegistrationStore.DualWrite + Wire.DomainRegistrationStore.Migration + Wire.DomainRegistrationStore.Postgres Wire.DomainVerificationChallengeStore Wire.DomainVerificationChallengeStore.Cassandra + Wire.DomainVerificationChallengeStore.DualWrite + Wire.DomainVerificationChallengeStore.Postgres Wire.EmailSending Wire.EmailSending.SES Wire.EmailSending.SMTP diff --git a/postgres-schema.sql b/postgres-schema.sql index 070828aa351..36b3260dfcd 100644 --- a/postgres-schema.sql +++ b/postgres-schema.sql @@ -9,8 +9,8 @@ \restrict 79bbfb4630959c48307653a5cd3d83f2582b3c2210f75f10d79e3ebf0015620 --- Dumped from database version 17.7 --- Dumped by pg_dump version 17.7 +-- Dumped from database version 17.9 +-- Dumped by pg_dump version 17.9 SET statement_timeout = 0; SET lock_timeout = 0; @@ -178,6 +178,41 @@ CREATE TABLE public.conversation_out_of_sync ( ALTER TABLE public.conversation_out_of_sync OWNER TO "wire-server"; +-- +-- Name: domain_registration; Type: TABLE; Schema: public; Owner: wire-server +-- + +CREATE TABLE public.domain_registration ( + domain text NOT NULL, + authorized_team uuid, + domain_redirect integer, + team_invite integer, + idp_id uuid, + backend_url bytea, + team uuid, + dns_verification_token text, + ownership_token_hash bytea, + webapp_url bytea +); + + +ALTER TABLE public.domain_registration OWNER TO "wire-server"; + +-- +-- Name: domain_registration_challenge; Type: TABLE; Schema: public; Owner: wire-server +-- + +CREATE TABLE public.domain_registration_challenge ( + id uuid NOT NULL, + domain text NOT NULL, + challenge_token_hash bytea NOT NULL, + dns_verification_token text NOT NULL, + expires_at timestamp with time zone NOT NULL +); + + +ALTER TABLE public.domain_registration_challenge OWNER TO "wire-server"; + -- -- Name: local_conversation_remote_member; Type: TABLE; Schema: public; Owner: wire-server -- @@ -393,6 +428,22 @@ ALTER TABLE ONLY public.conversation ADD CONSTRAINT conversation_pkey PRIMARY KEY (id); +-- +-- Name: domain_registration_challenge domain_registration_challenge_pkey; Type: CONSTRAINT; Schema: public; Owner: wire-server +-- + +ALTER TABLE ONLY public.domain_registration_challenge + ADD CONSTRAINT domain_registration_challenge_pkey PRIMARY KEY (id); + + +-- +-- Name: domain_registration domain_registration_pkey; Type: CONSTRAINT; Schema: public; Owner: wire-server +-- + +ALTER TABLE ONLY public.domain_registration + ADD CONSTRAINT domain_registration_pkey PRIMARY KEY (domain); + + -- -- Name: local_conversation_remote_member local_conversation_remote_member_pkey; Type: CONSTRAINT; Schema: public; Owner: wire-server -- @@ -522,6 +573,20 @@ CREATE INDEX conversation_team_group_type_lower_name_id_idx ON public.conversati CREATE INDEX conversation_team_idx ON public.conversation USING btree (team); +-- +-- Name: domain_registration_authorized_team_idx; Type: INDEX; Schema: public; Owner: wire-server +-- + +CREATE INDEX domain_registration_authorized_team_idx ON public.domain_registration USING btree (authorized_team); + + +-- +-- Name: domain_registration_challenge_expires_at_idx; Type: INDEX; Schema: public; Owner: wire-server +-- + +CREATE INDEX domain_registration_challenge_expires_at_idx ON public.domain_registration_challenge USING btree (expires_at); + + -- -- Name: idx_meetings_conversation; Type: INDEX; Schema: public; Owner: wire-server -- diff --git a/services/background-worker/background-worker.integration.yaml b/services/background-worker/background-worker.integration.yaml index eaee5a414c4..fa398766188 100644 --- a/services/background-worker/background-worker.integration.yaml +++ b/services/background-worker/background-worker.integration.yaml @@ -57,6 +57,7 @@ migrateConversationsOptions: parallelism: 2 migrateConversationCodes: false migrateTeamFeatures: false +migrateDomainRegistration: false # Background jobs consumer configuration for integration backgroundJobs: @@ -68,3 +69,4 @@ postgresMigration: conversation: postgresql conversationCodes: postgresql teamFeatures: postgresql + domainRegistration: postgresql diff --git a/services/background-worker/src/Wire/BackgroundWorker.hs b/services/background-worker/src/Wire/BackgroundWorker.hs index e89ed926f43..315bea5bd3b 100644 --- a/services/background-worker/src/Wire/BackgroundWorker.hs +++ b/services/background-worker/src/Wire/BackgroundWorker.hs @@ -71,6 +71,13 @@ run opts galleyOpts = do withNamedLogger "migrate-team-features" $ Migrations.teamFeatures (MigrationOptions 1000 1) else pure $ pure () + cleanupDomainRegistrationMigration <- + if opts.migrateDomainRegistration + then + runAppT env $ + withNamedLogger "migrate-domain-registration" $ + Migrations.domainRegistration (MigrationOptions 1000 1) + else pure $ pure () cleanupJobs <- runAppT env $ withNamedLogger "background-job-consumer" $ @@ -78,12 +85,13 @@ run opts galleyOpts = do let cleanup = void $ runConcurrently $ - (,,,,,) + (,,,,,,) <$> Concurrently cleanupDeadUserNotifWatcher <*> Concurrently cleanupBackendNotifPusher <*> Concurrently cleanupConvMigration <*> Concurrently cleanUpConvCodesMigration <*> Concurrently cleanupTeamFeaturesMigration + <*> Concurrently cleanupDomainRegistrationMigration <*> Concurrently cleanupJobs let server = defaultServer (T.unpack opts.backgroundWorker.host) opts.backgroundWorker.port env.logger diff --git a/services/background-worker/src/Wire/BackgroundWorker/Options.hs b/services/background-worker/src/Wire/BackgroundWorker/Options.hs index c616d1e5a4e..2d1078fe8f1 100644 --- a/services/background-worker/src/Wire/BackgroundWorker/Options.hs +++ b/services/background-worker/src/Wire/BackgroundWorker/Options.hs @@ -50,6 +50,7 @@ data Opts = Opts migrateConversationsOptions :: !MigrationOptions, migrateConversationCodes :: !Bool, migrateTeamFeatures :: !Bool, + migrateDomainRegistration :: !Bool, backgroundJobs :: BackgroundJobsConfig } deriving (Show, Generic) diff --git a/services/background-worker/src/Wire/PostgresMigrations.hs b/services/background-worker/src/Wire/PostgresMigrations.hs index 541716d0aec..ea8212a9d35 100644 --- a/services/background-worker/src/Wire/PostgresMigrations.hs +++ b/services/background-worker/src/Wire/PostgresMigrations.hs @@ -25,6 +25,7 @@ import Wire.BackgroundWorker.Env import Wire.BackgroundWorker.Util import Wire.CodeStore.Migration import Wire.ConversationStore.Migration +import Wire.DomainRegistrationStore.Migration import Wire.Migration (MigrationOptions) import Wire.TeamFeatureStore.Migration @@ -84,3 +85,20 @@ teamFeatures migOpts = do pure $ do Log.info logger $ Log.msg (Log.val "cancelling team features migration") cancel migrationLoop + +domainRegistration :: MigrationOptions -> AppT IO CleanupAction +domainRegistration migOpts = do + cassClient <- asks (.cassandraBrig) + pgPool <- asks (.hasqlPool) + logger <- asks (.logger) + Log.info logger $ Log.msg (Log.val "starting domain registration migration") + count <- register $ counter $ Prometheus.Info "wire_domain_registration_migrated_to_pg" "Number of domain registration rows migrated to Postgresql" + finished <- register $ counter $ Prometheus.Info "wire_domain_registration_migration_finished" "Whether the domain registration migration to Postgresql is finished successfully" + failed <- register $ counter $ Prometheus.Info "wire_domain_registration_migration_failed" "Whether the domain registration migration to Postgresql has failed" + + migrationLoop <- async . lift $ migrateDomainRegistrationsLoop migOpts cassClient pgPool logger count finished failed + + Log.info logger $ Log.msg (Log.val "started domain registration migration") + pure $ do + Log.info logger $ Log.msg (Log.val "cancelling domain registration migration") + cancel migrationLoop diff --git a/services/background-worker/test/Test/Wire/BackendNotificationPusherSpec.hs b/services/background-worker/test/Test/Wire/BackendNotificationPusherSpec.hs index a3730bdaf27..1b4715d07a1 100644 --- a/services/background-worker/test/Test/Wire/BackendNotificationPusherSpec.hs +++ b/services/background-worker/test/Test/Wire/BackendNotificationPusherSpec.hs @@ -366,7 +366,8 @@ spec = do PostgresMigrationOpts { conversation = CassandraStorage, conversationCodes = CassandraStorage, - teamFeatures = CassandraStorage + teamFeatures = CassandraStorage, + domainRegistration = CassandraStorage } gundeckEndpoint = undefined brigEndpoint = undefined @@ -417,7 +418,8 @@ spec = do PostgresMigrationOpts { conversation = CassandraStorage, conversationCodes = CassandraStorage, - teamFeatures = CassandraStorage + teamFeatures = CassandraStorage, + domainRegistration = CassandraStorage } gundeckEndpoint = undefined brigEndpoint = undefined diff --git a/services/background-worker/test/Test/Wire/Util.hs b/services/background-worker/test/Test/Wire/Util.hs index 6aa6afa8c91..a3e23d4ea56 100644 --- a/services/background-worker/test/Test/Wire/Util.hs +++ b/services/background-worker/test/Test/Wire/Util.hs @@ -44,7 +44,8 @@ testEnv = do PostgresMigrationOpts { conversation = CassandraStorage, conversationCodes = CassandraStorage, - teamFeatures = CassandraStorage + teamFeatures = CassandraStorage, + domainRegistration = CassandraStorage } statuses <- newIORef mempty backendNotificationMetrics <- mkBackendNotificationMetrics diff --git a/services/brig/brig.integration.yaml b/services/brig/brig.integration.yaml index fc23b069f74..e59957d04a6 100644 --- a/services/brig/brig.integration.yaml +++ b/services/brig/brig.integration.yaml @@ -171,6 +171,12 @@ turn: configTTL: 3600 tokenTTL: 21600 +postgresMigration: + conversation: postgresql + conversationCodes: postgresql + teamFeatures: postgresql + domainRegistration: postgresql + optSettings: setActivationTimeout: 4 setVerificationTimeout: 4 diff --git a/services/brig/src/Brig/App.hs b/services/brig/src/Brig/App.hs index d110b47f644..1e50c5179ed 100644 --- a/services/brig/src/Brig/App.hs +++ b/services/brig/src/Brig/App.hs @@ -73,6 +73,7 @@ module Brig.App enableSFTFederationLens, rateLimitEnvLens, amqpJobsPublisherChannelLens, + postgresMigrationLens, initZAuth, initLogger, initPostgresPool, @@ -167,6 +168,7 @@ import Wire.EmailSending.SMTP qualified as SMTP import Wire.EmailSubsystem.Template (Localised, TemplateBranding, forLocale) import Wire.EmailSubsystem.Templates.User import Wire.ExternalAccess.External +import Wire.PostgresMigrationOpts import Wire.RateLimit.Interpreter import Wire.SessionStore import Wire.SessionStore.Cassandra @@ -217,7 +219,8 @@ data Env = Env disabledVersions :: Set Version, enableSFTFederation :: Maybe Bool, rateLimitEnv :: RateLimitEnv, - amqpJobsPublisherChannel :: MVar Q.Channel + amqpJobsPublisherChannel :: MVar Q.Channel, + postgresMigration :: PostgresMigrationOpts } makeLensesWith (lensRules & lensField .~ suffixNamer) ''Env @@ -314,7 +317,8 @@ newEnv opts = do disabledVersions = allDisabledVersions, enableSFTFederation = opts.multiSFT, rateLimitEnv, - amqpJobsPublisherChannel + amqpJobsPublisherChannel, + postgresMigration = opts.postgresMigration } where emailConn _ (Opt.EmailAWS aws) = pure (Just aws, Nothing) diff --git a/services/brig/src/Brig/CanonicalInterpreter.hs b/services/brig/src/Brig/CanonicalInterpreter.hs index 270f1a7affa..06eb36f10ff 100644 --- a/services/brig/src/Brig/CanonicalInterpreter.hs +++ b/services/brig/src/Brig/CanonicalInterpreter.hs @@ -81,8 +81,12 @@ import Wire.ClientSubsystem.Interpreter import Wire.DeleteQueue import Wire.DomainRegistrationStore import Wire.DomainRegistrationStore.Cassandra +import Wire.DomainRegistrationStore.DualWrite +import Wire.DomainRegistrationStore.Postgres (interpretDomainRegistrationStoreToPostgres) import Wire.DomainVerificationChallengeStore import Wire.DomainVerificationChallengeStore.Cassandra +import Wire.DomainVerificationChallengeStore.DualWrite (interpretDomainVerificationChallengeStoreToCassandraAndPostgres) +import Wire.DomainVerificationChallengeStore.Postgres (interpretDomainVerificationChallengeStoreToPostgres) import Wire.EmailSending import Wire.EmailSending.SES import Wire.EmailSending.SMTP @@ -107,6 +111,7 @@ import Wire.IndexedUserStore import Wire.IndexedUserStore.ElasticSearch import Wire.InvitationStore (InvitationStore) import Wire.InvitationStore.Cassandra (interpretInvitationStoreToCassandra) +import Wire.MigrationLock import Wire.NotificationSubsystem import Wire.NotificationSubsystem.Interpreter (defaultNotificationSubsystemConfig, runNotificationSubsystemGundeck) import Wire.ParseException @@ -114,6 +119,7 @@ import Wire.PasswordResetCodeStore (PasswordResetCodeStore) import Wire.PasswordResetCodeStore.Cassandra (interpretClientToIO, passwordResetCodeStoreToCassandra) import Wire.PasswordStore (PasswordStore) import Wire.PasswordStore.Cassandra (interpretPasswordStore) +import Wire.PostgresMigrationOpts import Wire.PropertyStore import Wire.PropertyStore.Cassandra import Wire.PropertySubsystem @@ -200,10 +206,13 @@ type BrigLowerLevelEffects = BackgroundJobsPublisher, RateLimit, UserGroupStore, + DomainRegistrationStore, + DomainVerificationChallengeStore, Error AppSubsystemError, Error TeamCollaboratorsError, Error UsageError, Error EnterpriseLoginSubsystemError, + Error MigrationLockError, Error UserSubsystemError, Error UserGroupSubsystemError, Error TeamInvitationSubsystemError, @@ -216,8 +225,6 @@ type BrigLowerLevelEffects = ErrorS 'TeamNotFound, Error Wai.Error, Wire.FederationAPIAccess.FederationAPIAccess Wire.API.Federation.Client.FederatorClient, - DomainVerificationChallengeStore, - DomainRegistrationStore, CryptoSign, HashPassword, ClientStore, @@ -232,6 +239,7 @@ type BrigLowerLevelEffects = PropertyStore, SFT, ConnectionStore InternalPaging, + Input Cas.ClientState, Input Hasql.Pool, Input AppSubsystemConfig, Input UserSubsystemConfig, @@ -387,6 +395,15 @@ runBrigToIO e (AppT ma) = do local = localUnit, requestId = e.requestId } + domainRegistrationStore = case e.postgresMigration.domainRegistration of + CassandraStorage -> interpretDomainRegistrationStoreToCassandra e.casClient + PostgresqlStorage -> interpretDomainRegistrationStoreToPostgres + MigrationToPostgresql -> interpretDomainRegistrationStoreToCassandraAndPostgres e.casClient + + domainVerificationChallengeStore = case e.postgresMigration.domainRegistration of + CassandraStorage -> interpretDomainVerificationChallengeStoreToCassandra e.settings.challengeTTL + PostgresqlStorage -> interpretDomainVerificationChallengeStoreToPostgres e.settings.challengeTTL + MigrationToPostgresql -> interpretDomainVerificationChallengeStoreToCassandraAndPostgres e.settings.challengeTTL ( either throwM pure <=< ( runFinal @@ -426,6 +443,7 @@ runBrigToIO e (AppT ma) = do . runInputConst userSubsystemConfig . runInputConst appSubsystemConfig . runInputConst e.hasqlPool + . runInputConst e.casClient . connectionStoreToCassandra . interpretSFT e.httpManager . interpretPropertyStoreCassandra e.casClient @@ -440,8 +458,6 @@ runBrigToIO e (AppT ma) = do . interpretClientStoreCassandra clientStoreCassandraEnv . runHashPassword e.settings.passwordHashingOptions . runCryptoSign - . interpretDomainRegistrationStoreToCassandra e.casClient - . interpretDomainVerificationChallengeStoreToCassandra e.casClient e.settings.challengeTTL . interpretFederationAPIAccess federationApiAccessConfig . mapError StdError -- Wai.Error . mapError (const $ errorToWai @'TeamNotFound) -- ErrorS 'TeamNotFound @@ -454,10 +470,13 @@ runBrigToIO e (AppT ma) = do . mapError teamInvitationErrorToHttpError . mapError userGroupSubsystemErrorToHttpError . mapError userSubsystemErrorToHttpError + . mapError migrationLockErrorToHttpError . mapError enterpriseLoginSubsystemErrorToHttpError . mapError postgresUsageErrorToHttpError . mapError teamCollaboratorsSubsystemErrorToHttpError . mapError appSubsystemErrorToHttpError + . domainVerificationChallengeStore + . domainRegistrationStore . interpretUserGroupStoreToPostgres . interpretRateLimit e.rateLimitEnv . interpretBackgroundJobsPublisherRabbitMQ e.requestId e.amqpJobsPublisherChannel diff --git a/services/brig/src/Brig/Options.hs b/services/brig/src/Brig/Options.hs index cd2ae315b0b..da75b90138a 100644 --- a/services/brig/src/Brig/Options.hs +++ b/services/brig/src/Brig/Options.hs @@ -59,6 +59,7 @@ import Wire.AuthenticationSubsystem.Config (ZAuthSettings) import Wire.AuthenticationSubsystem.Cookie.Limit import Wire.EmailSending.SMTP (SMTPConnType (..)) import Wire.EmailSubsystem.Template (TeamOpts) +import Wire.PostgresMigrationOpts import Wire.RateLimit.Interpreter data ElasticSearchOpts = ElasticSearchOpts @@ -382,6 +383,7 @@ data Opts = Opts postgresql :: !(Map Text Text), postgresqlPassword :: !(Maybe FilePathSecrets), postgresqlPool :: !PoolConfig, + postgresMigration :: !PostgresMigrationOpts, -- | SFT Federation multiSFT :: !(Maybe Bool), -- | RabbitMQ settings, required when federation is enabled. diff --git a/services/galley/galley.integration.yaml b/services/galley/galley.integration.yaml index 10a7499f798..9d1c23291cb 100644 --- a/services/galley/galley.integration.yaml +++ b/services/galley/galley.integration.yaml @@ -252,3 +252,4 @@ postgresMigration: conversation: postgresql conversationCodes: postgresql teamFeatures: postgresql + domainRegistration: postgresql From e1984fc732d56a23f54a4f720139f0344ab0799b Mon Sep 17 00:00:00 2001 From: Stefan Matting Date: Mon, 11 May 2026 14:26:52 +0200 Subject: [PATCH 2/5] Add support for envoy gateway (#5150) --- Makefile | 5 +- changelog.d/5-internal/WPB-23903 | 1 + .../backoffice/templates/tests/configmap.yaml | 2 +- .../federator/templates/tests/configmap.yaml | 6 +- charts/federator/values.yaml | 4 + charts/integration/templates/_helpers.tpl | 45 +++ charts/integration/templates/configmap.yaml | 6 +- .../integration/templates/envoy-gateway.yaml | 161 ++++++++++ .../{ingress.yaml => ingress-nginx.yaml} | 23 ++ .../templates/integration-integration.yaml | 2 +- charts/integration/templates/service.yaml | 24 -- charts/integration/values.yaml | 23 ++ charts/wire-ingress/.helmignore | 21 ++ charts/wire-ingress/Chart.yaml | 4 + charts/wire-ingress/README.md | 298 ++++++++++++++++++ charts/wire-ingress/templates/_helpers.tpl | 75 +++++ .../backendtrafficpolicy-websockets.yaml | 26 ++ .../templates/certificate-federator.yaml | 37 +++ .../wire-ingress/templates/certificate.yaml | 45 +++ .../clienttrafficpolicy-federator.yaml | 28 ++ .../clienttrafficpolicy-proxy-protocol.yaml | 18 ++ .../envoyextensionpolicy-federator.yaml | 33 ++ .../templates/envoypatchpolicy-federator.yaml | 41 +++ charts/wire-ingress/templates/envoyproxy.yaml | 23 ++ charts/wire-ingress/templates/gateway.yaml | 64 ++++ .../templates/httproute-account-pages.yaml | 28 ++ .../templates/httproute-federator.yaml | 28 ++ .../templates/httproute-nginz-websockets.yaml | 28 ++ .../templates/httproute-nginz.yaml | 26 ++ .../wire-ingress/templates/httproute-s3.yaml | 30 ++ .../templates/httproute-team-settings.yaml | 28 ++ .../templates/httproute-webapp.yaml | 28 ++ charts/wire-ingress/templates/issuer.yaml | 38 +++ charts/wire-ingress/templates/secret.yaml | 17 + .../templates/service-account-pages.yaml | 18 ++ .../templates/service-team-settings.yaml | 18 ++ .../templates/service-test-fed.yaml | 38 +++ .../templates/service-webapp.yaml | 18 ++ charts/wire-ingress/values.yaml | 223 +++++++++++++ .../brig/tests/brig-integration.yaml | 2 +- .../templates/brig/tests/configmap.yaml | 2 +- .../templates/brig/tests/nginz-service.yaml | 2 +- .../templates/brig/tests/secret.yaml | 2 +- .../templates/cargohold/tests/configmap.yaml | 2 +- .../templates/galley/tests/configmap.yaml | 2 +- .../galley/tests/galley-integration.yaml | 2 +- .../templates/galley/tests/secret.yaml | 2 +- .../templates/gundeck/tests/configmap.yaml | 2 +- .../templates/spar/tests/configmap.yaml | 2 +- hack/bin/integration-setup-federation.sh | 36 ++- .../helm_vars/wire-ingress/values.yaml.gotmpl | 42 +++ hack/helm_vars/wire-server/values.yaml.gotmpl | 10 +- hack/helmfile-federation-v0.yaml.gotmpl | 110 ------- hack/helmfile.yaml.gotmpl | 36 ++- integration/test/Testlib/ModService.hs | 12 +- 55 files changed, 1671 insertions(+), 176 deletions(-) create mode 100644 changelog.d/5-internal/WPB-23903 create mode 100644 charts/integration/templates/envoy-gateway.yaml rename charts/integration/templates/{ingress.yaml => ingress-nginx.yaml} (70%) create mode 100644 charts/wire-ingress/.helmignore create mode 100644 charts/wire-ingress/Chart.yaml create mode 100644 charts/wire-ingress/README.md create mode 100644 charts/wire-ingress/templates/_helpers.tpl create mode 100644 charts/wire-ingress/templates/backendtrafficpolicy-websockets.yaml create mode 100644 charts/wire-ingress/templates/certificate-federator.yaml create mode 100644 charts/wire-ingress/templates/certificate.yaml create mode 100644 charts/wire-ingress/templates/clienttrafficpolicy-federator.yaml create mode 100644 charts/wire-ingress/templates/clienttrafficpolicy-proxy-protocol.yaml create mode 100644 charts/wire-ingress/templates/envoyextensionpolicy-federator.yaml create mode 100644 charts/wire-ingress/templates/envoypatchpolicy-federator.yaml create mode 100644 charts/wire-ingress/templates/envoyproxy.yaml create mode 100644 charts/wire-ingress/templates/gateway.yaml create mode 100644 charts/wire-ingress/templates/httproute-account-pages.yaml create mode 100644 charts/wire-ingress/templates/httproute-federator.yaml create mode 100644 charts/wire-ingress/templates/httproute-nginz-websockets.yaml create mode 100644 charts/wire-ingress/templates/httproute-nginz.yaml create mode 100644 charts/wire-ingress/templates/httproute-s3.yaml create mode 100644 charts/wire-ingress/templates/httproute-team-settings.yaml create mode 100644 charts/wire-ingress/templates/httproute-webapp.yaml create mode 100644 charts/wire-ingress/templates/issuer.yaml create mode 100644 charts/wire-ingress/templates/secret.yaml create mode 100644 charts/wire-ingress/templates/service-account-pages.yaml create mode 100644 charts/wire-ingress/templates/service-team-settings.yaml create mode 100644 charts/wire-ingress/templates/service-test-fed.yaml create mode 100644 charts/wire-ingress/templates/service-webapp.yaml create mode 100644 charts/wire-ingress/values.yaml create mode 100644 hack/helm_vars/wire-ingress/values.yaml.gotmpl delete mode 100644 hack/helmfile-federation-v0.yaml.gotmpl diff --git a/Makefile b/Makefile index e366edbb91e..6ecf57183ef 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ DOCKER_TAG ?= $(USER) # default helm chart version must be 0.0.42 for local development (because 42 is the answer to the universe and everything) HELM_SEMVER ?= 0.0.42 # The list of helm charts needed on internal kubernetes testing environments -CHARTS_INTEGRATION := wire-server databases-ephemeral rabbitmq fake-aws ingress-nginx-controller nginx-ingress-services fluent-bit kibana k8ssandra-test-cluster wire-server-enterprise +CHARTS_INTEGRATION := wire-server databases-ephemeral rabbitmq fake-aws ingress-nginx-controller nginx-ingress-services wire-ingress fluent-bit kibana k8ssandra-test-cluster wire-server-enterprise # The list of helm charts to publish on S3 # FUTUREWORK: after we "inline local subcharts", # (e.g. move charts/brig to charts/wire-server/brig) @@ -18,7 +18,8 @@ fake-aws fake-aws-s3 fake-aws-sqs aws-ingress fluent-bit kibana backoffice \ calling-test demo-smtp elasticsearch-curator elasticsearch-external \ elasticsearch-ephemeral minio-external cassandra-external \ ingress-nginx-controller nginx-ingress-services reaper \ -k8ssandra-test-cluster ldap-scim-bridge wire-server-enterprise +k8ssandra-test-cluster ldap-scim-bridge wire-server-enterprise \ +wire-ingress KIND_CLUSTER_NAME := wire-server HELM_PARALLELISM ?= 1 # 1 for sequential tests; 6 for all-parallel tests PSQL_DB ?= backendA diff --git a/changelog.d/5-internal/WPB-23903 b/changelog.d/5-internal/WPB-23903 new file mode 100644 index 00000000000..c7a1e11ef4f --- /dev/null +++ b/changelog.d/5-internal/WPB-23903 @@ -0,0 +1 @@ +New `wire-ingress` Helm chart — Gateway API / Envoy Gateway replacement for `nginx-ingress-services`. Not yet production-ready. diff --git a/charts/backoffice/templates/tests/configmap.yaml b/charts/backoffice/templates/tests/configmap.yaml index a20785b354e..b4bff0a2e03 100644 --- a/charts/backoffice/templates/tests/configmap.yaml +++ b/charts/backoffice/templates/tests/configmap.yaml @@ -3,7 +3,7 @@ kind: ConfigMap metadata: name: "stern-integration" annotations: - "helm.sh/hook": post-install + "helm.sh/hook": post-install,post-upgrade "helm.sh/hook-delete-policy": before-hook-creation data: integration.yaml: | diff --git a/charts/federator/templates/tests/configmap.yaml b/charts/federator/templates/tests/configmap.yaml index 44146840bd4..4612ab2a3c5 100644 --- a/charts/federator/templates/tests/configmap.yaml +++ b/charts/federator/templates/tests/configmap.yaml @@ -3,7 +3,7 @@ kind: ConfigMap metadata: name: "federator-integration" annotations: - "helm.sh/hook": post-install + "helm.sh/hook": post-install,post-upgrade "helm.sh/hook-delete-policy": before-hook-creation data: integration.yaml: | @@ -23,6 +23,6 @@ data: host: cargohold port: 8080 nginxIngress: - host: federation-test-helper.{{ .Release.Namespace }}.svc.cluster.local + host: {{ .Values.tests.nginxIngressHost }} port: 443 - originDomain: federation-test-helper.{{ .Release.Namespace }}.svc.cluster.local + originDomain: {{ .Values.tests.nginxIngressHost }} diff --git a/charts/federator/values.yaml b/charts/federator/values.yaml index 6632fb21429..c0285e52d8c 100644 --- a/charts/federator/values.yaml +++ b/charts/federator/values.yaml @@ -57,6 +57,10 @@ podSecurityContext: type: RuntimeDefault tests: + # The host used for the nginxIngress endpoint and originDomain in the integration + # test config. Depends on the release name of the "wire-ingress" helm chart + # (see federation-test-helper.yaml in that chart). + nginxIngressHost: "set-me" config: {} # config: # uploadXml: diff --git a/charts/integration/templates/_helpers.tpl b/charts/integration/templates/_helpers.tpl index e3a33787bf9..1c0bd85ad52 100644 --- a/charts/integration/templates/_helpers.tpl +++ b/charts/integration/templates/_helpers.tpl @@ -1,4 +1,49 @@ +{{/* +Name of the Gateway resource for dynamic backends in envoy mode. +*/}} +{{- define "integration.getDynBackendsGatewayName" -}} +{{- if .Values.envoy.gateway.name -}} +{{ .Values.envoy.gateway.name }} +{{- else -}} +{{ .Release.Name }}-dynamic-backends +{{- end -}} +{{- end -}} + +{{/* +Federation origin domain for a given namespace (used as originDomain in the config). +Returns the SRV hostname that other backends use to reach this namespace's federator. +NOTE: Keep the naming assumption %s-fed in sync with the wire-ingress and nginx-ingress-services chart! +Args: list $namespace $envoyEnabled $controllerNamespace +*/}} +{{- define "integration.federationOriginDomain" -}} +{{- $namespace := index . 0 -}} +{{- $envoyEnabled := index . 1 -}} +{{- $controllerNs := index . 2 -}} +{{- if $envoyEnabled -}} +{{- printf "%s-fed.%s.svc.cluster.local" $namespace $controllerNs -}} +{{- else -}} +{{- printf "federation-test-helper.%s.svc.cluster.local" $namespace -}} +{{- end -}} +{{- end -}} + +{{/* +Domain for a dynamic backend. Returns the correct hostname depending on whether +envoy mode is enabled. +Args: list $dynamicBackend $namespace $envoyEnabled $controllerNamespace +*/}} +{{- define "integration.dynamicBackendDomain" -}} +{{- $dynamicBackend := index . 0 -}} +{{- $namespace := index . 1 -}} +{{- $envoyEnabled := index . 2 -}} +{{- $controllerNs := index . 3 -}} +{{- if $envoyEnabled -}} +{{- printf "%s-%s.%s.svc.cluster.local" $dynamicBackend.federatorExternalHostPrefix $namespace $controllerNs -}} +{{- else -}} +{{- printf "%s.%s.svc.cluster.local" $dynamicBackend.federatorExternalHostPrefix $namespace -}} +{{- end -}} +{{- end -}} + {{/* Allow KubeVersion to be overridden. */}} {{- define "kubeVersion" -}} {{- default $.Capabilities.KubeVersion.Version $.Values.kubeVersionOverride -}} diff --git a/charts/integration/templates/configmap.yaml b/charts/integration/templates/configmap.yaml index 82fc9895284..b5f2d351732 100644 --- a/charts/integration/templates/configmap.yaml +++ b/charts/integration/templates/configmap.yaml @@ -77,7 +77,7 @@ data: apiPort: 5380 dohPort: 5381 - originDomain: federation-test-helper.{{ .Release.Namespace }}.svc.cluster.local + originDomain: {{ include "integration.federationOriginDomain" (list .Release.Namespace .Values.envoy.enabled .Values.envoy.controllerNamespace) }} rabbitmq: host: rabbitmq @@ -158,12 +158,12 @@ data: rabbitMqVHost: / - originDomain: federation-test-helper.{{ .Release.Namespace }}-fed2.svc.cluster.local + originDomain: {{ include "integration.federationOriginDomain" (list (printf "%s-fed2" .Release.Namespace) .Values.envoy.enabled .Values.envoy.controllerNamespace) }} dynamicBackends: {{- range $name, $dynamicBackend := .Values.config.dynamicBackends }} {{ $name }}: - domain: {{ $dynamicBackend.federatorExternalHostPrefix }}.{{ $.Release.Namespace }}.svc.cluster.local + domain: {{ include "integration.dynamicBackendDomain" (list $dynamicBackend $.Release.Namespace $.Values.envoy.enabled $.Values.envoy.controllerNamespace) }} federatorExternalPort: {{ $dynamicBackend.federatorExternalPort }} mlsPrivateKeyPaths: removal: diff --git a/charts/integration/templates/envoy-gateway.yaml b/charts/integration/templates/envoy-gateway.yaml new file mode 100644 index 00000000000..98ddae3617e --- /dev/null +++ b/charts/integration/templates/envoy-gateway.yaml @@ -0,0 +1,161 @@ +{{- if .Values.envoy.enabled }} +{{- $gatewayName := include "integration.getDynBackendsGatewayName" . }} +{{- $httpsPort := int .Values.envoy.gateway.listeners.https.port }} +{{- $controllerNs := .Values.envoy.controllerNamespace }} +{{- if lt $httpsPort 1024 }} +{{- fail (printf "envoy.gateway.listeners.https.port is %d (privileged, <1024). Envoy Gateway remaps it to %d on the proxy pod. Set envoy.gateway.listeners.https.port to the actual container port (e.g. %d)." $httpsPort (add $httpsPort 10000) (add $httpsPort 10000)) }} +{{- end }} +--- +# EnvoyProxy configures the proxy deployment/service for the dynamic-backends Gateway. +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyProxy +metadata: + name: {{ $gatewayName }} +spec: + provider: + type: Kubernetes + kubernetes: + envoyService: + # ClusterIP: no external load balancer needed for in-cluster integration tests. + type: ClusterIP +--- +# Gateway for all dynamic backends. A single HTTPS listener covers all backend hostnames. +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: {{ $gatewayName }} +spec: + gatewayClassName: {{ required "envoy.gateway.className is required when envoy.enabled is true" .Values.envoy.gateway.className | quote }} + infrastructure: + parametersRef: + group: gateway.envoyproxy.io + kind: EnvoyProxy + name: {{ $gatewayName | quote }} + listeners: + - name: https + port: {{ $httpsPort }} + protocol: HTTPS + tls: + mode: Terminate + certificateRefs: + - name: {{ .Values.envoy.federator.tls.secretName | quote }} + kind: Secret +--- +# ClientTrafficPolicy enforces optional mTLS client cert validation on all dynamic-backend +# connections (mirrors the nginx auth-tls-verify-client: "on" annotation). +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: ClientTrafficPolicy +metadata: + name: {{ $gatewayName }}-mtls +spec: + targetRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: {{ $gatewayName | quote }} + sectionName: https + tls: + clientValidation: + optional: true + caCertificateRefs: + - name: federator-ca + kind: ConfigMap +--- +{{- $backendNames := keys .Values.config.dynamicBackends | sortAlpha }} +{{- range $index, $name := $backendNames }} +{{- $dynamicBackend := index $.Values.config.dynamicBackends $name }} +{{- $httpRouteName := printf "%s-dynbackend-%s" $gatewayName $name }} +{{- $svcDomain := printf "%s-%s.%s.svc.cluster.local" $dynamicBackend.federatorExternalHostPrefix $.Release.Namespace $controllerNs }} +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: {{ $httpRouteName }} +spec: + parentRefs: + - name: {{ $gatewayName | quote }} + namespace: {{ $.Release.Namespace | quote }} + kind: Gateway + sectionName: https + hostnames: + - {{ $svcDomain | quote }} + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - name: integration + port: {{ $dynamicBackend.federatorExternalPort }} + kind: Service +--- +# EnvoyExtensionPolicy injects the mTLS client certificate as X-SSL-Certificate request +# header, matching the nginx $ssl_client_escaped_cert behaviour expected by federator. +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyExtensionPolicy +metadata: + name: {{ $httpRouteName }}-cert-header +spec: + targetRefs: + - group: gateway.networking.k8s.io + kind: HTTPRoute + name: {{ $httpRouteName }} + lua: + - type: Inline + inline: | + function envoy_on_request(request_handle) + {{/* Strip any client-provided header to prevent spoofing */}} + request_handle:headers():remove("X-SSL-Certificate") + local ssl = request_handle:connection():ssl() + if ssl ~= nil then + local cert = ssl:urlEncodedPemEncodedPeerCertificate() + if cert ~= nil and cert ~= "" then + request_handle:headers():add("X-SSL-Certificate", cert) + end + end + end +--- +# EnvoyPatchPolicy adds the FQDN variant (with trailing dot) of the backend domain +# to the virtual host's domain list. Wire federator resolves targets via DNS SRV records; +# per RFC 2782, SRV record targets are FQDNs (e.g. "backend-fed.ns.svc.cluster.local."). +# HTTP/2 passes that dot in :authority; without this patch the virtual host only matches +# the bare domain and returns route_not_found. Adding the FQDN allows Envoy to match both. +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyPatchPolicy +metadata: + name: {{ $httpRouteName }}-fqdn-domain +spec: + targetRef: + group: gateway.networking.k8s.io + kind: Gateway + name: {{ $gatewayName | quote }} + type: JSONPatch + jsonPatches: + - type: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration" + # RouteConfiguration is per-listener, named // + name: {{ printf "%s/%s/https" $.Release.Namespace $gatewayName | quote }} + operation: + op: add + # Virtual hosts are indexed in the order of stable key sorting (sortAlpha). + path: {{ printf "/virtual_hosts/%d/domains/-" $index | quote }} + value: {{ printf "%s." $svcDomain | quote }} +--- +# ClusterIP service in {{ $controllerNs }} selects the Envoy proxy pods for this Gateway. +# The service name determines the SRV record used by federation discovery: +# _wire-server-federator._tcp.{{ $svcDomain }} +apiVersion: v1 +kind: Service +metadata: + name: {{ $dynamicBackend.federatorExternalHostPrefix }}-{{ $.Release.Namespace }} + namespace: {{ $controllerNs }} +spec: + type: ClusterIP + ports: + - name: wire-server-federator + port: 443 + protocol: TCP + targetPort: {{ $httpsPort }} + selector: + gateway.envoyproxy.io/owning-gateway-name: {{ $gatewayName }} + gateway.envoyproxy.io/owning-gateway-namespace: {{ $.Release.Namespace }} +{{- end }} +{{- end }} diff --git a/charts/integration/templates/ingress.yaml b/charts/integration/templates/ingress-nginx.yaml similarity index 70% rename from charts/integration/templates/ingress.yaml rename to charts/integration/templates/ingress-nginx.yaml index 362b7b0d8f9..81fc91013a1 100644 --- a/charts/integration/templates/ingress.yaml +++ b/charts/integration/templates/ingress-nginx.yaml @@ -1,3 +1,5 @@ +{{- if not .Values.envoy.enabled }} +{{- $newLabels := eq (include "integrationTestHelperNewLabels" .) "true" -}} {{- range $name, $dynamicBackend := .Values.config.dynamicBackends }} --- apiVersion: networking.k8s.io/v1 @@ -29,4 +31,25 @@ spec: name: integration port: number: {{ $dynamicBackend.federatorExternalPort }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ $dynamicBackend.federatorExternalHostPrefix }} +spec: + ports: + - name: wire-server-federator + port: 443 + protocol: TCP + targetPort: https + selector: + {{- if $newLabels }} + app.kubernetes.io/component: controller + app.kubernetes.io/name: ingress-nginx + {{- else }} + app: nginx-ingress + component: controller + {{- end }} + type: ClusterIP +{{- end }} {{- end }} diff --git a/charts/integration/templates/integration-integration.yaml b/charts/integration/templates/integration-integration.yaml index 7ebad70c179..bf4f676fabd 100644 --- a/charts/integration/templates/integration-integration.yaml +++ b/charts/integration/templates/integration-integration.yaml @@ -168,7 +168,7 @@ spec: integration-dynamic-backends-ses.sh {{ .Values.config.sesEndpointUrl }} integration-dynamic-backends-s3.sh {{ .Values.config.s3EndpointUrl }} {{- range $name, $dynamicBackend := .Values.config.dynamicBackends }} - integration-dynamic-backends-vhosts.sh {{ $.Values.config.rabbitmqPutVHostUrl }} {{ $dynamicBackend.federatorExternalHostPrefix}}.{{ $.Release.Namespace }}.svc.cluster.local + integration-dynamic-backends-vhosts.sh {{ $.Values.config.rabbitmqPutVHostUrl }} {{ include "integration.dynamicBackendDomain" (list $dynamicBackend $.Release.Namespace $.Values.envoy.enabled $.Values.envoy.controllerNamespace) }} {{- end }} resources: requests: diff --git a/charts/integration/templates/service.yaml b/charts/integration/templates/service.yaml index 350b33f11f7..a97d1e58c8e 100644 --- a/charts/integration/templates/service.yaml +++ b/charts/integration/templates/service.yaml @@ -1,4 +1,3 @@ -{{- $newLabels := eq (include "integrationTestHelperNewLabels" .) "true" -}} --- apiVersion: v1 kind: Service @@ -26,26 +25,3 @@ spec: selector: app: integration-integration type: ClusterIP - -{{- range $name, $dynamicBackend := .Values.config.dynamicBackends }} ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ $dynamicBackend.federatorExternalHostPrefix }} -spec: - ports: - - name: wire-server-federator - port: 443 - protocol: TCP - targetPort: https - selector: - {{- if $newLabels }} - app.kubernetes.io/component: controller - app.kubernetes.io/name: ingress-nginx - {{- else }} - app: nginx-ingress - component: controller - {{- end }} - type: ClusterIP -{{- end }} diff --git a/charts/integration/values.yaml b/charts/integration/values.yaml index 36305b2be75..65c7963b0d8 100644 --- a/charts/integration/values.yaml +++ b/charts/integration/values.yaml @@ -129,4 +129,27 @@ tls: ingress: class: nginx +envoy: + # Set to true to deploy Gateway API resources instead of nginx Ingress objects + # for the dynamic backends. Requires an Envoy Gateway controller in the cluster. + enabled: false + # Namespace where the Envoy Gateway controller runs its proxy pods. + # Change only if you installed Envoy Gateway into a non-default namespace. + controllerNamespace: envoy-gateway-system + gateway: + # Name of the Gateway resource. Defaults to -dynamic-backends if empty. + name: "" + # Name of the GatewayClass installed by the Envoy Gateway controller (e.g. "envoy"). + className: "" + listeners: + https: + # Use a non-privileged port (>=1024) to avoid the +10000 container-port + # remapping applied by Envoy Gateway to privileged ports. + port: 10443 + federator: + tls: + # Name of the TLS Secret presented by the Gateway for the dynamic-backend + # listeners. Must exist before deploying (created by the wire-ingress chart). + secretName: "federator-certificate-secret" + secrets: {} diff --git a/charts/wire-ingress/.helmignore b/charts/wire-ingress/.helmignore new file mode 100644 index 00000000000..f0c13194444 --- /dev/null +++ b/charts/wire-ingress/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/wire-ingress/Chart.yaml b/charts/wire-ingress/Chart.yaml new file mode 100644 index 00000000000..fabe75062d3 --- /dev/null +++ b/charts/wire-ingress/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v2 +description: A Helm chart for Wire server ingress using the Kubernetes Gateway API +name: wire-ingress +version: 0.1.0 diff --git a/charts/wire-ingress/README.md b/charts/wire-ingress/README.md new file mode 100644 index 00000000000..05dce813be1 --- /dev/null +++ b/charts/wire-ingress/README.md @@ -0,0 +1,298 @@ +# wire-ingress + +A Helm chart for Wire server ingress using the **Kubernetes Gateway API**. + +The chart targets **Envoy Gateway** as the Gateway API controller. + +--- + +## Status + +**This chart is in development. Don't use it in production yet! See FUTUREWORK below** + +--- + +## Prerequisites + +### Gateway API + +Install the [Gateway API](https://gateway-api.sigs.k8s.io/) into your cluster. +This chart makes use of the kinds defined in the `gateway.networking.k8s.io/v1` API. + +You must use install it in the same namespace as the `wire-server` helm chart, otherwise references will not work. +FUTUREWORK: Make this helm chart a subchart of `wire-server` before releasing it and remove this paragraph. + +### Envoy Gateway + +[Envoy Gateway](https://gateway.envoyproxy.io/) must be installed in the cluster before deploying +this chart. The `EnvoyPatchPolicy` extension API must be enabled (required for federation — see +[EnvoyPatchPolicy](#envoypatchpolicy)): + +```yaml +config: + envoyGateway: + extensionApis: + enableEnvoyPatchPolicy: true +``` + +Also make sure you've created a `GatewayClass` object with +``` +spec: + controllerName: gateway.envoyproxy.io/gatewayclass-controller +``` + +You need to refer to this object in the `gateway.className` parameter. + +--- + +## Backwards compatibility + + +### Migrating from the `nginx-ingress-services` chart + +The chart preserves the `values.yaml` structure of the `nginx-ingress-services` chart wherever +possible. Most existing values files should work with minimal changes. + +Add a `gateway` block to your values and review at least the following keys: + +- `gateway.className` — set to the `GatewayClass` name created during installation (see above). +- `gateway.create` — if `false`, you must create a `Gateway` object yourself and set `gateway.name` to its name. +- `gateway.listeners.https.hostname` — set to `*.`. This assumes all domains under + `config.dns.*` are subdomains of ``. If that is not the case, create your own + `Gateway` and set `gateway.create: false`. +- `gateway.proxyProtocol.enabled` — set to `true` if your load balancer sends PROXY protocol headers. +- `gateway.patchPolicies.targetGatewayClass` — depends on your setup; see [EnvoyPatchPolicy](#envoypatchpolicy). +- `gateway.envoyProxy.create` and `gateway.manageServiceType` — depend on your setup; see the parameter table below. + +`secrets.tlsClientCA` is no longer needed and can be removed. + +### Behavior changes + +* non-tls ingress disabled by default. If you want to make use of automated certificate validation via http01, you need `gateway.listeners.http.enabled: true` +* s3 ingress `/minio/` path blocking. Returns 301 redirect to "/" (was 403). + +### New values (no equivalent in nginx-ingress-services) + +Only values that require explanation are listed. Trivial or self-explanatory values (ports, +name overrides, etc.) can be found in `values.yaml`. + +| Key | Default | Description | +|---|---|---| +| `gateway.create` | `true` | If `false`, no `Gateway` resource is created — set `gateway.name` to reference an existing one. Useful when sharing a Gateway across multiple releases. | +| `gateway.className` | `""` | **Required.** Name of the `GatewayClass` installed by the Envoy Gateway controller (e.g. `envoy`). Must match the `GatewayClass` object whose `spec.controllerName` is `gateway.envoyproxy.io/gatewayclass-controller`. | +| `gateway.listeners.https.hostname` | `""` | **Required when `federator.enabled: true`.** Restricts the HTTPS listener to a specific hostname (e.g. `*.example.com`). Without this, both the HTTPS and federator listeners are catch-all on the same port, causing Envoy to degrade ALPN to HTTP/1.1-only (`OverlappingTLSConfig`). | +| `gateway.listeners.http.enabled` | `false` | Enables the HTTP listener on port 80. Required for HTTP01 ACME challenges via cert-manager's `gatewayHTTPRoute` solver — see [HTTP01 certificate challenges](#http01-certificate-challenges). | +| `gateway.envoyProxy.create` | `true` | If `false`, no `EnvoyProxy` resource is created. Set `gateway.envoyProxy.name` to reference an existing one, or leave it empty to inherit the GatewayClass-level `EnvoyProxy`. | +| `gateway.envoyProxy.name` | _(derived)_ | When `create: true` — name of the created resource. When `create: false` — name of an existing `EnvoyProxy` to reference via `infrastructure.parametersRef`. | +| `gateway.envoyProxy.spec` | `{}` | Free-form [EnvoyProxySpec](https://gateway.envoyproxy.io/docs/api/extension_types/#envoyproxyspec) merged verbatim. Use to set `mergeGateways`, custom service annotations, etc. | +| `gateway.manageServiceType` | `true` | Shorthand that sets `envoyService.type` to `gateway.serviceType`. Disable when managing the service type via `gateway.envoyProxy.spec` directly. | +| `gateway.serviceType` | `LoadBalancer` | Service type for the Envoy proxy service. Only used when `gateway.manageServiceType: true`. | +| `gateway.infrastructure.annotations` | `{}` | Annotations forwarded to the LoadBalancer Service provisioned by Envoy Gateway — see [Gateway API docs](https://gateway-api.sigs.k8s.io/reference/spec/#gateway.networking.k8s.io/v1.GatewayInfrastructure). Use for cloud-specific LB settings (e.g. AWS NLB). | +| `gateway.proxyProtocol.enabled` | `false` | Creates a `ClientTrafficPolicy` enabling PROXY protocol on all listeners. Required when the upstream load balancer is configured to send PROXY protocol headers. | +| `gateway.patchPolicies.enabled` | `true` | Controls whether `EnvoyPatchPolicy` resources are created — see [EnvoyPatchPolicy](#envoypatchpolicy). | +| `gateway.patchPolicies.targetGatewayClass` | `false` | When `true`, `EnvoyPatchPolicy` targets the `GatewayClass` instead of the `Gateway`. **Required when `gateway.envoyProxy.spec.mergeGateways: true`**: with merged Gateways, policies targeting a `Gateway` are not applied — they must target the `GatewayClass`. Leave `false` for single-Gateway deployments (e.g. integration tests). | +| `gateway.controllerNamespace` | `envoy-gateway-system` | Can be ignored, relevant only for integration tests. Namespace where Envoy Gateway runs its proxy pods. Change only if Envoy Gateway was installed into a non-default namespace. | +| `tls.secret.create` | `true` | If `false`, the TLS Secret is not created by this chart. Use when the secret is managed externally (e.g. by another operator). | +| `federator.tls.useCertManager` | `true` | Controls cert-manager for the federator TLS secret independently of `tls.useCertManager`. Requires a private CA — see [Federator TLS certificate](#federator-tls-certificate-federatortlsusecertmanager). | + +### Dropped values + +| Old key | Reason | +|---|---| +| `config.ingressClass` | | +| `ingressName` | Multi-ingress out of scope | +| `config.isAdditionalIngress` | Multi-ingress out of scope | +| `config.renderCSPInIngress` | Multi-ingress out of scope | +| `config.dns.base` | Only used for CSP header rendering, which is a multi-ingress feature | +| `tls.verify_depth` | Envoy Gateway `ClientTrafficPolicy` does not expose a direct verify-depth knob; the CA chain itself controls this | +| `tls.enabled` | Removed — had no effect; all routes are always TLS-terminated | +| `secrets.tlsClientCA` | No longer supplied via values. The `federator-ca` ConfigMap is created by the wire-server chart and referenced directly. | +| `secrets.certManager.customSolversSecret` | No longer supported. Create a custom Issuer instead. | + +### Fully backwards compatible values + +All keys below are accepted unchanged. Their names, types, and semantics are identical to +`nginx-ingress-services`. + +| Key | +|---| +| `nameOverride` | +| `teamSettings.enabled` | +| `accountPages.enabled` | +| `websockets.enabled` | +| `webapp.enabled` | +| `fakeS3.enabled` | +| `federator.enabled` | +| `federator.integrationTestHelper` | +| `federator.tls.duration` | +| `federator.tls.renewBefore` | +| `federator.tls.privateKey.rotationPolicy` | +| `federator.tls.issuer.name` | +| `federator.tls.issuer.kind` | +| `federator.tls.issuer.group` | +| `tls.useCertManager` | +| `tls.createIssuer` | +| `tls.privateKey.rotationPolicy` | +| `tls.privateKey.algorithm` | +| `tls.privateKey.size` | +| `tls.issuer.name` | +| `tls.issuer.kind` | +| `tls.caNamespace` | +| `certManager.inTestMode` | +| `certManager.certmasterEmail` | +| `certManager.customSolvers` | +| `service.webapp.externalPort` | +| `service.s3.externalPort` | +| `service.s3.serviceName` | +| `service.useFakeS3` | +| `service.teamSettings.externalPort` | +| `service.accountPages.externalPort` | +| `config.dns.https` | +| `config.dns.ssl` | +| `config.dns.webapp` | +| `config.dns.fakeS3` | +| `config.dns.federator` | +| `config.dns.certificateDomain` | +| `config.dns.teamSettings` | +| `config.dns.accountPages` | +| `secrets.tlsWildcardCert` | +| `secrets.tlsWildcardKey` | + + +## Design decisions + +### Gateway API controller: Envoy Gateway + +The chart targets [Envoy Gateway](https://gateway.envoyproxy.io/). Implementation-specific +resources (`ClientTrafficPolicy`, `SecurityPolicy`, `HTTPRouteFilter` with `directResponse`) are +used where the standard Gateway API has gaps. These resources are clearly marked in each template. + +### Gateway creation is optional + +The chart can optionally create a `Gateway` resource (controlled by `gateway.create: true`). +When `gateway.create: false`, all `HTTPRoute` and policy resources still reference the gateway by +name (`gateway.name`). This allows operators to share a Gateway across multiple charts or manage it +separately. + +The default values create the Gateway. The default `gateway.name` is derived from the release name, +so that self-referencing is consistent by default. + +### EnvoyProxy resource + +The chart creates an `EnvoyProxy` resource (when `gateway.envoyProxy.create: true`) and wires it +to the `Gateway` via `infrastructure.parametersRef`. Use `gateway.envoyProxy.spec` to pass +arbitrary fields from the [EnvoyProxySpec](https://gateway.envoyproxy.io/docs/api/extension_types/#envoyproxyspec). + +Set `gateway.envoyProxy.create: false` when a shared `EnvoyProxy` is managed at the +`GatewayClass` level (e.g. shared load balancer across deployments) — leave `gateway.envoyProxy.name` +empty and the Gateway will have no `infrastructure.parametersRef`, letting the `GatewayClass`-level +`EnvoyProxy` take effect automatically. + +Set `gateway.envoyProxy.name` (with `create: false`) to reference an existing `EnvoyProxy` in the +**same namespace** via `infrastructure.parametersRef`. + +`gateway.manageServiceType: true` (default) is a shorthand that sets +`provider.kubernetes.envoyService.type` to `gateway.serviceType`. Disable it when managing +the service type via `envoyProxy.spec` or a cluster-level `EnvoyProxy`. + +### GatewayClass is not created + +`GatewayClass` is installed by the Envoy Gateway Helm chart and is cluster-scoped. This chart only +references it by name via `gateway.className`. + +### EnvoyPatchPolicy + +When `federator.enabled: true`, the chart creates an `EnvoyPatchPolicy` resource that adds the +FQDN variant of the federator hostname (e.g. `federator.example.com.`, with trailing dot) to the +Envoy virtual host's domain list. + +**Why this is needed:** Wire federation resolves remote backends via DNS SRV records. Per the DNS +specification, SRV record targets are always FQDNs — they include a trailing dot +(e.g. `peer.example.com.`). The federator passes this FQDN directly as the HTTP/2 `:authority` +header. Envoy's virtual-host matching is exact, so the trailing dot causes a `route_not_found` +error. Adding the FQDN as an additional domain in the route configuration allows Envoy to match +both the bare hostname and the FQDN. + +The policy patches the `RouteConfiguration` named `//federator`. Route +configuration names are per-namespace even when multiple Gateways share a single Envoy proxy, so +the name is predictable from chart values. + +**`gateway.patchPolicies.targetGatewayClass`** controls what the policy targets: + +- **`false` (default)** — targets `kind: Gateway` by name. Use for standard single-Gateway + deployments, including integration tests. +- **`true`** — targets `kind: GatewayClass` (using `gateway.className`). **Required when + `gateway.envoyProxy.spec.mergeGateways: true`.** With merged Gateways, all Gateways of the same + GatewayClass share one Envoy proxy. + +> **Future note:** If future versions of the Wire federator stop sending FQDNs in the +> `:authority` header, this patch policy will no longer be needed. `gateway.patchPolicies.enabled` +> exists so it can be disabled at that point without a chart change. + +--- + +### Multi-ingress is out of scope + +Single-domain deployments are the only supported topology. Multi-domain support can be added later. + +### HTTP01 certificate challenges + +cert-manager can complete ACME HTTP01 challenges through the Gateway using the `gatewayHTTPRoute` +solver (cert-manager >= 1.14). The **default solver** in this chart uses `gatewayHTTPRoute` — it +requires the HTTP listener to be enabled: + +```yaml +gateway: + listeners: + http: + enabled: true # required for HTTP01 challenges +``` + +If you cannot or do not want to open port 80, use a DNS01 solver instead by setting + +```yaml +certManager: + customSolvers: + - dns01: + # .. provider-specific settings +``` + +DNS01 requires credentials for your DNS provider but does not need +port 80 to be open. + +### Federator TLS certificate (`federator.tls.useCertManager`) + +When `federator.tls.useCertManager: true`, cert-manager issues the federator TLS certificate. +The certificate requires both **server auth** and **client auth** Extended Key Usages (EKUs), +because federator connections are mutually authenticated. + +**Most public CAs (including Let's Encrypt) no longer issue certificates with the client auth +EKU.** You will need a **private CA** (e.g. a cert-manager `ClusterIssuer` backed by an internal +CA) to issue the federator certificate. Using the same public ACME issuer as for the main +wildcard certificate will not work. + +A typical setup uses a cert-manager `ClusterIssuer` of type `CA`, referencing a private CA +secret: + +```yaml +federator: + tls: + useCertManager: true + issuer: + name: my-private-ca + kind: ClusterIssuer +``` + +--- + +### Federator mTLS uses Envoy Gateway policies + +Federator mTLS is implemented using: + +- `ClientTrafficPolicy` to configure TLS settings on the federator `Gateway` listener (client + certificate validation, verify depth) +- A separate `Gateway` listener (or dedicated `Gateway`) for the federator so that mTLS settings + apply only to that listener +- `X-SSL-Certificate` header forwarding is handled via an `EnvoyExtensionPolicy` with an inline + Lua filter that reads the URL-encoded PEM client certificate from the connection and injects it + as a request header, matching nginx's `$ssl_client_escaped_cert` behaviour diff --git a/charts/wire-ingress/templates/_helpers.tpl b/charts/wire-ingress/templates/_helpers.tpl new file mode 100644 index 00000000000..263f190c90c --- /dev/null +++ b/charts/wire-ingress/templates/_helpers.tpl @@ -0,0 +1,75 @@ +{{/* vim: set filetype=mustache: */}} + +{{- define "wire-ingress.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "wire-ingress.fullname" -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Determine DNS zone based on the HTTPS FQDN (e.g. "nginz-https.example.com" → "example.com") +*/}} +{{- define "wire-ingress.zone" -}} +{{- $zones := splitList "." .Values.config.dns.https -}} +{{- slice $zones 1 | join "." -}} +{{- end -}} + +{{/* +Name of the TLS certificate secret. Differs based on whether cert-manager is used. +*/}} +{{- define "wire-ingress.certificateSecretName" -}} +{{- if .Values.tls.secret.nameOverride -}} + {{- .Values.tls.secret.nameOverride -}} +{{- else -}} + {{- $nameParts := list (include "wire-ingress.fullname" .) -}} + {{- if .Values.tls.useCertManager -}} + {{- $nameParts = append $nameParts "managed" -}} + {{- else -}} + {{- $nameParts = append $nameParts "wildcard" -}} + {{- end -}} + {{- $nameParts = append $nameParts "tls-certificate" -}} + {{- join "-" $nameParts -}} +{{- end -}} +{{- end -}} + +{{/* +Name of the custom ACME solver secret. +*/}} +{{- define "wire-ingress.customSolversSecretName" -}} +{{- $nameParts := list (include "wire-ingress.fullname" .) -}} +{{- $nameParts = append $nameParts "cert-manager-custom-solvers" -}} +{{- join "-" $nameParts -}} +{{- end -}} + +{{/* +Returns the Letsencrypt ACME API server URL. +*/}} +{{- define "wire-ingress.certManagerAPIServerURL" -}} +{{- $hostnameParts := list "acme" -}} +{{- if .Values.certManager.inTestMode -}} + {{- $hostnameParts = append $hostnameParts "staging" -}} +{{- end -}} +{{- $hostnameParts = append $hostnameParts "v02" -}} +{{- join "-" $hostnameParts | printf "https://%s.api.letsencrypt.org/directory" -}} +{{- end -}} + +{{/* +Name of the cert-manager Issuer / ClusterIssuer. +*/}} +{{- define "wire-ingress.issuerName" -}} +{{ .Values.tls.issuer.name }} +{{- end -}} + +{{/* +Name of the Gateway resource. Uses gateway.name if set, otherwise derives one from the release name. +*/}} +{{- define "wire-ingress.gatewayName" -}} +{{- if .Values.gateway.name -}} +{{ .Values.gateway.name }} +{{- else -}} +{{ include "wire-ingress.fullname" . }}-gateway +{{- end -}} +{{- end -}} diff --git a/charts/wire-ingress/templates/backendtrafficpolicy-websockets.yaml b/charts/wire-ingress/templates/backendtrafficpolicy-websockets.yaml new file mode 100644 index 00000000000..b9445e5c26d --- /dev/null +++ b/charts/wire-ingress/templates/backendtrafficpolicy-websockets.yaml @@ -0,0 +1,26 @@ +{{- if .Values.websockets.enabled }} +{{/* Disables the stream idle timeout for WebSocket connections. + Envoy's default stream_idle_timeout is 5 minutes; once an HTTP connection + is upgraded to WebSocket, the timer fires if no frames flow in either + direction, closing the connection. Wire clients can be idle much longer + than that (infrequent push notifications), causing unnecessary reconnects. + Setting streamIdleTimeout to "0s" disables the idle timer entirely. + Envoy Gateway-specific (gateway.envoyproxy.io/v1alpha1). */}} +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: BackendTrafficPolicy +metadata: + name: {{ include "wire-ingress.fullname" . }}-nginz-websockets + namespace: {{ .Release.Namespace }} + labels: + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +spec: + targetRefs: + - group: gateway.networking.k8s.io + kind: HTTPRoute + name: {{ include "wire-ingress.fullname" . }}-nginz-websockets + timeout: + http: + streamIdleTimeout: "0s" +{{- end }} diff --git a/charts/wire-ingress/templates/certificate-federator.yaml b/charts/wire-ingress/templates/certificate-federator.yaml new file mode 100644 index 00000000000..0e5ef5219cc --- /dev/null +++ b/charts/wire-ingress/templates/certificate-federator.yaml @@ -0,0 +1,37 @@ +{{- if and .Values.federator.enabled .Values.federator.tls.useCertManager }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: "federator-{{ include "wire-ingress.zone" . | replace "." "-" }}-csr" + namespace: {{ .Release.Namespace }} + labels: + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +spec: + issuerRef: + {{- if .Values.federator.tls.issuer.name }} + name: {{ .Values.federator.tls.issuer.name | quote }} + kind: {{ .Values.federator.tls.issuer.kind | default .Values.tls.issuer.kind }} + {{- if .Values.federator.tls.issuer.group }} + group: {{ .Values.federator.tls.issuer.group }} + {{- end }} + {{- else }} + name: {{ include "wire-ingress.issuerName" . | quote }} + kind: {{ .Values.tls.issuer.kind }} + {{- end }} + usages: + - server auth + - client auth + duration: {{ .Values.federator.tls.duration }} + renewBefore: {{ .Values.federator.tls.renewBefore }} + isCA: false + secretName: {{ .Values.federator.tls.secretName | quote }} + privateKey: + algorithm: ECDSA + size: 256 + encoding: PKCS1 + rotationPolicy: {{ .Values.federator.tls.privateKey.rotationPolicy }} + dnsNames: + - {{ or .Values.config.dns.certificateDomain .Values.config.dns.federator | quote }} +{{- end }} diff --git a/charts/wire-ingress/templates/certificate.yaml b/charts/wire-ingress/templates/certificate.yaml new file mode 100644 index 00000000000..61ee9cb272b --- /dev/null +++ b/charts/wire-ingress/templates/certificate.yaml @@ -0,0 +1,45 @@ +{{- if .Values.tls.useCertManager -}} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: "{{ include "wire-ingress.zone" . | replace "." "-" }}-csr" + namespace: {{ .Release.Namespace }} + labels: + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +spec: + issuerRef: + name: {{ include "wire-ingress.issuerName" . | quote }} + kind: {{ .Values.tls.issuer.kind }} + usages: + - server auth + duration: 2160h # 90d, Letsencrypt default; NOTE: changes are ignored by Letsencrypt + renewBefore: 360h # 15d + isCA: false + secretName: {{ include "wire-ingress.certificateSecretName" . | quote }} + + privateKey: + algorithm: {{ .Values.tls.privateKey.algorithm }} + size: {{ .Values.tls.privateKey.size }} + encoding: PKCS1 + rotationPolicy: {{ .Values.tls.privateKey.rotationPolicy }} + + dnsNames: + - {{ .Values.config.dns.https }} + {{- if .Values.websockets.enabled }} + - {{ .Values.config.dns.ssl }} + {{- end }} + {{- if .Values.webapp.enabled }} + - {{ .Values.config.dns.webapp }} + {{- end }} + {{- if .Values.fakeS3.enabled }} + - {{ .Values.config.dns.fakeS3 }} + {{- end }} + {{- if .Values.teamSettings.enabled }} + - {{ .Values.config.dns.teamSettings }} + {{- end }} + {{- if .Values.accountPages.enabled }} + - {{ .Values.config.dns.accountPages }} + {{- end }} +{{- end -}} diff --git a/charts/wire-ingress/templates/clienttrafficpolicy-federator.yaml b/charts/wire-ingress/templates/clienttrafficpolicy-federator.yaml new file mode 100644 index 00000000000..f51a72058cb --- /dev/null +++ b/charts/wire-ingress/templates/clienttrafficpolicy-federator.yaml @@ -0,0 +1,28 @@ +{{- if .Values.federator.enabled }} +{{/* Envoy Gateway-specific (gateway.envoyproxy.io/v1alpha1). + Enforces mTLS client certificate validation on the federator listener only. */}} +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: ClientTrafficPolicy +metadata: + name: {{ include "wire-ingress.gatewayName" . }}-federator-mtls + namespace: {{ .Release.Namespace }} + labels: + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +spec: + targetRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: {{ include "wire-ingress.gatewayName" . | quote }} + sectionName: federator + tls: + {{/* Optional at the ingress level so the federator service can throw the + appropriate error when a cert is missing. Lua filter strips spoofed + X-SSL-Certificate header if no cert was actually provided. */}} + clientValidation: + optional: true + caCertificateRefs: + - name: federator-ca + kind: ConfigMap +{{- end }} diff --git a/charts/wire-ingress/templates/clienttrafficpolicy-proxy-protocol.yaml b/charts/wire-ingress/templates/clienttrafficpolicy-proxy-protocol.yaml new file mode 100644 index 00000000000..ed856304088 --- /dev/null +++ b/charts/wire-ingress/templates/clienttrafficpolicy-proxy-protocol.yaml @@ -0,0 +1,18 @@ +{{- if .Values.gateway.proxyProtocol.enabled }} +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: ClientTrafficPolicy +metadata: + name: {{ include "wire-ingress.gatewayName" . }}-proxy-protocol + namespace: {{ .Release.Namespace }} + labels: + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +spec: + targetRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: {{ include "wire-ingress.gatewayName" . | quote }} + proxyProtocol: + optional: {{ .Values.gateway.proxyProtocol.optional }} +{{- end }} diff --git a/charts/wire-ingress/templates/envoyextensionpolicy-federator.yaml b/charts/wire-ingress/templates/envoyextensionpolicy-federator.yaml new file mode 100644 index 00000000000..fa1db73025f --- /dev/null +++ b/charts/wire-ingress/templates/envoyextensionpolicy-federator.yaml @@ -0,0 +1,33 @@ +{{- if .Values.federator.enabled }} +{{/* Injects the mTLS client certificate as X-SSL-Certificate request header, + matching nginx's $ssl_client_escaped_cert behaviour. + Envoy Gateway-specific (gateway.envoyproxy.io/v1alpha1). */}} +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyExtensionPolicy +metadata: + name: {{ include "wire-ingress.fullname" . }}-federator-cert-header + namespace: {{ .Release.Namespace }} + labels: + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +spec: + targetRefs: + - group: gateway.networking.k8s.io + kind: HTTPRoute + name: {{ include "wire-ingress.fullname" . }}-federator + lua: + - type: Inline + inline: | + function envoy_on_request(request_handle) + {{/* Strip any client-provided header to prevent spoofing */}} + request_handle:headers():remove("X-SSL-Certificate") + local ssl = request_handle:connection():ssl() + if ssl ~= nil then + local cert = ssl:urlEncodedPemEncodedPeerCertificate() + if cert ~= nil and cert ~= "" then + request_handle:headers():add("X-SSL-Certificate", cert) + end + end + end +{{- end }} diff --git a/charts/wire-ingress/templates/envoypatchpolicy-federator.yaml b/charts/wire-ingress/templates/envoypatchpolicy-federator.yaml new file mode 100644 index 00000000000..5b52b9ed938 --- /dev/null +++ b/charts/wire-ingress/templates/envoypatchpolicy-federator.yaml @@ -0,0 +1,41 @@ +{{- if and .Values.federator.enabled .Values.gateway.patchPolicies.enabled }} +{{/* Adds the FQDN variant (trailing dot) of the federator hostname to the + virtual host's domain list so Envoy matches requests whose :authority + header carries a trailing dot. + Wire federator resolves federation targets via DNS SRV lookups; per + RFC 2782, SRV records return FQDNs (e.g. "federator.example.com."). + HTTP/2 passes that dot in :authority; without this patch the virtual + host only matches "federator.example.com" and returns route_not_found. + Envoy Gateway-specific (gateway.envoyproxy.io/v1alpha1). + Requires extensionApis.enableEnvoyPatchPolicy: true in the + EnvoyGateway ConfigMap. */}} +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyPatchPolicy +metadata: + name: {{ include "wire-ingress.fullname" . }}-federator-fqdn-domain + namespace: {{ .Release.Namespace }} + labels: + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +spec: + targetRef: + group: gateway.networking.k8s.io + {{- if .Values.gateway.patchPolicies.targetGatewayClass }} + kind: GatewayClass + name: {{ .Values.gateway.className | quote }} + {{- else }} + kind: Gateway + name: {{ include "wire-ingress.gatewayName" . | quote }} + namespace: {{ .Release.Namespace | quote }} + {{- end }} + type: JSONPatch + jsonPatches: + - type: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration" + # Route config name: //federator + name: {{ printf "%s/%s/federator" .Release.Namespace (include "wire-ingress.gatewayName" .) | quote }} + operation: + op: add + path: "/virtual_hosts/0/domains/-" + value: {{ printf "%s." .Values.config.dns.federator | quote }} +{{- end }} diff --git a/charts/wire-ingress/templates/envoyproxy.yaml b/charts/wire-ingress/templates/envoyproxy.yaml new file mode 100644 index 00000000000..12ff6f42b32 --- /dev/null +++ b/charts/wire-ingress/templates/envoyproxy.yaml @@ -0,0 +1,23 @@ +{{- if (and .Values.gateway.create .Values.gateway.envoyProxy.create) }} +{{- $name := .Values.gateway.envoyProxy.name | default (include "wire-ingress.gatewayName" .) }} +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyProxy +metadata: + name: {{ $name | quote }} + namespace: {{ .Release.Namespace }} + labels: + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +spec: + {{- with .Values.gateway.envoyProxy.spec }} + {{- toYaml . | nindent 2 }} + {{- end }} + {{- if .Values.gateway.manageServiceType }} + provider: + type: Kubernetes + kubernetes: + envoyService: + type: {{ .Values.gateway.serviceType | quote }} + {{- end }} +{{- end }} diff --git a/charts/wire-ingress/templates/gateway.yaml b/charts/wire-ingress/templates/gateway.yaml new file mode 100644 index 00000000000..16b3f7bb720 --- /dev/null +++ b/charts/wire-ingress/templates/gateway.yaml @@ -0,0 +1,64 @@ +{{- if .Values.gateway.create }} +{{- if not .Values.gateway.className }} +{{- fail "gateway.className must be set when gateway.create is true (set it to the name of your GatewayClass, e.g. 'envoy')" }} +{{- end }} +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: {{ include "wire-ingress.gatewayName" . | quote }} + namespace: {{ .Release.Namespace }} + labels: + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +spec: + gatewayClassName: {{ .Values.gateway.className | quote }} + {{- $envoyProxyName := "" }} + {{- if .Values.gateway.envoyProxy.create }} + {{- $envoyProxyName = .Values.gateway.envoyProxy.name | default (include "wire-ingress.gatewayName" .) }} + {{- else if .Values.gateway.envoyProxy.name }} + {{- $envoyProxyName = .Values.gateway.envoyProxy.name }} + {{- end }} + {{- if (or .Values.gateway.infrastructure.annotations $envoyProxyName) }} + infrastructure: + {{- if .Values.gateway.infrastructure.annotations }} + annotations: + {{- toYaml .Values.gateway.infrastructure.annotations | nindent 6 }} + {{- end }} + {{- if $envoyProxyName }} + parametersRef: + group: gateway.envoyproxy.io + kind: EnvoyProxy + name: {{ $envoyProxyName | quote }} + {{- end }} + {{- end }} + listeners: + - name: https + port: {{ .Values.gateway.listeners.https.port }} + protocol: HTTPS + hostname: {{ required "gateway.listeners.https.hostname is required (see values.yaml for details)" .Values.gateway.listeners.https.hostname | quote }} + tls: + mode: Terminate + certificateRefs: + - name: {{ include "wire-ingress.certificateSecretName" . | quote }} + kind: Secret + {{- if .Values.federator.enabled }} + - name: federator + port: {{ .Values.gateway.listeners.https.port }} + protocol: HTTPS + hostname: {{ required "config.dns.federator is required when federator.enabled is true" .Values.config.dns.federator | quote }} + tls: + mode: Terminate + certificateRefs: + - name: {{ required "federator.tls.secretName is required when federator.enabled is true" .Values.federator.tls.secretName | quote }} + kind: Secret + {{- end }} + {{- if .Values.gateway.listeners.http.enabled }} + - name: http + port: {{ .Values.gateway.listeners.http.port }} + protocol: HTTP + {{- if .Values.gateway.listeners.http.hostname }} + hostname: {{ .Values.gateway.listeners.http.hostname | quote }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/wire-ingress/templates/httproute-account-pages.yaml b/charts/wire-ingress/templates/httproute-account-pages.yaml new file mode 100644 index 00000000000..c3ef24a374b --- /dev/null +++ b/charts/wire-ingress/templates/httproute-account-pages.yaml @@ -0,0 +1,28 @@ +{{- if .Values.accountPages.enabled }} +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: {{ include "wire-ingress.fullname" . }}-account-pages + namespace: {{ .Release.Namespace }} + labels: + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +spec: + parentRefs: + - name: {{ include "wire-ingress.gatewayName" . | quote }} + namespace: {{ .Release.Namespace | quote }} + kind: Gateway + sectionName: https + hostnames: + - {{ required "config.dns.accountPages is required when accountPages.enabled is true" .Values.config.dns.accountPages | quote }} + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - name: account-pages-http + port: {{ .Values.service.accountPages.externalPort }} + kind: Service +{{- end }} diff --git a/charts/wire-ingress/templates/httproute-federator.yaml b/charts/wire-ingress/templates/httproute-federator.yaml new file mode 100644 index 00000000000..a747c27d8c2 --- /dev/null +++ b/charts/wire-ingress/templates/httproute-federator.yaml @@ -0,0 +1,28 @@ +{{- if .Values.federator.enabled }} +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: {{ include "wire-ingress.fullname" . }}-federator + namespace: {{ .Release.Namespace }} + labels: + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +spec: + parentRefs: + - name: {{ include "wire-ingress.gatewayName" . | quote }} + namespace: {{ .Release.Namespace | quote }} + kind: Gateway + sectionName: federator + hostnames: + - {{ .Values.config.dns.federator | quote }} + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - name: federator + port: {{ .Values.service.federator.externalPort }} + kind: Service +{{- end }} diff --git a/charts/wire-ingress/templates/httproute-nginz-websockets.yaml b/charts/wire-ingress/templates/httproute-nginz-websockets.yaml new file mode 100644 index 00000000000..5d9881d98d9 --- /dev/null +++ b/charts/wire-ingress/templates/httproute-nginz-websockets.yaml @@ -0,0 +1,28 @@ +{{- if .Values.websockets.enabled }} +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: {{ include "wire-ingress.fullname" . }}-nginz-websockets + namespace: {{ .Release.Namespace }} + labels: + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +spec: + parentRefs: + - name: {{ include "wire-ingress.gatewayName" . | quote }} + namespace: {{ .Release.Namespace | quote }} + kind: Gateway + sectionName: https + hostnames: + - {{ required "config.dns.ssl is required when websockets.enabled is true" .Values.config.dns.ssl | quote }} + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - name: nginz + port: {{ .Values.service.nginz.wsPort }} + kind: Service +{{- end }} diff --git a/charts/wire-ingress/templates/httproute-nginz.yaml b/charts/wire-ingress/templates/httproute-nginz.yaml new file mode 100644 index 00000000000..7bc50bb2890 --- /dev/null +++ b/charts/wire-ingress/templates/httproute-nginz.yaml @@ -0,0 +1,26 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: {{ include "wire-ingress.fullname" . }}-nginz + namespace: {{ .Release.Namespace }} + labels: + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +spec: + parentRefs: + - name: {{ include "wire-ingress.gatewayName" . | quote }} + namespace: {{ .Release.Namespace | quote }} + kind: Gateway + sectionName: https + hostnames: + - {{ required "config.dns.https is required" .Values.config.dns.https | quote }} + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - name: nginz + port: {{ .Values.service.nginz.httpPort }} + kind: Service diff --git a/charts/wire-ingress/templates/httproute-s3.yaml b/charts/wire-ingress/templates/httproute-s3.yaml new file mode 100644 index 00000000000..afb68412d74 --- /dev/null +++ b/charts/wire-ingress/templates/httproute-s3.yaml @@ -0,0 +1,30 @@ +{{- if .Values.fakeS3.enabled }} +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: {{ include "wire-ingress.fullname" . }}-minio + namespace: {{ .Release.Namespace }} + labels: + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +spec: + parentRefs: + - name: {{ include "wire-ingress.gatewayName" . | quote }} + namespace: {{ .Release.Namespace | quote }} + kind: Gateway + sectionName: https + hostnames: + - {{ required "config.dns.fakeS3 is required when fakeS3.enabled is true" .Values.config.dns.fakeS3 | quote }} + rules: + {{- toYaml .Values.fakeS3.guardingRules | nindent 4 }} + {{/* Default catch-all rule routes to the S3 backend */}} + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - name: {{ .Values.service.s3.serviceName }} + port: {{ .Values.service.s3.externalPort }} + kind: Service +{{- end }} diff --git a/charts/wire-ingress/templates/httproute-team-settings.yaml b/charts/wire-ingress/templates/httproute-team-settings.yaml new file mode 100644 index 00000000000..2de4cc0ea49 --- /dev/null +++ b/charts/wire-ingress/templates/httproute-team-settings.yaml @@ -0,0 +1,28 @@ +{{- if .Values.teamSettings.enabled }} +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: {{ include "wire-ingress.fullname" . }}-team-settings + namespace: {{ .Release.Namespace }} + labels: + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +spec: + parentRefs: + - name: {{ include "wire-ingress.gatewayName" . | quote }} + namespace: {{ .Release.Namespace | quote }} + kind: Gateway + sectionName: https + hostnames: + - {{ required "config.dns.teamSettings is required when teamSettings.enabled is true" .Values.config.dns.teamSettings | quote }} + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - name: team-settings-http + port: {{ .Values.service.teamSettings.externalPort }} + kind: Service +{{- end }} diff --git a/charts/wire-ingress/templates/httproute-webapp.yaml b/charts/wire-ingress/templates/httproute-webapp.yaml new file mode 100644 index 00000000000..158836040d1 --- /dev/null +++ b/charts/wire-ingress/templates/httproute-webapp.yaml @@ -0,0 +1,28 @@ +{{- if .Values.webapp.enabled }} +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: {{ include "wire-ingress.fullname" . }}-webapp + namespace: {{ .Release.Namespace }} + labels: + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +spec: + parentRefs: + - name: {{ include "wire-ingress.gatewayName" . | quote }} + namespace: {{ .Release.Namespace | quote }} + kind: Gateway + sectionName: https + hostnames: + - {{ required "config.dns.webapp is required when webapp.enabled is true" .Values.config.dns.webapp | quote }} + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - name: webapp-http + port: {{ .Values.service.webapp.externalPort }} + kind: Service +{{- end }} diff --git a/charts/wire-ingress/templates/issuer.yaml b/charts/wire-ingress/templates/issuer.yaml new file mode 100644 index 00000000000..6e6fb4820d6 --- /dev/null +++ b/charts/wire-ingress/templates/issuer.yaml @@ -0,0 +1,38 @@ +{{- if and .Values.tls.useCertManager .Values.tls.createIssuer -}} +apiVersion: cert-manager.io/v1 +{{- if or (eq .Values.tls.issuer.kind "Issuer") (eq .Values.tls.issuer.kind "ClusterIssuer") }} +kind: "{{ .Values.tls.issuer.kind }}" +{{- else }} +{{- fail (cat ".tls.issuer.kind can only be one of Issuer or ClusterIssuer, got: " .Values.tls.issuer.kind) }} +{{- end }} +metadata: + name: {{ include "wire-ingress.issuerName" . | quote }} + {{- if eq .Values.tls.issuer.kind "Issuer" }} + namespace: {{ .Release.Namespace }} + {{- end }} + labels: + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +spec: + acme: + server: {{ include "wire-ingress.certManagerAPIServerURL" . | quote }} + email: {{ required "Missing value: certmasterEmail" .Values.certManager.certmasterEmail | quote }} + # NOTE: this secret doesn't need to be created manually, cert-manager manages it + privateKeySecretRef: + name: {{ include "wire-ingress.issuerName" . -}}-account-key + solvers: +{{- if .Values.certManager.customSolvers }} +{{ toYaml .Values.certManager.customSolvers | indent 6 }} +{{- else }} +{{- if not .Values.gateway.listeners.http.enabled }} +{{- fail "The default HTTP01 solver requires gateway.listeners.http.enabled=true. Either enable the HTTP listener or supply certManager.customSolvers with a DNS01 solver." }} +{{- end }} + - http01: + gatewayHTTPRoute: + parentRefs: + - name: {{ include "wire-ingress.gatewayName" . | quote }} + namespace: {{ .Release.Namespace | quote }} + kind: Gateway +{{- end }} +{{- end -}} diff --git a/charts/wire-ingress/templates/secret.yaml b/charts/wire-ingress/templates/secret.yaml new file mode 100644 index 00000000000..52d1adbfe2b --- /dev/null +++ b/charts/wire-ingress/templates/secret.yaml @@ -0,0 +1,17 @@ +{{- if and (not .Values.tls.useCertManager) .Values.tls.secret.create .Values.secrets.tlsWildcardCert .Values.secrets.tlsWildcardKey }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "wire-ingress.certificateSecretName" . | quote }} + namespace: {{ .Release.Namespace }} + labels: + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +type: kubernetes.io/tls +data: + {{- with .Values.secrets }} + tls.crt: {{ .tlsWildcardCert | b64enc | quote }} + tls.key: {{ .tlsWildcardKey | b64enc | quote }} + {{- end -}} +{{- end -}} diff --git a/charts/wire-ingress/templates/service-account-pages.yaml b/charts/wire-ingress/templates/service-account-pages.yaml new file mode 100644 index 00000000000..7a77dbea0d6 --- /dev/null +++ b/charts/wire-ingress/templates/service-account-pages.yaml @@ -0,0 +1,18 @@ +{{- if .Values.accountPages.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: account-pages-http + namespace: {{ .Release.Namespace }} + labels: + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +spec: + type: ClusterIP + ports: + - port: {{ .Values.service.accountPages.externalPort }} + targetPort: 8080 + selector: + app: account-pages +{{- end }} diff --git a/charts/wire-ingress/templates/service-team-settings.yaml b/charts/wire-ingress/templates/service-team-settings.yaml new file mode 100644 index 00000000000..08b1fe6dfb1 --- /dev/null +++ b/charts/wire-ingress/templates/service-team-settings.yaml @@ -0,0 +1,18 @@ +{{- if .Values.teamSettings.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: team-settings-http + namespace: {{ .Release.Namespace }} + labels: + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +spec: + type: ClusterIP + ports: + - port: {{ .Values.service.teamSettings.externalPort }} + targetPort: 8080 + selector: + app: team-settings +{{- end }} diff --git a/charts/wire-ingress/templates/service-test-fed.yaml b/charts/wire-ingress/templates/service-test-fed.yaml new file mode 100644 index 00000000000..096a9b213ac --- /dev/null +++ b/charts/wire-ingress/templates/service-test-fed.yaml @@ -0,0 +1,38 @@ +{{- if (and .Values.federator.enabled .Values.federator.integrationTestHelper) }} +# This is used only for integration tests. +# Envoy Gateway runs proxy pods in {controllerNamespace}, not in the release namespace. +# Kubernetes Services cannot select pods across namespaces, so we create a ClusterIP +# Service in {controllerNamespace} that selects the proxy pods by their owning-gateway +# labels (set by Envoy Gateway). This service becomes the SRV target for Wire federation +# discovery: +# _wire-server-federator._tcp.-fed.{controllerNamespace}.svc.cluster.local +# +# targetPort must match the port the Envoy proxy pod actually listens on. +# When gateway.listeners.https.port is privileged (<1024), Envoy Gateway remaps it +# by adding 10000 on the container (e.g. 443→10443). Set gateway.listeners.https.port +# to the actual container port in your values (e.g. 10443) to avoid this. +{{- $httpsPort := int .Values.gateway.listeners.https.port }} +{{- if lt $httpsPort 1024 }} +{{- fail (printf "service-test-fed: gateway.listeners.https.port is %d (privileged, <1024). Envoy Gateway remaps it to %d on the proxy pod. Set gateway.listeners.https.port to the actual container port (e.g. %d)." $httpsPort (add $httpsPort 10000) (add $httpsPort 10000)) }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + # NOTE: keep this name in sync with with the "integrations" helm chart + name: {{ .Release.Namespace }}-fed + namespace: {{ .Values.gateway.controllerNamespace }} + labels: + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +spec: + type: ClusterIP + ports: + - name: wire-server-federator + port: 443 + protocol: TCP + targetPort: {{ $httpsPort }} + selector: + gateway.envoyproxy.io/owning-gateway-name: {{ include "wire-ingress.gatewayName" . }} + gateway.envoyproxy.io/owning-gateway-namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/charts/wire-ingress/templates/service-webapp.yaml b/charts/wire-ingress/templates/service-webapp.yaml new file mode 100644 index 00000000000..7e2d0d496a4 --- /dev/null +++ b/charts/wire-ingress/templates/service-webapp.yaml @@ -0,0 +1,18 @@ +{{- if .Values.webapp.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: webapp-http + namespace: {{ .Release.Namespace }} + labels: + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +spec: + type: ClusterIP + ports: + - port: {{ .Values.service.webapp.externalPort }} + targetPort: 8080 + selector: + app: webapp +{{- end }} diff --git a/charts/wire-ingress/values.yaml b/charts/wire-ingress/values.yaml new file mode 100644 index 00000000000..fb941ef7d5b --- /dev/null +++ b/charts/wire-ingress/values.yaml @@ -0,0 +1,223 @@ +# Default values for wire-ingress +gateway: + # If true, a Gateway resource is created by this chart. + # If false, set gateway.name to reference an existing Gateway. + create: true + # Name of the Gateway. Defaults to -wire-ingress-gateway if empty. + name: "" + # Name of the GatewayClass installed by the Envoy Gateway controller. + className: "" + envoyProxy: + # If true, an EnvoyProxy resource is created by this chart and the Gateway + # references it via infrastructure.parametersRef. + # Set to false when the GatewayClass already has a cluster-level EnvoyProxy + # (e.g. a shared load balancer managed by the cluster operator), or when you + # want to reference an existing EnvoyProxy via envoyProxy.name. + create: true + # Name of the EnvoyProxy resource. + # When create: true — the created resource uses this name (defaults to the Gateway name). + # When create: false — if non-empty, the Gateway references this existing EnvoyProxy + # via infrastructure.parametersRef (must be in the same namespace). + # If empty, no parametersRef is set on the Gateway (GatewayClass + # level EnvoyProxy takes effect). + name: "" + # Free-form EnvoyProxy spec, merged verbatim at the spec root. + # Only used when create: true. + # See https://gateway.envoyproxy.io/docs/api/extension_types/#envoyproxyspec + # Example - shared load balancer across Gateways (single wire-ingress chart): + # spec: + # mergeGateways: true + # Example - ClusterIP service (integration tests / no external LB): + # spec: + # provider: + # type: Kubernetes + # kubernetes: + # envoyService: + # type: ClusterIP + spec: {} + # Convenience shorthand for provider.kubernetes.envoyService.type. + # Only takes effect when manageServiceType: true. + # Do not combine with envoyProxy.spec.provider.kubernetes.envoyService. + # Namespace where the Envoy Gateway controller runs its proxy pods. + # Change only if you installed Envoy Gateway into a non-default namespace. + controllerNamespace: envoy-gateway-system + manageServiceType: true + serviceType: LoadBalancer + # Annotations to add to the Service created by Envoy Gateway. + # Requires Gateway API CRDs v1.1+. + # Example for AWS NLB: + # infrastructure: + # annotations: + # service.beta.kubernetes.io/aws-load-balancer-type: external + # service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip + infrastructure: + annotations: {} + # Enable if your load balancer sends PROXY protocol headers (e.g. Hetzner with + # load-balancer.hetzner.cloud/uses-proxyprotocol: "true"). Creates a + # ClientTrafficPolicy targeting the Gateway. + proxyProtocol: + enabled: false + # If true, connections without a PROXY protocol header are also accepted. + # Use this only if some traffic bypasses the load balancer (e.g. direct node access). + optional: false + listeners: + https: + port: 443 + # Required. Hostname restriction for the HTTPS listener. + # Must be a wildcard or exact hostname covering this deployment's domains, + # e.g. "*.example.com". This prevents OverlappingTLSConfig with the federator + # listener (which uses the federation domain on the same port) — without an + # explicit hostname both listeners are catch-all and Envoy degrades ALPN to + # HTTP/1.1-only. + # If your domains in config.dns span multiple subdomains, you need a custom + # Gateway object with multiple listeners. See the Gateway API docs. + hostname: "" + # Enable the HTTP listener if you want to use HTTP01 challenges via cert-manager's + # gatewayHTTPRoute solver. Requires cert-manager >= 1.14. + # See the README for configuration details. + http: + enabled: false + port: 80 + # Optional hostname restriction for the HTTP listener. Set alongside + # listeners.https.hostname when using mergeGateways. + hostname: "" + # Set to false to skip creating EnvoyPatchPolicy resources. + # EnvoyPatchPolicy requires extensionApis.enableEnvoyPatchPolicy: true + # in the EnvoyGateway ConfigMap (see README). + patchPolicies: + enabled: true + # Set to true when the GatewayClass uses mergeGateways: true. + # With mergeGateways, EnvoyPatchPolicy must target the GatewayClass + # rather than the individual Gateway. Default false targets the Gateway, + # which is correct for single-Gateway (non-merged) deployments. + targetGatewayClass: false + +# NOTE: Please provide names. Here are naming suggestions: +# You need to reference those in the nginz helm chart nginx_conf.deeplink.endpoints +# config: +# dns: +# https: nginz-https. +# ssl: nginz-ssl. # ignored if websockets.enabled == false +# webapp: webapp. # ignored if webapp.enabled == false +# fakeS3: assets. # ignored if fakeS3.enabled == false +# federator: federator. # ignored unless federator.enabled == true +# certificateDomain: federator. # domain to use in the federator CSR +# teamSettings: teams. # ignored unless teamSettings.enabled == true +# accountPages: account. # ignored unless accountPages.enabled == true + +websockets: + enabled: true +webapp: + enabled: true +fakeS3: + enabled: true + # Guard rules applied BEFORE the default catch-all rule. + # These handle provider-specific paths. Adjust if using a different S3 provider. + # Uses Gateway API HTTPRoute rule format (matches/filters/backendRefs). + guardingRules: + - matches: + - path: + type: PathPrefix + value: /minio/ + filters: + - type: RequestRedirect + requestRedirect: + path: + type: ReplaceFullPath + replaceFullPath: / + statusCode: 301 +teamSettings: + enabled: false +accountPages: + enabled: false + +service: + webapp: + externalPort: 8080 + teamSettings: + externalPort: 8080 + accountPages: + externalPort: 8080 + s3: + externalPort: 9000 + serviceName: fake-aws-s3 + # Set this to false if minio is not provided by the fake-aws-s3 chart + useFakeS3: true + federator: + # Must match service.externalFederatorPort in the federator chart. + externalPort: 8081 + nginz: + # Update this if you've overwritten the default HTTP port in nginz. + httpPort: 8080 + # Update this if you've overwritten the default WebSocket port in nginz. + wsPort: 8081 + +# Federator + +federator: + enabled: false + integrationTestHelper: false + tls: + # If true, a cert-manager Certificate is created for the federator TLS secret. + # Independent of the global tls.useCertManager setting. + useCertManager: true + # Name of the TLS Secret for the federator listener. + # When useCertManager is true, cert-manager writes the certificate into this secret. + # When useCertManager is false, the secret must exist before deploying. + secretName: "federator-certificate-secret" + duration: 2160h + renewBefore: 360h + privateKey: + rotationPolicy: Always + # Issuer for federator certificate (mTLS with Client Auth EKU). + # If not set, uses global tls.issuer configuration. + issuer: {} + # name: "" + # kind: "" + # group: "" + +# TLS settings + +tls: + secret: + # If true, a kubernetes.io/tls Secret is created from secrets.tlsWildcardCert + # and secrets.tlsWildcardKey. Set to false if the secret is managed externally. + create: true + # Override the name of the Secret. If not set, the name is derived from the release name. + nameOverride: "" + + # If set to true create Certificate object that request from tls.issuer + useCertManager: false + privateKey: + rotationPolicy: Always + algorithm: ECDSA + size: 384 # 521 is not supported by Let's Encrypt + + createIssuer: true + issuer: + name: letsencrypt-http01 + kind: Issuer # Issuer | ClusterIssuer + +certManager: + # If true, uses the Let's Encrypt staging API (certificates are NOT trusted): + # https://acme-staging-v02.api.letsencrypt.org/directory + # If false (default), uses the production API (certificates are trusted): + # https://acme-v02.api.letsencrypt.org/directory + inTestMode: false + certmasterEmail: + customSolvers: + +# For TLS (manual mode, tls.useCertManager: false, tls.secret.create: false): +# secrets: +# tlsWildcardCert: | +# -----BEGIN CERTIFICATE----- +# ... +# -----END CERTIFICATE----- +# tlsWildcardKey: | +# -----BEGIN PRIVATE KEY----- +# ... +# -----END PRIVATE KEY----- +# +# When federator.enabled is true, a ConfigMap named `federator-ca` with a `ca.crt` key +# must exist in the release namespace. It is created by the wire-server chart +# and is referenced by the ClientTrafficPolicy for mTLS. diff --git a/charts/wire-server/templates/brig/tests/brig-integration.yaml b/charts/wire-server/templates/brig/tests/brig-integration.yaml index c2c9372217b..ce8eeadc0fd 100644 --- a/charts/wire-server/templates/brig/tests/brig-integration.yaml +++ b/charts/wire-server/templates/brig/tests/brig-integration.yaml @@ -3,7 +3,7 @@ kind: Service metadata: name: "brig-integration" annotations: - "helm.sh/hook": post-install + "helm.sh/hook": post-install,post-upgrade "helm.sh/hook-delete-policy": before-hook-creation labels: app: brig-integration diff --git a/charts/wire-server/templates/brig/tests/configmap.yaml b/charts/wire-server/templates/brig/tests/configmap.yaml index e7540dc7c59..b327f32dbde 100644 --- a/charts/wire-server/templates/brig/tests/configmap.yaml +++ b/charts/wire-server/templates/brig/tests/configmap.yaml @@ -3,7 +3,7 @@ kind: ConfigMap metadata: name: "brig-integration" annotations: - "helm.sh/hook": post-install + "helm.sh/hook": post-install,post-upgrade "helm.sh/hook-delete-policy": before-hook-creation data: integration.yaml: | diff --git a/charts/wire-server/templates/brig/tests/nginz-service.yaml b/charts/wire-server/templates/brig/tests/nginz-service.yaml index 6eda016c82d..b7895d51415 100644 --- a/charts/wire-server/templates/brig/tests/nginz-service.yaml +++ b/charts/wire-server/templates/brig/tests/nginz-service.yaml @@ -6,7 +6,7 @@ kind: Service metadata: name: nginz-integration-http annotations: - "helm.sh/hook": post-install + "helm.sh/hook": post-install,post-upgrade "helm.sh/hook-delete-policy": before-hook-creation spec: type: ClusterIP diff --git a/charts/wire-server/templates/brig/tests/secret.yaml b/charts/wire-server/templates/brig/tests/secret.yaml index 86177dd7d3e..5bda33f6be2 100644 --- a/charts/wire-server/templates/brig/tests/secret.yaml +++ b/charts/wire-server/templates/brig/tests/secret.yaml @@ -8,7 +8,7 @@ metadata: release: "{{ .Release.Name }}" heritage: "{{ .Release.Service }}" annotations: - "helm.sh/hook": post-install + "helm.sh/hook": post-install,post-upgrade "helm.sh/hook-delete-policy": before-hook-creation type: Opaque data: diff --git a/charts/wire-server/templates/cargohold/tests/configmap.yaml b/charts/wire-server/templates/cargohold/tests/configmap.yaml index 1542a14aa06..71a5db1bc33 100644 --- a/charts/wire-server/templates/cargohold/tests/configmap.yaml +++ b/charts/wire-server/templates/cargohold/tests/configmap.yaml @@ -3,7 +3,7 @@ kind: ConfigMap metadata: name: "cargohold-integration" annotations: - "helm.sh/hook": post-install + "helm.sh/hook": post-install,post-upgrade "helm.sh/hook-delete-policy": before-hook-creation data: integration.yaml: | diff --git a/charts/wire-server/templates/galley/tests/configmap.yaml b/charts/wire-server/templates/galley/tests/configmap.yaml index 86be4122364..deb81aaeabb 100644 --- a/charts/wire-server/templates/galley/tests/configmap.yaml +++ b/charts/wire-server/templates/galley/tests/configmap.yaml @@ -3,7 +3,7 @@ kind: ConfigMap metadata: name: "galley-integration" annotations: - "helm.sh/hook": post-install + "helm.sh/hook": post-install,post-upgrade "helm.sh/hook-delete-policy": before-hook-creation data: integration.yaml: | diff --git a/charts/wire-server/templates/galley/tests/galley-integration.yaml b/charts/wire-server/templates/galley/tests/galley-integration.yaml index 9256af1db6a..6135bff3878 100644 --- a/charts/wire-server/templates/galley/tests/galley-integration.yaml +++ b/charts/wire-server/templates/galley/tests/galley-integration.yaml @@ -3,7 +3,7 @@ kind: Service metadata: name: "galley-integration" annotations: - "helm.sh/hook": post-install + "helm.sh/hook": post-install,post-upgrade "helm.sh/hook-delete-policy": before-hook-creation labels: app: galley-integration diff --git a/charts/wire-server/templates/galley/tests/secret.yaml b/charts/wire-server/templates/galley/tests/secret.yaml index b204dd4eddc..cf8e2e04d0c 100644 --- a/charts/wire-server/templates/galley/tests/secret.yaml +++ b/charts/wire-server/templates/galley/tests/secret.yaml @@ -8,7 +8,7 @@ metadata: release: "{{ .Release.Name }}" heritage: "{{ .Release.Service }}" annotations: - "helm.sh/hook": post-install + "helm.sh/hook": post-install,post-upgrade "helm.sh/hook-delete-policy": before-hook-creation type: Opaque data: diff --git a/charts/wire-server/templates/gundeck/tests/configmap.yaml b/charts/wire-server/templates/gundeck/tests/configmap.yaml index acba48d7f16..c8c23ce5185 100644 --- a/charts/wire-server/templates/gundeck/tests/configmap.yaml +++ b/charts/wire-server/templates/gundeck/tests/configmap.yaml @@ -3,7 +3,7 @@ kind: ConfigMap metadata: name: "gundeck-integration" annotations: - "helm.sh/hook": post-install + "helm.sh/hook": post-install,post-upgrade "helm.sh/hook-delete-policy": before-hook-creation data: integration.yaml: | diff --git a/charts/wire-server/templates/spar/tests/configmap.yaml b/charts/wire-server/templates/spar/tests/configmap.yaml index 2eb1966099a..9cc5c79b796 100644 --- a/charts/wire-server/templates/spar/tests/configmap.yaml +++ b/charts/wire-server/templates/spar/tests/configmap.yaml @@ -3,7 +3,7 @@ kind: ConfigMap metadata: name: "spar-integration" annotations: - "helm.sh/hook": post-install + "helm.sh/hook": post-install,post-upgrade "helm.sh/hook-delete-policy": before-hook-creation data: integration.yaml: | diff --git a/hack/bin/integration-setup-federation.sh b/hack/bin/integration-setup-federation.sh index 355abb417ab..75e2bc5ce4e 100755 --- a/hack/bin/integration-setup-federation.sh +++ b/hack/bin/integration-setup-federation.sh @@ -7,9 +7,26 @@ TOP_LEVEL="$DIR/../.." export NAMESPACE=${NAMESPACE:-test-integration} # Available $HELMFILE_ENV profiles: default, default-ssl, kind, kind-ssl HELMFILE_ENV=${HELMFILE_ENV:-default} +# This controls if integration tests run against ingress-nginx or envoy-gateway +WIRE_INGRESS_MODE=${WIRE_INGRESS_MODE:-envoy} +export WIRE_INGRESS_MODE +ENVOY_GATEWAY_NAMESPACE=${ENVOY_GATEWAY_NAMESPACE:-envoy-gateway-system} +export ENVOY_GATEWAY_NAMESPACE CHARTS_DIR="${TOP_LEVEL}/.local/charts" HELM_PARALLELISM=${HELM_PARALLELISM:-1} +changed_files=$(git --no-pager diff-tree --no-commit-id -r --name-only HEAD) + +if [[ "$WIRE_INGRESS_MODE" != "nginx" ]] && echo "$changed_files" | grep -q "^charts/nginx-ingress-services"; then + echo "ERROR: Changes detected in charts/nginx-ingress-services but WIRE_INGRESS_MODE is '${WIRE_INGRESS_MODE}'." + echo "This failure is intentional: changes to nginx-ingress-services are not exercised by the" + echo "integration test suite when running in envoy mode, and would be merged without any test coverage." + echo "To test these changes, change to WIRE_INGRESS_MODE=nginx. and merge the changes" + echo "Then in a follow-up PR change WIRE_INGRESS_MODE=envoy and re-implement the changes also in charts/wire-ingress" + echo "FUTUREWORK: Remove WIRE_INGRESS_MODE once ingress-nginx is no longer supported by wire-server" + exit 1 +fi + # shellcheck disable=SC1091 . "$DIR/helm_overrides.sh" "${DIR}"/integration-cleanup.sh @@ -22,17 +39,24 @@ HELM_PARALLELISM=${HELM_PARALLELISM:-1} # script beforehand on all relevant charts to download the nested dependencies # (e.g. cassandra from underneath databases-ephemeral) echo "updating recursive dependencies ..." -charts=(fake-aws databases-ephemeral rabbitmq wire-server ingress-nginx-controller nginx-ingress-services) +charts=(fake-aws databases-ephemeral rabbitmq wire-server ingress-nginx-controller nginx-ingress-services wire-ingress) mkdir -p ~/.parallel && touch ~/.parallel/will-cite printf '%s\n' "${charts[@]}" | parallel -P "${HELM_PARALLELISM}" "$DIR/update.sh" "$CHARTS_DIR/{}" export NAMESPACE_1="$NAMESPACE" -export FEDERATION_DOMAIN_BASE_1="$NAMESPACE_1.svc.cluster.local" -export FEDERATION_DOMAIN_1="federation-test-helper.$FEDERATION_DOMAIN_BASE_1" - export NAMESPACE_2="$NAMESPACE-fed2" -export FEDERATION_DOMAIN_BASE_2="$NAMESPACE_2.svc.cluster.local" -export FEDERATION_DOMAIN_2="federation-test-helper.$FEDERATION_DOMAIN_BASE_2" + +if [[ "$WIRE_INGRESS_MODE" == "nginx" ]]; then + export FEDERATION_DOMAIN_BASE_1="${NAMESPACE_1}.svc.cluster.local" + export FEDERATION_DOMAIN_1="federation-test-helper.${FEDERATION_DOMAIN_BASE_1}" + export FEDERATION_DOMAIN_BASE_2="${NAMESPACE_2}.svc.cluster.local" + export FEDERATION_DOMAIN_2="federation-test-helper.${FEDERATION_DOMAIN_BASE_2}" +else + export FEDERATION_DOMAIN_BASE_1="${ENVOY_GATEWAY_NAMESPACE}.svc.cluster.local" + export FEDERATION_DOMAIN_1="${NAMESPACE_1}-fed.${FEDERATION_DOMAIN_BASE_1}" + export FEDERATION_DOMAIN_BASE_2="${ENVOY_GATEWAY_NAMESPACE}.svc.cluster.local" + export FEDERATION_DOMAIN_2="${NAMESPACE_2}-fed.${FEDERATION_DOMAIN_BASE_2}" +fi echo "Fetch federation-ca secret from cert-manager namespace" FEDERATION_CA_CERTIFICATE=$(kubectl -n cert-manager get secrets federation-ca -o json -o jsonpath="{.data['tls\.crt']}" | base64 -d) diff --git a/hack/helm_vars/wire-ingress/values.yaml.gotmpl b/hack/helm_vars/wire-ingress/values.yaml.gotmpl new file mode 100644 index 00000000000..b4cf20a464b --- /dev/null +++ b/hack/helm_vars/wire-ingress/values.yaml.gotmpl @@ -0,0 +1,42 @@ +teamSettings: + enabled: true +accountPages: + enabled: true +federator: + enabled: true + integrationTestHelper: true + tls: + useCertManager: true + issuer: + name: federation + kind: ClusterIssuer +tls: + useCertManager: true + issuer: + name: federation + kind: ClusterIssuer + createIssuer: false + +gateway: + className: "envoy" + controllerNamespace: {{ env "ENVOY_GATEWAY_NAMESPACE" }} + manageServiceType: true + serviceType: ClusterIP + listeners: + https: + # Envoy Gateway remaps privileged ports (+10000) on the proxy pod container. + # We use 10443 directly so targetPort in the service-test-fed.yaml matches the + # actual container port without needing a separate override. + port: 10443 + hostname: "*.{{ .Release.Namespace }}-integration.example.com" + +config: + dns: + https: "nginz-https.{{ .Release.Namespace }}-integration.example.com" + ssl: "nginz-ssl.{{ .Release.Namespace }}-integration.example.com" + webapp: "webapp.{{ .Release.Namespace }}-integration.example.com" + fakeS3: "assets.{{ .Release.Namespace }}-integration.example.com" + teamSettings: "teams.{{ .Release.Namespace }}-integration.example.com" + accountPages: "account.{{ .Release.Namespace }}-integration.example.com" + # federator: dynamically set by hack/helmfile.yaml.gotmpl + # certificateDomain: dynamically set by hack/helmfile.yaml.gotmpl diff --git a/hack/helm_vars/wire-server/values.yaml.gotmpl b/hack/helm_vars/wire-server/values.yaml.gotmpl index 8ed568c72ca..116b9315c68 100644 --- a/hack/helm_vars/wire-server/values.yaml.gotmpl +++ b/hack/helm_vars/wire-server/values.yaml.gotmpl @@ -615,8 +615,9 @@ federator: optSettings: useSystemCAStore: false logLevel: Debug - {{- if .Values.uploadXml }} tests: + nginxIngressHost: {{ .Values.federationDomain1 }} + {{- if .Values.uploadXml }} config: uploadXml: baseUrl: {{ .Values.uploadXml.baseUrl }} @@ -676,8 +677,15 @@ background-worker: password: {{ .Values.rabbitmqPassword }} integration: + envoy: + enabled: {{ ne (env "WIRE_INGRESS_MODE") "nginx" }} + controllerNamespace: {{ env "ENVOY_GATEWAY_NAMESPACE" }} + gateway: + className: "envoy" + {{- if eq (env "WIRE_INGRESS_MODE") "nginx" }} ingress: class: "nginx-{{ .Release.Namespace }}" + {{- end }} config: cassandra: host: {{ .Values.cassandraHost }} diff --git a/hack/helmfile-federation-v0.yaml.gotmpl b/hack/helmfile-federation-v0.yaml.gotmpl deleted file mode 100644 index 5400307d84b..00000000000 --- a/hack/helmfile-federation-v0.yaml.gotmpl +++ /dev/null @@ -1,110 +0,0 @@ ---- -helmDefaults: - wait: true - timeout: 600 - devel: true - createNamespace: true - -environments: - default: - values: - - federationCACertificate: {{ readFile "../services/nginz/integration-test/conf/nginz/integration-ca.pem" | quote }} - - rabbitmqUsername: guest - - rabbitmqPassword: guest ---- -repositories: - - name: jetstack - url: 'https://charts.jetstack.io' - - - name: bedag - url: 'https://bedag.github.io/helm-charts/' - - - name: wire - url: 'https://s3-eu-west-1.amazonaws.com/public.wire.com/charts-develop' - -releases: - - name: 'cert-manager' - namespace: cert-manager - chart: jetstack/cert-manager - set: - - name: installCRDs - value: true - - - name: 'federation-certs' - namespace: cert-manager - chart: bedag/raw - values: - - resources: - - apiVersion: v1 - kind: Secret - metadata: - name: federation-ca - namespace: cert-manager - data: - tls.crt: {{ readFile "../services/nginz/integration-test/conf/nginz/integration-ca.pem" | b64enc | quote }} - tls.key: {{ readFile "../services/nginz/integration-test/conf/nginz/integration-ca-key.pem" | b64enc | quote }} - - apiVersion: cert-manager.io/v1 - kind: ClusterIssuer - metadata: - name: federation - spec: - ca: - secretName: federation-ca - needs: - - 'cert-manager/cert-manager' - - - name: 'fake-aws' - namespace: wire-federation-v0 - chart: wire/fake-aws - version: 4.38.0-mandarin.14 - values: - - './helm_vars/fake-aws/values.yaml' - - - name: 'databases-ephemeral' - namespace: wire-federation-v0 - chart: 'wire/databases-ephemeral' - version: 4.38.0-mandarin.14 - - - name: 'rabbitmq' - namespace: wire-federation-v0 - chart: 'wire/rabbitmq' - version: 4.38.0-mandarin.14 - values: - - './helm_vars/rabbitmq/values.yaml.gotmpl' - - - name: 'ingress' - namespace: wire-federation-v0 - chart: 'wire/ingress-nginx-controller' - version: 4.38.0-mandarin.14 - values: - - './helm_vars/ingress-nginx-controller/values.yaml.gotmpl' - - - name: 'ingress-svc' - namespace: wire-federation-v0 - chart: 'wire/nginx-ingress-services' - version: 4.38.0-mandarin.14 - values: - - './helm_vars/nginx-ingress-services/values.yaml.gotmpl' - set: - # Federation domain is also the SRV record created by the - # federation-test-helper service. Maybe we can find a way to make these - # differ, so we don't make any silly assumptions in the code. - - name: config.dns.federator - value: wire-federation-v0.svc.cluster.local - - name: config.dns.certificateDomain - value: '*.wire-federation-v0.svc.cluster.local' - needs: - - 'ingress' - - 'cert-manager/cert-manager' - - 'cert-manager/federation-certs' - - - name: wire-server - namespace: wire-federation-v0 - chart: wire/wire-server - version: 4.38.0-mandarin.14 - values: - - './helm_vars/wire-federation-v0/values.yaml.gotmpl' - needs: - - 'cert-manager/cert-manager' - - 'cert-manager/federation-certs' - diff --git a/hack/helmfile.yaml.gotmpl b/hack/helmfile.yaml.gotmpl index fbd83410307..bb1bedad9bc 100644 --- a/hack/helmfile.yaml.gotmpl +++ b/hack/helmfile.yaml.gotmpl @@ -276,37 +276,57 @@ releases: values: - './helm_vars/{{ .Values.ingressChart }}/values.yaml.gotmpl' + {{- if eq (env "WIRE_INGRESS_MODE") "nginx" }} - name: 'ingress-svc' namespace: '{{ .Values.namespace1 }}' chart: '../.local/charts/nginx-ingress-services' values: - './helm_vars/nginx-ingress-services/values.yaml.gotmpl' set: - # Federation domain is also the SRV record created by the - # federation-test-helper service. Maybe we can find a way to make these - # differ, so we don't make any silly assumptions in the code. - name: config.dns.federator value: '{{ .Values.federationDomain1 }}' - name: config.dns.certificateDomain value: '*.{{ .Values.federationDomainBase1 }}' needs: - - 'ingress' + - ingress + {{- else }} + - name: 'ingress-svc' + namespace: '{{ .Values.namespace1 }}' + chart: '../.local/charts/wire-ingress' + values: + - './helm_vars/wire-ingress/values.yaml.gotmpl' + set: + - name: config.dns.federator + value: '{{ .Values.federationDomain1 }}' + - name: config.dns.certificateDomain + value: '*.{{ .Values.federationDomainBase1 }}' + {{- end }} + {{- if eq (env "WIRE_INGRESS_MODE") "nginx" }} - name: 'ingress-svc' namespace: '{{ .Values.namespace2 }}' chart: '../.local/charts/nginx-ingress-services' values: - './helm_vars/nginx-ingress-services/values.yaml.gotmpl' set: - # Federation domain is also the SRV record created by the - # federation-test-helper service. Maybe we can find a way to make these - # differ, so we don't make any silly assumptions in the code. - name: config.dns.federator value: '{{ .Values.federationDomain2 }}' - name: config.dns.certificateDomain value: '*.{{ .Values.federationDomainBase2 }}' needs: - - 'ingress' + - ingress + {{- else }} + - name: 'ingress-svc' + namespace: '{{ .Values.namespace2 }}' + chart: '../.local/charts/wire-ingress' + values: + - './helm_vars/wire-ingress/values.yaml.gotmpl' + set: + - name: config.dns.federator + value: '{{ .Values.federationDomain2 }}' + - name: config.dns.certificateDomain + value: '*.{{ .Values.federationDomainBase2 }}' + {{- end }} # Note that wire-server depends on databases-ephemeral being up; and in some # cases on nginx-ingress also being up. If installing helm charts in a diff --git a/integration/test/Testlib/ModService.hs b/integration/test/Testlib/ModService.hs index 5dddcb56c4f..b3a42bd2671 100644 --- a/integration/test/Testlib/ModService.hs +++ b/integration/test/Testlib/ModService.hs @@ -646,12 +646,12 @@ prepareNginzRuntimeFiles resource = do sm <- getServiceMap domain mBaseDir <- asks (.servicesCwdBase) case mBaseDir of - Nothing -> liftIO $ prepareNginzK8sRuntimeFiles domain sm + Nothing -> liftIO $ prepareNginzK8sRuntimeFiles sm Just basedir -> liftIO $ prepareNginzLocalRuntimeFiles resource sm basedir -prepareNginzK8sRuntimeFiles :: String -> ServiceMap -> IO (FilePath, FilePath, FilePath) -prepareNginzK8sRuntimeFiles domain sm = do - tmpDir <- createTempDirectory "/tmp" ("nginz" <> "-" <> domain) +prepareNginzK8sRuntimeFiles :: ServiceMap -> IO (FilePath, FilePath, FilePath) +prepareNginzK8sRuntimeFiles sm = do + tmpDir <- createTempDirectory "/tmp" "nginz-" copyDirectoryRecursively "/etc/wire/nginz/" tmpDir let nginxConfFile = tmpDir "conf" "nginx.conf" @@ -670,12 +670,10 @@ prepareNginzK8sRuntimeFiles domain sm = do prepareNginzLocalRuntimeFiles :: BackendResource -> ServiceMap -> FilePath -> IO (FilePath, FilePath, FilePath) prepareNginzLocalRuntimeFiles resource sm basedir = do - let domain = berDomain resource - -- Create a whole temporary directory and copy all nginx's config files. -- This is necessary because nginx assumes local imports are relative to -- the location of the main configuration file. - tmpDir <- createTempDirectory "/tmp" ("nginz" <> "-" <> domain) + tmpDir <- createTempDirectory "/tmp" "nginz-" -- copy all config files into the tmp dir let from = basedir "nginz" "integration-test" From 4326bf08f96fa46c21bebb4e3be66b944126efc5 Mon Sep 17 00:00:00 2001 From: Matthias Fischmann Date: Mon, 11 May 2026 17:24:10 +0200 Subject: [PATCH 3/5] Improve swagger API docs. (#5220) --- .../4-docs/WPB-24978-improve-swagger-api-docs | 1 + .../src/Wire/API/Routes/Public/Brig.hs | 22 +++++++++++++++++-- .../Wire/API/Routes/Public/Brig/Services.hs | 1 + .../Routes/Public/Galley/TeamNotification.hs | 22 ++++++++++--------- .../src/Wire/API/Routes/Public/Gundeck.hs | 1 + 5 files changed, 35 insertions(+), 12 deletions(-) create mode 100644 changelog.d/4-docs/WPB-24978-improve-swagger-api-docs diff --git a/changelog.d/4-docs/WPB-24978-improve-swagger-api-docs b/changelog.d/4-docs/WPB-24978-improve-swagger-api-docs new file mode 100644 index 00000000000..98ec58d6a35 --- /dev/null +++ b/changelog.d/4-docs/WPB-24978-improve-swagger-api-docs @@ -0,0 +1 @@ +Improve swagger API docs. diff --git a/libs/wire-api/src/Wire/API/Routes/Public/Brig.hs b/libs/wire-api/src/Wire/API/Routes/Public/Brig.hs index 2dbe0c784dc..c4c3ebd7815 100644 --- a/libs/wire-api/src/Wire/API/Routes/Public/Brig.hs +++ b/libs/wire-api/src/Wire/API/Routes/Public/Brig.hs @@ -1242,6 +1242,7 @@ type ClientAPI = :<|> Named "get-user-client-qualified" ( Summary "Get a specific client of a user" + :> Description "Hint: to list all clients of one or more users, use POST /users/list-clients." :> "users" :> QualifiedCaptureUserId "uid" :> "clients" @@ -1445,12 +1446,29 @@ type ConnectionAPI = :<|> Named "search-contacts" ( Summary "Search for users" - :> Description "Optional user-type filter semantics: omitted or empty (type=) means no filtering." :> ZLocalUser :> CanThrow 'InsufficientPermissions :> "search" :> "contacts" - :> QueryParam' '[Required, Strict, Description "Search query"] "q" Text + :> QueryParam' + '[ Required, + Strict, + Description + "Search query\ + \

The search query is normalized: lower-cased and diacritics are removed ('Björn' becomes 'bjorn').\ + \ The normalized search query matches accounts that, in this order of priority:

\ + \
    \ + \
  • are equal to the normalized full handle;\ + \
  • are equal to the normalized full user display name;\ + \
  • prefix-match the normalized handle;\ + \
  • prefix-match the normalized user display name.\ + \
\ + \

NB: '@' Does NOT do anything special, ignoring user display names.

\ + \

See also: [authoritative ElasticSearch query](https://github.com/wireapp/wire-server/blob/83c25cca6a5e9d2205c102410b452eb78fc50a00/libs/wire-subsystems/src/Wire/IndexedUserStore/ElasticSearch.hs#L251-L288)

\ + \" + ] + "q" + Text :> QueryParam' '[Optional, Strict, Description "Searched domain. Note: This is optional only for backwards compatibility, future versions will mandate this."] "domain" Domain :> QueryParam' '[Optional, Strict, Description "Number of results to return (min: 1, max: 500, default 15)"] "size" (Range 1 500 Int32) :> QueryParam' '[Optional, Strict, Description "Only user types. Omitted or empty (type=) means no filtering."] "type" (CommaSeparatedList UserTypeFilter) diff --git a/libs/wire-api/src/Wire/API/Routes/Public/Brig/Services.hs b/libs/wire-api/src/Wire/API/Routes/Public/Brig/Services.hs index df62901e3ee..1fd66aba987 100644 --- a/libs/wire-api/src/Wire/API/Routes/Public/Brig/Services.hs +++ b/libs/wire-api/src/Wire/API/Routes/Public/Brig/Services.hs @@ -163,6 +163,7 @@ type ServicesAPI = :<|> Named "post-team-whitelist-by-team-id" ( Summary "Update service whitelist" + :> CanThrow 'MLSServicesNotAllowed :> ZUser :> ZConn :> "teams" diff --git a/libs/wire-api/src/Wire/API/Routes/Public/Galley/TeamNotification.hs b/libs/wire-api/src/Wire/API/Routes/Public/Galley/TeamNotification.hs index 7d3a6b58bec..e783a8fba9a 100644 --- a/libs/wire-api/src/Wire/API/Routes/Public/Galley/TeamNotification.hs +++ b/libs/wire-api/src/Wire/API/Routes/Public/Galley/TeamNotification.hs @@ -54,24 +54,26 @@ type TeamNotificationAPI = ) type GetTeamNotificationsDescription = - "This is a work-around for scalability issues with gundeck user event fan-out. \ - \It does not track all team-wide events, but only `member-join`.\ - \\n\ - \Note that `/teams/notifications` behaves differently from `/notifications`:\ - \\n\ - \- If there is a gap between the notification id requested with `since` and the \ + "

This is a work-around for scalability issues with gundeck user event fan-out. \ + \It does not track all team-wide events, but only `member-join`.

\ + \

Note that `/teams/notifications` behaves differently from `/notifications`:

\ + \
    \ + \
  • If there is a gap between the notification id requested with `since` and the \ \available data, team queues respond with 200 and the data that could be found. \ \They do NOT respond with status 404, but valid data in the body.\ \\n\ - \- The notification with the id given via `since` is included in the \ + \
  • The notification with the id given via `since` is included in the \ \response if it exists. You should remove this and only use it to decide whether \ \there was a gap between your last request and this one.\ \\n\ - \- If the notification id does *not* exist, you get the more recent events from the queue \ + \
  • If the notification id does *not* exist, you get the more recent events from the queue \ \(instead of all of them). This can be done because a notification id is a UUIDv1, which \ \is essentially a time stamp.\ \\n\ - \- There is no corresponding `/last` end-point to get only the most recent event. \ + \
  • There is no corresponding `/last` end-point to get only the most recent event. \ \That end-point was only useful to avoid having to pull the entire queue. In team \ \queues, if you have never requested the queue before and \ - \have no prior notification id, just pull with timestamp 'now'." + \have no prior notification id, just pull with timestamp 'now'. \ + \
\ + \

See also: GET /notifications

\ + \" diff --git a/libs/wire-api/src/Wire/API/Routes/Public/Gundeck.hs b/libs/wire-api/src/Wire/API/Routes/Public/Gundeck.hs index 1b66cd396e1..f37500dd196 100644 --- a/libs/wire-api/src/Wire/API/Routes/Public/Gundeck.hs +++ b/libs/wire-api/src/Wire/API/Routes/Public/Gundeck.hs @@ -115,6 +115,7 @@ type NotificationAPI = :<|> Named "get-notifications" ( Summary "Fetch notifications" + :> Description "See also: GET /teams/notifications" :> From 'V3 :> ZUser :> "notifications" From a759c9c2f735bd886555d58a2684c73d293cfe9b Mon Sep 17 00:00:00 2001 From: Matthias Fischmann Date: Tue, 12 May 2026 11:00:37 +0200 Subject: [PATCH 4/5] [WPB-24978] Discontinue redundant end-point for fetching user clients. (#5222) --- ...undant-end-point-for-fetching-user-clients | 1 + integration/test/API/Brig.hs | 28 ++++++++++------- .../src/Wire/API/Routes/Public/Brig.hs | 2 ++ .../brig/test/integration/API/User/Util.hs | 31 +++++++++++++++---- 4 files changed, 45 insertions(+), 17 deletions(-) create mode 100644 changelog.d/1-api-changes/WPB-24978-discontinue-redundant-end-point-for-fetching-user-clients diff --git a/changelog.d/1-api-changes/WPB-24978-discontinue-redundant-end-point-for-fetching-user-clients b/changelog.d/1-api-changes/WPB-24978-discontinue-redundant-end-point-for-fetching-user-clients new file mode 100644 index 00000000000..72af6768bbe --- /dev/null +++ b/changelog.d/1-api-changes/WPB-24978-discontinue-redundant-end-point-for-fetching-user-clients @@ -0,0 +1 @@ +Discontinue redundant end-point for fetching user clients. diff --git a/integration/test/API/Brig.hs b/integration/test/API/Brig.hs index 56b0d0b5867..de619699591 100644 --- a/integration/test/API/Brig.hs +++ b/integration/test/API/Brig.hs @@ -20,6 +20,7 @@ module API.Brig where import API.BrigCommon import API.Common import qualified Data.Aeson as Aeson +import qualified Data.Aeson.KeyMap as KM import qualified Data.ByteString.Base64 as Base64 import Data.Foldable import Data.Function @@ -222,6 +223,9 @@ deleteClient user client = do ] -- | https://staging-nginz-https.zinfra.io/v5/api/swagger-ui/#/default/get_users__uid_domain___uid__clients +-- +-- this end-point has been removed from V16, but we keep this helper +-- and call listUserClients under the hood. getClientsQualified :: ( HasCallStack, MakesValue user, @@ -232,17 +236,19 @@ getClientsQualified :: domain -> otherUser -> App Response -getClientsQualified user domain otherUser = do - ouid <- objId otherUser - d <- objDomain domain - req <- - baseRequest user Brig Versioned $ - "/users/" - <> d - <> "/" - <> ouid - <> "/clients" - submit "GET" req +getClientsQualified user _domain otherUser = do + (dom, uid) <- objQid otherUser + r <- listUsersClients user [otherUser] + case (r.status, r.json) of + (200, Just v) -> do + let lookupKey k (Object m) = KM.lookup (fromString k) m + lookupKey _ _ = Nothing + clients = fromMaybe (Array mempty) $ do + qum <- lookupKey "qualified_user_map" v + domMap <- lookupKey dom qum + lookupKey uid domMap + pure r {json = Just clients} + _ -> pure r -- | https://staging-nginz-https.zinfra.io/v5/api/swagger-ui/#/default/post_users_list_clients listUsersClients :: (HasCallStack, MakesValue user, MakesValue qualifiedUserIds) => user -> [qualifiedUserIds] -> App Response diff --git a/libs/wire-api/src/Wire/API/Routes/Public/Brig.hs b/libs/wire-api/src/Wire/API/Routes/Public/Brig.hs index c4c3ebd7815..330d7869697 100644 --- a/libs/wire-api/src/Wire/API/Routes/Public/Brig.hs +++ b/libs/wire-api/src/Wire/API/Routes/Public/Brig.hs @@ -1224,6 +1224,8 @@ type ClientAPI = :<|> Named "get-user-clients-qualified" ( Summary "Get all of a user's clients" + :> Description "This will go away in V16, please use POST /users/list-clients instead." + :> Until 'V16 :> "users" :> QualifiedCaptureUserId "uid" :> "clients" diff --git a/services/brig/test/integration/API/User/Util.hs b/services/brig/test/integration/API/User/Util.hs index 97be590cba8..5dae8712961 100644 --- a/services/brig/test/integration/API/User/Util.hs +++ b/services/brig/test/integration/API/User/Util.hs @@ -28,6 +28,7 @@ import Codec.MIME.Type qualified as MIME import Control.Lens (preview, (^?)) import Control.Monad.Catch (MonadCatch) import Data.Aeson +import Data.Aeson.KeyMap qualified as KM import Data.Aeson.Lens import Data.ByteString.Builder (toLazyByteString) import Data.ByteString.Char8 (pack) @@ -225,12 +226,30 @@ getUserClientsUnqualified brig uid = . paths ["users", toByteString' uid, "clients"] . zUser uid -getUserClientsQualified :: Brig -> UserId -> Domain -> UserId -> (MonadHttp m) => m ResponseLBS -getUserClientsQualified brig zusr domain uid = - get $ - brig - . paths ["users", toByteString' domain, toByteString' uid, "clients"] - . zUser zusr +-- https://staging-nginz-https.zinfra.io/v15/api/swagger-ui/#/default/get-user-clients-qualified +-- has been removed from the API in v16, so we call +-- https://staging-nginz-https.zinfra.io/v14/api/swagger-ui/#/default/list-clients-bulk%40v2 +-- under the hood. +getUserClientsQualified :: forall m. (Monad m, Applicative m, MonadHttp m) => Brig -> UserId -> Domain -> UserId -> m ResponseLBS +getUserClientsQualified brig zusr domain uid = do + r <- + post $ + brig + . paths ["users", "list-clients"] + . contentJson + . zUser zusr + . body (RequestBodyLBS $ encode $ object ["qualified_users" .= [object ["domain" .= domain, "id" .= uid]]]) + pure r {responseBody = fmap extractClients (responseBody r)} + where + extractClients bs = + fromMaybe bs $ do + v <- decode bs + clients <- lookupKey "qualified_user_map" v >>= lookupKey domStr >>= lookupKey uidStr + pure (encode clients) + lookupKey k (Object m) = KM.lookup (fromString k) m + lookupKey _ _ = Nothing + domStr = cs (toByteString' domain) :: String + uidStr = cs (toByteString' uid) :: String deleteClient :: Brig -> UserId -> ClientId -> Maybe Text -> (MonadHttp m) => m ResponseLBS deleteClient brig u c pw = From 244073941e483ca2705271c4b9cb179ca1fcdb0e Mon Sep 17 00:00:00 2001 From: Zebot Date: Tue, 12 May 2026 10:18:30 +0000 Subject: [PATCH 5/5] Add changelog for Release 2026-05-12 --- CHANGELOG.md | 30 +++++++++++++++++++ changelog.d/0-release-notes/WPB-22963 | 5 ---- ...undant-end-point-for-fetching-user-clients | 1 - .../4-docs/WPB-24978-improve-swagger-api-docs | 1 - changelog.d/5-internal/WPB-23903 | 1 - 5 files changed, 30 insertions(+), 8 deletions(-) delete mode 100644 changelog.d/0-release-notes/WPB-22963 delete mode 100644 changelog.d/1-api-changes/WPB-24978-discontinue-redundant-end-point-for-fetching-user-clients delete mode 100644 changelog.d/4-docs/WPB-24978-improve-swagger-api-docs delete mode 100644 changelog.d/5-internal/WPB-23903 diff --git a/CHANGELOG.md b/CHANGELOG.md index 758a603d3f2..c0f0753aa19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,33 @@ +# [2026-05-12] (Chart Release 5.32.0) + +## Release notes + + +* - `postgresMigration` now has a single source of truth in the Galley chart values. Galley, Brig, and background-worker all read their PostgreSQL migration settings from there. + - If your deployment overrides the full `postgresMigration` object, add the new `domainRegistration` field to that override. Otherwise services may fail to start because the config is incomplete. + - To migrate domain registration data to PostgreSQL, set `postgresMigration.domainRegistration` to `migration-to-postgresql`, run the background-worker migration with `migrateDomainRegistration: true`, and switch the setting to `postgresql` after completion. + - The domain registration migration covers these Cassandra tables: + `domain_registration`, `domain_registration_by_team`, and `domain_registration_challenge`. (#5195) + + +## API changes + + +* Discontinue redundant end-point for fetching user clients. (#5222) + + +## Documentation + + +* Improve swagger API docs. (#5220) + + +## Internal changes + + +* New `wire-ingress` Helm chart — Gateway API / Envoy Gateway replacement for `nginx-ingress-services`. Not yet production-ready. (#5150) + + # [2026-05-08] (Chart Release 5.31.0) ## API changes diff --git a/changelog.d/0-release-notes/WPB-22963 b/changelog.d/0-release-notes/WPB-22963 deleted file mode 100644 index db32715b668..00000000000 --- a/changelog.d/0-release-notes/WPB-22963 +++ /dev/null @@ -1,5 +0,0 @@ -- `postgresMigration` now has a single source of truth in the Galley chart values. Galley, Brig, and background-worker all read their PostgreSQL migration settings from there. -- If your deployment overrides the full `postgresMigration` object, add the new `domainRegistration` field to that override. Otherwise services may fail to start because the config is incomplete. -- To migrate domain registration data to PostgreSQL, set `postgresMigration.domainRegistration` to `migration-to-postgresql`, run the background-worker migration with `migrateDomainRegistration: true`, and switch the setting to `postgresql` after completion. -- The domain registration migration covers these Cassandra tables: - `domain_registration`, `domain_registration_by_team`, and `domain_registration_challenge`. diff --git a/changelog.d/1-api-changes/WPB-24978-discontinue-redundant-end-point-for-fetching-user-clients b/changelog.d/1-api-changes/WPB-24978-discontinue-redundant-end-point-for-fetching-user-clients deleted file mode 100644 index 72af6768bbe..00000000000 --- a/changelog.d/1-api-changes/WPB-24978-discontinue-redundant-end-point-for-fetching-user-clients +++ /dev/null @@ -1 +0,0 @@ -Discontinue redundant end-point for fetching user clients. diff --git a/changelog.d/4-docs/WPB-24978-improve-swagger-api-docs b/changelog.d/4-docs/WPB-24978-improve-swagger-api-docs deleted file mode 100644 index 98ec58d6a35..00000000000 --- a/changelog.d/4-docs/WPB-24978-improve-swagger-api-docs +++ /dev/null @@ -1 +0,0 @@ -Improve swagger API docs. diff --git a/changelog.d/5-internal/WPB-23903 b/changelog.d/5-internal/WPB-23903 deleted file mode 100644 index c7a1e11ef4f..00000000000 --- a/changelog.d/5-internal/WPB-23903 +++ /dev/null @@ -1 +0,0 @@ -New `wire-ingress` Helm chart — Gateway API / Envoy Gateway replacement for `nginx-ingress-services`. Not yet production-ready.