fix(databases): preserve encoded postgres credentials

api/formance.com/v1beta1/database_types.go

@@ -47,6 +47,7 @@ type DatabaseStatus struct {
 //
 // It uses the settings `postgres.<module-name>.uri` which must have the following uri format: `postgresql://[<username>@<password>]@<host>/<db-name>`
 // Additionally, the uri can define a query param `secret` indicating a k8s secret, than must be used to retrieve database credentials.
+// Credentials in the secret are expected to be URL-encoded by default. Set `secretCredentialsEncoding=raw` to let the operator encode them.
 //
 // On creation, the reconciler behind the Database object will create the database on the postgresql server using a k8s job.
 // On Deletion, by default, the reconciler will let the database untouched.

config/crd/bases/formance.com_databases.yaml

@@ -39,6 +39,7 @@ spec:
 
           It uses the settings `postgres.<module-name>.uri` which must have the following uri format: `postgresql://[<username>@<password>]@<host>/<db-name>`
           Additionally, the uri can define a query param `secret` indicating a k8s secret, than must be used to retrieve database credentials.
+          Credentials in the secret are expected to be URL-encoded by default. Set `secretCredentialsEncoding=raw` to let the operator encode them.
 
           On creation, the reconciler behind the Database object will create the database on the postgresql server using a k8s job.
           On Deletion, by default, the reconciler will let the database untouched.

docs/07-Upgrade/01-Upgrade from the operator.md

@@ -13,6 +13,20 @@ Once the upgrade is complete, you can verify the operator is running the latest
 kubectl -n formance-system describe deployments operator
 ```
 
+## PostgreSQL secret credentials in Operator v3.12.0
+
+Operator `v3.12.0`, shipped with the Formance Helm chart `formance-1.15.0`, introduced URL encoding for PostgreSQL credentials read from Kubernetes Secrets. Earlier operator versions injected these values directly into the connection URI, so deployments with special characters commonly stored credentials in URL-encoded form already.
+
+To preserve compatibility with these deployments, credentials referenced by the `secret` PostgreSQL URI parameter are treated as URL-encoded by default. No configuration change is required when the Secret already contains encoded values such as `p%5Ess%20word`.
+
+When the Secret contains raw credentials, opt in to operator-managed encoding with `secretCredentialsEncoding=raw`:
+
+```text
+postgresql://postgres:5432?secret=postgres&secretCredentialsEncoding=raw
+```
+
+The operator consumes both `secret` and `secretCredentialsEncoding`; neither parameter is included in the final PostgreSQL connection URI.
+
 ## Components upgrade
 The upgrade process is managed by the operator, who will upgrade the components one by one, as specified in the `versions` CRD. Any migration that needs to be carried out will also be managed by the operator.
 
@@ -58,4 +72,4 @@ helm upgrade regions oci://ghcr.io/formancehq/helm/regions \
 
 As soon as Operator v2 starts, it will begin migrating v1 objects to v2 objects, then create the new associated resources. All steps have been designed to be carried out without any service interruption. However, we recommend performing this update in Staging before doing it in Production.
 
-If you encounter any issues, do not hesitate to contact us. We will be happy to help you with the migration process.
+If you encounter any issues, do not hesitate to contact us. We will be happy to help you with the migration process.

docs/09-Configuration reference/01-Settings.md

@@ -92,10 +92,11 @@ Scheme: postgresql
 
 Query params :
 
-| Name           | Type   | Default | Description                                    |
-| -------------- | ------ | ------- | ---------------------------------------------- |
-| secret         | string |         | Specify a secret where credentials are defined |
-| disableSSLMode | bool   | false   | Disable SSL on Postgres connection             |
+| Name                      | Type   | Default      | Description                                                                    |
+| ------------------------- | ------ | ------------ | ------------------------------------------------------------------------------ |
+| secret                    | string |              | Specify a secret where credentials are defined                                 |
+| secretCredentialsEncoding | string | `urlEncoded` | Use `raw` to let the operator URL-encode the credentials read from the secret   |
+| disableSSLMode            | bool   | false        | Disable SSL on Postgres connection                                             |
 
 In addition to the parameters above, any extra query parameters included in the URI are passed through to the final `POSTGRES_URI` environment variable. This is useful for managed PostgreSQL providers (e.g. Google Cloud SQL, Azure Database) that require additional connection parameters.
 
@@ -105,7 +106,9 @@ For example:
 postgresql://user:pass@host:5432?sslmode=require&tcpKeepAlive=true
 ```
 
-The `secret` and `disableSSLMode` parameters are consumed by the operator and will not appear in the resulting connection string. When `disableSSLMode=true` is set, it overrides any `sslmode` parameter with `sslmode=disable`.
+The `secret`, `secretCredentialsEncoding`, and `disableSSLMode` parameters are consumed by the operator and will not appear in the resulting connection string. When `disableSSLMode=true` is set, it overrides any `sslmode` parameter with `sslmode=disable`.
+
+Credentials stored in the referenced secret are assumed to be URL-encoded by default for compatibility with operator versions before `v3.12.0`. Set `secretCredentialsEncoding=raw` when the `username` and `password` keys contain raw values that the operator must encode before building the PostgreSQL URI.
 
 ### ElasticSearch URI format
 

docs/09-Configuration reference/02-Custom Resource Definitions.md

@@ -2141,6 +2141,7 @@ Database represent a concrete database on a PostgreSQL server, it is created by
 
 It uses the settings `postgres.<module-name>.uri` which must have the following uri format: `postgresql://[<username>@<password>]@<host>/<db-name>`
 Additionally, the uri can define a query param `secret` indicating a k8s secret, than must be used to retrieve database credentials.
+Credentials in the secret are expected to be URL-encoded by default. Set `secretCredentialsEncoding=raw` to let the operator encode them.
 
 On creation, the reconciler behind the Database object will create the database on the postgresql server using a k8s job.
 On Deletion, by default, the reconciler will let the database untouched.

docs/09-Configuration reference/settings.catalog.json

@@ -43,7 +43,7 @@
       "key": "clear-database",
       "valueType": "bool",
       "sources": [
-        "internal/resources/databases/init.go:139"
+        "internal/resources/databases/init.go:143"
       ]
     },
     {
@@ -519,7 +519,7 @@
         }
       ],
       "sources": [
-        "internal/resources/databases/env.go:80"
+        "internal/resources/databases/env.go:85"
       ]
     },
     {

helm/crds/templates/crds/apiextensions.k8s.io_v1_customresourcedefinition_databases.formance.com.yaml

@@ -42,6 +42,7 @@ spec:
 
           It uses the settings `postgres.<module-name>.uri` which must have the following uri format: `postgresql://[<username>@<password>]@<host>/<db-name>`
           Additionally, the uri can define a query param `secret` indicating a k8s secret, than must be used to retrieve database credentials.
+          Credentials in the secret are expected to be URL-encoded by default. Set `secretCredentialsEncoding=raw` to let the operator encode them.
 
           On creation, the reconciler behind the Database object will create the database on the postgresql server using a k8s job.
           On Deletion, by default, the reconciler will let the database untouched.

internal/resources/databases/env.go

@@ -31,6 +31,10 @@ func GetPostgresEnvVars(ctx core.Context, stack *v1beta1.Stack, database *v1beta
 			)
 		} else {
 			secret := database.Status.URI.Query().Get("secret")
+			credentialsEncoding, err := parsePostgresCredentialsEncoding(database.Status.URI.Query().Get(postgresCredentialsEncodingQueryParam))
+			if err != nil {
+				return nil, err
+			}
 			postgresURIUsernameEnv = "POSTGRES_URL_ENCODED_USERNAME"
 			postgresURIPasswordEnv = "POSTGRES_URL_ENCODED_PASSWORD"
 			encodedSecretName := getEncodedPostgresCredentialsSecretName(database, secret)
@@ -39,6 +43,7 @@ func GetPostgresEnvVars(ctx core.Context, stack *v1beta1.Stack, database *v1beta
 				core.EnvFromSecret("POSTGRES_PASSWORD", secret, postgresCredentialsPasswordKey),
 				core.EnvFromSecret("POSTGRES_URL_ENCODED_USERNAME", encodedSecretName, postgresCredentialsUsernameKey),
 				core.EnvFromSecret("POSTGRES_URL_ENCODED_PASSWORD", encodedSecretName, postgresCredentialsPasswordKey),
+				core.Env("POSTGRES_CREDENTIALS_ENCODING", string(credentialsEncoding)),
 			)
 		}
 		ret = append(ret,
@@ -130,6 +135,7 @@ func BuildPostgresQueryString(rawQuery url.Values) string {
 	delete(params, "disableSSLMode")
 	delete(params, "secret")
 	delete(params, "awsRole")
+	delete(params, postgresCredentialsEncodingQueryParam)
 	if settings.IsTrue(rawQuery.Get("disableSSLMode")) {
 		params.Set("sslmode", "disable")
 	}

internal/resources/databases/env_test.go

@@ -69,6 +69,11 @@ func TestBuildPostgresQueryString(t *testing.T) {
 			uri:           "postgresql://user:pass@host:5432?awsRole=my-role&sslmode=require",
 			expectedQuery: "sslmode=require",
 		},
+		{
+			name:          "secret credentials encoding is filtered out",
+			uri:           "postgresql://host:5432?secret=creds&secretCredentialsEncoding=raw&sslmode=require",
+			expectedQuery: "sslmode=require",
+		},
 	}
 
 	for _, tc := range testCases {
@@ -196,6 +201,55 @@ func TestGetPostgresEnvVarsUsesEncodedSecretForURI(t *testing.T) {
 	require.Equal(t, "$(POSTGRES_NO_DATABASE_URI)/$(POSTGRES_DATABASE)", envByName["POSTGRES_URI"].Value)
 }
 
+func TestGetPostgresEnvVarsIncludesCredentialsEncodingRolloutSignal(t *testing.T) {
+	t.Parallel()
+
+	testCases := []struct {
+		name     string
+		uri      string
+		expected string
+	}{
+		{
+			name:     "defaults to url encoded",
+			uri:      "postgresql://postgres:5432?secret=postgres",
+			expected: "urlEncoded",
+		},
+		{
+			name:     "raw",
+			uri:      "postgresql://postgres:5432?secret=postgres&secretCredentialsEncoding=raw",
+			expected: "raw",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			postgresURI, err := v1beta1.ParseURL(tc.uri)
+			require.NoError(t, err)
+			database := &v1beta1.Database{
+				ObjectMeta: metav1.ObjectMeta{Name: "stack-ledger"},
+				Spec: v1beta1.DatabaseSpec{
+					Service: "ledger",
+				},
+				Status: v1beta1.DatabaseStatus{
+					URI:      postgresURI,
+					Database: "ledger",
+				},
+			}
+
+			envVars, err := GetPostgresEnvVars(newTestContext(t), &v1beta1.Stack{ObjectMeta: metav1.ObjectMeta{Name: "stack"}}, database)
+			require.NoError(t, err)
+
+			envByName := make(map[string]corev1.EnvVar, len(envVars))
+			for _, envVar := range envVars {
+				envByName[envVar.Name] = envVar
+			}
+			require.Equal(t, tc.expected, envByName["POSTGRES_CREDENTIALS_ENCODING"].Value)
+		})
+	}
+}
+
 func TestGetPostgresEnvVarsAvoidsEncodedSecretNameCollision(t *testing.T) {
 	t.Parallel()
 

internal/resources/databases/init.go

@@ -57,7 +57,11 @@ func Reconcile(ctx core.Context, stack *v1beta1.Stack, database *v1beta1.Databas
 		if err != nil {
 			return err
 		}
-		if err := reconcileEncodedPostgresCredentialsSecret(ctx, stack, database, secret); err != nil {
+		credentialsEncoding, err := parsePostgresCredentialsEncoding(databaseURL.Query().Get(postgresCredentialsEncodingQueryParam))
+		if err != nil {
+			return err
+		}
+		if err := reconcileEncodedPostgresCredentialsSecret(ctx, stack, database, secret, credentialsEncoding); err != nil {
 			return err
 		}
 	} else {
@@ -208,7 +212,11 @@ func reconcileEncodedPostgresCredentialsSecretBeforeDatabaseDelete(ctx core.Cont
 	if sourceSecretName == "" {
 		return nil
 	}
-	return reconcileEncodedPostgresCredentialsSecret(ctx, stack, database, sourceSecretName)
+	credentialsEncoding, err := parsePostgresCredentialsEncoding(database.Status.URI.Query().Get(postgresCredentialsEncodingQueryParam))
+	if err != nil {
+		return err
+	}
+	return reconcileEncodedPostgresCredentialsSecret(ctx, stack, database, sourceSecretName, credentialsEncoding)
 }
 
 func handleDatabaseJob(ctx core.Context, stack *v1beta1.Stack, database *v1beta1.Database, name string, args ...string) error {

internal/resources/databases/secret.go

@@ -23,8 +23,14 @@ const (
 	encodedPostgresCredentialsSecretSuffix              = "postgres-uri-credentials"
 	collisionSafeEncodedPostgresCredentialsSecretSuffix = "encoded-postgres-uri-credentials"
 	encodedPostgresCredentialsSecretAnnotation          = "formance.com/encoded-postgres-credentials-secret"
+	postgresCredentialsEncodingQueryParam               = "secretCredentialsEncoding"
+
+	postgresCredentialsEncodingURLEncoded postgresCredentialsEncoding = "urlEncoded"
+	postgresCredentialsEncodingRaw        postgresCredentialsEncoding = "raw"
 )
 
+type postgresCredentialsEncoding string
+
 func getEncodedPostgresCredentialsSecretName(database *v1beta1.Database, sourceSecretName ...string) string {
 	name := fmt.Sprintf("%s-%s", database.Name, encodedPostgresCredentialsSecretSuffix)
 	if len(sourceSecretName) > 0 && sourceSecretName[0] == name {
@@ -64,7 +70,29 @@ func dedupePostgresCredentialsSecretNames(names []string) []string {
 	return ret
 }
 
-func reconcileEncodedPostgresCredentialsSecret(ctx core.Context, stack *v1beta1.Stack, database *v1beta1.Database, secretName string) error {
+func parsePostgresCredentialsEncoding(value string) (postgresCredentialsEncoding, error) {
+	switch postgresCredentialsEncoding(value) {
+	case "", postgresCredentialsEncodingURLEncoded:
+		return postgresCredentialsEncodingURLEncoded, nil
+	case postgresCredentialsEncodingRaw:
+		return postgresCredentialsEncodingRaw, nil
+	default:
+		return "", fmt.Errorf("invalid %s %q: expected %q or %q",
+			postgresCredentialsEncodingQueryParam,
+			value,
+			postgresCredentialsEncodingURLEncoded,
+			postgresCredentialsEncodingRaw,
+		)
+	}
+}
+
+func reconcileEncodedPostgresCredentialsSecret(
+	ctx core.Context,
+	stack *v1beta1.Stack,
+	database *v1beta1.Database,
+	secretName string,
+	credentialsEncoding postgresCredentialsEncoding,
+) error {
 	sourceSecret := &corev1.Secret{}
 	if err := ctx.GetClient().Get(ctx, types.NamespacedName{
 		Namespace: stack.Name,
@@ -84,6 +112,10 @@ func reconcileEncodedPostgresCredentialsSecret(ctx core.Context, stack *v1beta1.
 	if !ok {
 		return fmt.Errorf("postgres credentials secret %s/%s is missing %q", stack.Name, secretName, postgresCredentialsPasswordKey)
 	}
+	if credentialsEncoding == postgresCredentialsEncodingRaw {
+		username = []byte(escapePostgresCredentialForURI(string(username)))
+		password = []byte(escapePostgresCredentialForURI(string(password)))
+	}
 
 	encodedSecretName := getEncodedPostgresCredentialsSecretName(database, secretName)
 	if err := ensureEncodedPostgresCredentialsSecretCanBeWritten(ctx, stack, database, encodedSecretName); err != nil {
@@ -99,8 +131,8 @@ func reconcileEncodedPostgresCredentialsSecret(ctx core.Context, stack *v1beta1.
 		}
 		secret.Annotations[encodedPostgresCredentialsSecretAnnotation] = "true"
 		secret.Data = map[string][]byte{
-			postgresCredentialsUsernameKey: []byte(escapePostgresCredentialForURI(string(username))),
-			postgresCredentialsPasswordKey: []byte(escapePostgresCredentialForURI(string(password))),
+			postgresCredentialsUsernameKey: username,
+			postgresCredentialsPasswordKey: password,
 		}
 		return nil
 	}, core.WithController[*corev1.Secret](ctx.GetScheme(), database))