From e0059ec712b0fd3cabe322f87d38693646a555dd Mon Sep 17 00:00:00 2001 From: Geoffrey Ragot Date: Mon, 25 May 2026 15:51:08 +0200 Subject: [PATCH 01/30] feat(gateway): add GatewayGRPCAPI CRD for gRPC service exposure Introduce a new GatewayGRPCAPI custom resource that allows modules to expose gRPC services through the gateway, following the same pattern as GatewayHTTPAPI. gRPC routing uses fully-qualified protobuf service names (e.g. formance.ledger.v1.LedgerService) instead of HTTP path prefixes. Traffic is served on the same port (8080) with h2c protocol enabled conditionally when gRPC APIs are registered. Each GatewayGRPCAPI creates a dedicated -grpc Kubernetes Service pointing to the module's gRPC port, avoiding conflicts with existing HTTP services. Co-Authored-By: Claude Opus 4.6 (1M context) --- api/formance.com/v1beta1/gateway_types.go | 3 + .../v1beta1/gatewaygrpcapi_types.go | 88 +++++++++++ .../v1beta1/zz_generated.deepcopy.go | 101 +++++++++++++ .../bases/formance.com_gatewaygrpcapis.yaml | 140 +++++++++++++++++ config/crd/bases/formance.com_gateways.yaml | 5 + config/crd/kustomization.yaml | 1 + config/rbac/role.yaml | 3 + .../02-Custom Resource Definitions.md | 90 +++++++++++ .../settings.catalog.json | 10 +- ...finition_gatewaygrpcapis.formance.com.yaml | 143 ++++++++++++++++++ ...ourcedefinition_gateways.formance.com.yaml | 5 + ..._v1_clusterrole_formance-manager-role.yaml | 3 + internal/resources/all.go | 1 + internal/resources/gatewaygrpcapis/create.go | 45 ++++++ internal/resources/gatewaygrpcapis/init.go | 54 +++++++ internal/resources/gateways/Caddyfile.gotpl | 15 ++ internal/resources/gateways/caddyfile.go | 6 +- internal/resources/gateways/configuration.go | 5 +- internal/resources/gateways/init.go | 17 ++- internal/tests/gateway_controller_test.go | 32 ++++ .../tests/gatewaygrpcapi_controller_test.go | 78 ++++++++++ .../configmap-with-ledger-and-grpc.yaml | 64 ++++++++ .../26-gatewaygrpcapi-sync/chainsaw-test.yaml | 99 ++++++++++++ .../resources/gateway.yaml | 7 + .../resources/grpcapi-updated.yaml | 10 ++ .../resources/grpcapi.yaml | 11 ++ .../resources/httpapi-ledger.yaml | 8 + .../resources/stack.yaml | 6 + 28 files changed, 1041 insertions(+), 9 deletions(-) create mode 100644 api/formance.com/v1beta1/gatewaygrpcapi_types.go create mode 100644 config/crd/bases/formance.com_gatewaygrpcapis.yaml create mode 100644 helm/crds/templates/crds/apiextensions.k8s.io_v1_customresourcedefinition_gatewaygrpcapis.formance.com.yaml create mode 100644 internal/resources/gatewaygrpcapis/create.go create mode 100644 internal/resources/gatewaygrpcapis/init.go create mode 100644 internal/tests/gatewaygrpcapi_controller_test.go create mode 100644 internal/tests/testdata/resources/gateway-controller/configmap-with-ledger-and-grpc.yaml create mode 100644 tests/e2e/chainsaw/26-gatewaygrpcapi-sync/chainsaw-test.yaml create mode 100644 tests/e2e/chainsaw/26-gatewaygrpcapi-sync/resources/gateway.yaml create mode 100644 tests/e2e/chainsaw/26-gatewaygrpcapi-sync/resources/grpcapi-updated.yaml create mode 100644 tests/e2e/chainsaw/26-gatewaygrpcapi-sync/resources/grpcapi.yaml create mode 100644 tests/e2e/chainsaw/26-gatewaygrpcapi-sync/resources/httpapi-ledger.yaml create mode 100644 tests/e2e/chainsaw/26-gatewaygrpcapi-sync/resources/stack.yaml diff --git a/api/formance.com/v1beta1/gateway_types.go b/api/formance.com/v1beta1/gateway_types.go index 5828da2a7..836fff3fb 100644 --- a/api/formance.com/v1beta1/gateway_types.go +++ b/api/formance.com/v1beta1/gateway_types.go @@ -86,6 +86,9 @@ type GatewayStatus struct { // Detected http apis. See [GatewayHTTPAPI](#gatewayhttpapi) //+optional SyncHTTPAPIs []string `json:"syncHTTPAPIs"` + // Detected grpc apis. See [GatewayGRPCAPI](#gatewaygrpcapi) + //+optional + SyncGRPCAPIs []string `json:"syncGRPCAPIs,omitempty"` } //+kubebuilder:object:root=true diff --git a/api/formance.com/v1beta1/gatewaygrpcapi_types.go b/api/formance.com/v1beta1/gatewaygrpcapi_types.go new file mode 100644 index 000000000..760ba134f --- /dev/null +++ b/api/formance.com/v1beta1/gatewaygrpcapi_types.go @@ -0,0 +1,88 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type GatewayGRPCAPISpec struct { + StackDependency `json:",inline"` + // Name indicates the module name (e.g. "ledger") + Name string `json:"name"` + // GRPCServices is the list of fully-qualified gRPC service names + // exposed by this module (e.g. "formance.ledger.v1.LedgerService") + GRPCServices []string `json:"grpcServices"` + // Port is the gRPC port on the backend service + //+optional + //+kubebuilder:default:=8081 + Port int32 `json:"port,omitempty"` +} + +type GatewayGRPCAPIStatus struct { + Status `json:",inline"` + //+optional + Ready bool `json:"ready,omitempty"` +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status +//+kubebuilder:resource:scope=Cluster +//+kubebuilder:printcolumn:name="Stack",type=string,JSONPath=".spec.stack",description="Stack" +//+kubebuilder:printcolumn:name="Ready",type=string,JSONPath=".status.ready",description="Ready" + +// GatewayGRPCAPI is the Schema for the GRPCAPIs API +type GatewayGRPCAPI struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec GatewayGRPCAPISpec `json:"spec,omitempty"` + Status GatewayGRPCAPIStatus `json:"status,omitempty"` +} + +func (in *GatewayGRPCAPI) SetReady(b bool) { + in.Status.Ready = b +} + +func (in *GatewayGRPCAPI) IsReady() bool { + return in.Status.Ready +} + +func (in *GatewayGRPCAPI) SetError(s string) { + in.Status.Info = s +} + +func (a GatewayGRPCAPI) GetStack() string { + return a.Spec.Stack +} + +func (in *GatewayGRPCAPI) GetConditions() *Conditions { + return &in.Status.Conditions +} + +//+kubebuilder:object:root=true + +// GatewayGRPCAPIList contains a list of GatewayGRPCAPI +type GatewayGRPCAPIList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []GatewayGRPCAPI `json:"items"` +} + +func init() { + SchemeBuilder.Register(&GatewayGRPCAPI{}, &GatewayGRPCAPIList{}) +} diff --git a/api/formance.com/v1beta1/zz_generated.deepcopy.go b/api/formance.com/v1beta1/zz_generated.deepcopy.go index 552862474..cafd67bb6 100644 --- a/api/formance.com/v1beta1/zz_generated.deepcopy.go +++ b/api/formance.com/v1beta1/zz_generated.deepcopy.go @@ -965,6 +965,102 @@ func (in *Gateway) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GatewayGRPCAPI) DeepCopyInto(out *GatewayGRPCAPI) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GatewayGRPCAPI. +func (in *GatewayGRPCAPI) DeepCopy() *GatewayGRPCAPI { + if in == nil { + return nil + } + out := new(GatewayGRPCAPI) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *GatewayGRPCAPI) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GatewayGRPCAPIList) DeepCopyInto(out *GatewayGRPCAPIList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]GatewayGRPCAPI, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GatewayGRPCAPIList. +func (in *GatewayGRPCAPIList) DeepCopy() *GatewayGRPCAPIList { + if in == nil { + return nil + } + out := new(GatewayGRPCAPIList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *GatewayGRPCAPIList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GatewayGRPCAPISpec) DeepCopyInto(out *GatewayGRPCAPISpec) { + *out = *in + out.StackDependency = in.StackDependency + if in.GRPCServices != nil { + in, out := &in.GRPCServices, &out.GRPCServices + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GatewayGRPCAPISpec. +func (in *GatewayGRPCAPISpec) DeepCopy() *GatewayGRPCAPISpec { + if in == nil { + return nil + } + out := new(GatewayGRPCAPISpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GatewayGRPCAPIStatus) DeepCopyInto(out *GatewayGRPCAPIStatus) { + *out = *in + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GatewayGRPCAPIStatus. +func (in *GatewayGRPCAPIStatus) DeepCopy() *GatewayGRPCAPIStatus { + if in == nil { + return nil + } + out := new(GatewayGRPCAPIStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GatewayHTTPAPI) DeepCopyInto(out *GatewayHTTPAPI) { *out = *in @@ -1198,6 +1294,11 @@ func (in *GatewayStatus) DeepCopyInto(out *GatewayStatus) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.SyncGRPCAPIs != nil { + in, out := &in.SyncGRPCAPIs, &out.SyncGRPCAPIs + *out = make([]string, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GatewayStatus. diff --git a/config/crd/bases/formance.com_gatewaygrpcapis.yaml b/config/crd/bases/formance.com_gatewaygrpcapis.yaml new file mode 100644 index 000000000..0145e7e0a --- /dev/null +++ b/config/crd/bases/formance.com_gatewaygrpcapis.yaml @@ -0,0 +1,140 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.18.0 + name: gatewaygrpcapis.formance.com +spec: + group: formance.com + names: + kind: GatewayGRPCAPI + listKind: GatewayGRPCAPIList + plural: gatewaygrpcapis + singular: gatewaygrpcapi + scope: Cluster + versions: + - additionalPrinterColumns: + - description: Stack + jsonPath: .spec.stack + name: Stack + type: string + - description: Ready + jsonPath: .status.ready + name: Ready + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: GatewayGRPCAPI is the Schema for the GRPCAPIs API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + grpcServices: + description: |- + GRPCServices is the list of fully-qualified gRPC service names + exposed by this module (e.g. "formance.ledger.v1.LedgerService") + items: + type: string + type: array + name: + description: Name indicates the module name (e.g. "ledger") + type: string + port: + default: 8081 + description: Port is the gRPC port on the backend service + format: int32 + type: integer + stack: + description: Stack indicates the stack on which the module is installed + type: string + required: + - grpcServices + - name + type: object + status: + properties: + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + pattern: ^([A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?)?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - status + - type + type: object + type: array + info: + description: Info can contain any additional like reconciliation errors + type: string + ready: + description: Ready indicates if the resource is seen as completely + reconciled + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/formance.com_gateways.yaml b/config/crd/bases/formance.com_gateways.yaml index aaf0f6acc..71265f185 100644 --- a/config/crd/bases/formance.com_gateways.yaml +++ b/config/crd/bases/formance.com_gateways.yaml @@ -184,6 +184,11 @@ spec: description: Ready indicates if the resource is seen as completely reconciled type: boolean + syncGRPCAPIs: + description: Detected grpc apis. See [GatewayGRPCAPI](#gatewaygrpcapi) + items: + type: string + type: array syncHTTPAPIs: description: Detected http apis. See [GatewayHTTPAPI](#gatewayhttpapi) items: diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 65e9b2c58..7d21bf0ce 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -5,6 +5,7 @@ resources: - bases/formance.com_databases.yaml - bases/formance.com_stacks.yaml - bases/formance.com_brokertopics.yaml +- bases/formance.com_gatewaygrpcapis.yaml - bases/formance.com_gatewayhttpapis.yaml - bases/formance.com_ledgers.yaml - bases/formance.com_gateways.yaml diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 9e2bae12a..87f27b9d5 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -89,6 +89,7 @@ rules: - brokers - brokertopics - databases + - gatewaygrpcapis - gatewayhttpapis - gateways - ledgers @@ -124,6 +125,7 @@ rules: - brokers/finalizers - brokertopics/finalizers - databases/finalizers + - gatewaygrpcapis/finalizers - gatewayhttpapis/finalizers - gateways/finalizers - ledgers/finalizers @@ -153,6 +155,7 @@ rules: - brokers/status - brokertopics/status - databases/status + - gatewaygrpcapis/status - gatewayhttpapis/status - gateways/status - ledgers/status diff --git a/docs/09-Configuration reference/02-Custom Resource Definitions.md b/docs/09-Configuration reference/02-Custom Resource Definitions.md index 3d03b7035..85134da07 100644 --- a/docs/09-Configuration reference/02-Custom Resource Definitions.md +++ b/docs/09-Configuration reference/02-Custom Resource Definitions.md @@ -39,6 +39,7 @@ Other resources : - [BrokerConsumer](#brokerconsumer) - [BrokerTopic](#brokertopic) - [Database](#database) +- [GatewayGRPCAPI](#gatewaygrpcapi) - [GatewayHTTPAPI](#gatewayhttpapi) - [ResourceReference](#resourcereference) - [Versions](#versions) @@ -561,6 +562,7 @@ Gateway is the Schema for the gateways API | `ready` _boolean_ | Ready indicates if the resource is seen as completely reconciled | | | | `info` _string_ | Info can contain any additional like reconciliation errors | | | | `syncHTTPAPIs` _string array_ | Detected http apis. See [GatewayHTTPAPI](#gatewayhttpapi) | | | +| `syncGRPCAPIs` _string array_ | Detected grpc apis. See [GatewayGRPCAPI](#gatewaygrpcapi) | | | #### Ledger @@ -2089,6 +2091,94 @@ It will be recreated with correct uri. | `outOfSync` _boolean_ | OutOfSync indicates than a settings changed the uri of the postgres server
The Database object need to be removed to be recreated | | | +#### GatewayGRPCAPI + + + +GatewayGRPCAPI is the Schema for the GRPCAPIs API + + + + + + + + + + + + + + + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `formance.com/v1beta1` | | | +| `kind` _string_ | `GatewayGRPCAPI` | | | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `spec` _[GatewayGRPCAPISpec](#gatewaygrpcapispec)_ | | | | +| `status` _[GatewayGRPCAPIStatus](#gatewaygrpcapistatus)_ | | | | + + + +##### GatewayGRPCAPISpec + + + + + + + + + + + + + + + + + + + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `stack` _string_ | Stack indicates the stack on which the module is installed | | | +| `name` _string_ | Name indicates the module name (e.g. "ledger") | | | +| `grpcServices` _string array_ | GRPCServices is the list of fully-qualified gRPC service names
exposed by this module (e.g. "formance.ledger.v1.LedgerService") | | | +| `port` _integer_ | Port is the gRPC port on the backend service | 8081 | | + + + + + +##### GatewayGRPCAPIStatus + + + + + + + + + + + + + + + + + + + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `ready` _boolean_ | Ready indicates if the resource is seen as completely reconciled | | | +| `info` _string_ | Info can contain any additional like reconciliation errors | | | +| `ready` _boolean_ | | | | + + #### GatewayHTTPAPI diff --git a/docs/09-Configuration reference/settings.catalog.json b/docs/09-Configuration reference/settings.catalog.json index a4316ca86..1965cbfc3 100644 --- a/docs/09-Configuration reference/settings.catalog.json +++ b/docs/09-Configuration reference/settings.catalog.json @@ -195,35 +195,35 @@ "key": "gateway.caddyfile.grace-period", "valueType": "string", "sources": [ - "internal/resources/gateways/configuration.go:41" + "internal/resources/gateways/configuration.go:42" ] }, { "key": "gateway.caddyfile.shutdown-delay", "valueType": "string", "sources": [ - "internal/resources/gateways/configuration.go:33" + "internal/resources/gateways/configuration.go:34" ] }, { "key": "gateway.caddyfile.trusted-proxies", "valueType": "string[]", "sources": [ - "internal/resources/gateways/configuration.go:17" + "internal/resources/gateways/configuration.go:18" ] }, { "key": "gateway.caddyfile.trusted-proxies-strict", "valueType": "bool", "sources": [ - "internal/resources/gateways/configuration.go:25" + "internal/resources/gateways/configuration.go:26" ] }, { "key": "gateway.config.idle-timeout", "valueType": "string", "sources": [ - "internal/resources/gateways/configuration.go:49" + "internal/resources/gateways/configuration.go:50" ] }, { diff --git a/helm/crds/templates/crds/apiextensions.k8s.io_v1_customresourcedefinition_gatewaygrpcapis.formance.com.yaml b/helm/crds/templates/crds/apiextensions.k8s.io_v1_customresourcedefinition_gatewaygrpcapis.formance.com.yaml new file mode 100644 index 000000000..43e2e0be0 --- /dev/null +++ b/helm/crds/templates/crds/apiextensions.k8s.io_v1_customresourcedefinition_gatewaygrpcapis.formance.com.yaml @@ -0,0 +1,143 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.18.0 + helm.sh/resource-policy: keep + {{- with .Values.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: gatewaygrpcapis.formance.com +spec: + group: formance.com + names: + kind: GatewayGRPCAPI + listKind: GatewayGRPCAPIList + plural: gatewaygrpcapis + singular: gatewaygrpcapi + scope: Cluster + versions: + - additionalPrinterColumns: + - description: Stack + jsonPath: .spec.stack + name: Stack + type: string + - description: Ready + jsonPath: .status.ready + name: Ready + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: GatewayGRPCAPI is the Schema for the GRPCAPIs API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + grpcServices: + description: |- + GRPCServices is the list of fully-qualified gRPC service names + exposed by this module (e.g. "formance.ledger.v1.LedgerService") + items: + type: string + type: array + name: + description: Name indicates the module name (e.g. "ledger") + type: string + port: + default: 8081 + description: Port is the gRPC port on the backend service + format: int32 + type: integer + stack: + description: Stack indicates the stack on which the module is installed + type: string + required: + - grpcServices + - name + type: object + status: + properties: + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + pattern: ^([A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?)?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - status + - type + type: object + type: array + info: + description: Info can contain any additional like reconciliation errors + type: string + ready: + description: Ready indicates if the resource is seen as completely + reconciled + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/helm/crds/templates/crds/apiextensions.k8s.io_v1_customresourcedefinition_gateways.formance.com.yaml b/helm/crds/templates/crds/apiextensions.k8s.io_v1_customresourcedefinition_gateways.formance.com.yaml index 91c103c20..525405009 100644 --- a/helm/crds/templates/crds/apiextensions.k8s.io_v1_customresourcedefinition_gateways.formance.com.yaml +++ b/helm/crds/templates/crds/apiextensions.k8s.io_v1_customresourcedefinition_gateways.formance.com.yaml @@ -187,6 +187,11 @@ spec: description: Ready indicates if the resource is seen as completely reconciled type: boolean + syncGRPCAPIs: + description: Detected grpc apis. See [GatewayGRPCAPI](#gatewaygrpcapi) + items: + type: string + type: array syncHTTPAPIs: description: Detected http apis. See [GatewayHTTPAPI](#gatewayhttpapi) items: diff --git a/helm/operator/templates/gen/rbac.authorization.k8s.io_v1_clusterrole_formance-manager-role.yaml b/helm/operator/templates/gen/rbac.authorization.k8s.io_v1_clusterrole_formance-manager-role.yaml index 24e51fa55..09a9934c5 100644 --- a/helm/operator/templates/gen/rbac.authorization.k8s.io_v1_clusterrole_formance-manager-role.yaml +++ b/helm/operator/templates/gen/rbac.authorization.k8s.io_v1_clusterrole_formance-manager-role.yaml @@ -88,6 +88,7 @@ rules: - brokers - brokertopics - databases + - gatewaygrpcapis - gatewayhttpapis - gateways - ledgers @@ -123,6 +124,7 @@ rules: - brokers/finalizers - brokertopics/finalizers - databases/finalizers + - gatewaygrpcapis/finalizers - gatewayhttpapis/finalizers - gateways/finalizers - ledgers/finalizers @@ -152,6 +154,7 @@ rules: - brokers/status - brokertopics/status - databases/status + - gatewaygrpcapis/status - gatewayhttpapis/status - gateways/status - ledgers/status diff --git a/internal/resources/all.go b/internal/resources/all.go index 619fde7af..d05c5e5fe 100644 --- a/internal/resources/all.go +++ b/internal/resources/all.go @@ -8,6 +8,7 @@ import ( _ "github.com/formancehq/operator/v3/internal/resources/brokers" _ "github.com/formancehq/operator/v3/internal/resources/brokertopics" _ "github.com/formancehq/operator/v3/internal/resources/databases" + _ "github.com/formancehq/operator/v3/internal/resources/gatewaygrpcapis" _ "github.com/formancehq/operator/v3/internal/resources/gatewayhttpapis" _ "github.com/formancehq/operator/v3/internal/resources/gateways" _ "github.com/formancehq/operator/v3/internal/resources/ledgers" diff --git a/internal/resources/gatewaygrpcapis/create.go b/internal/resources/gatewaygrpcapis/create.go new file mode 100644 index 000000000..0d2b2bde0 --- /dev/null +++ b/internal/resources/gatewaygrpcapis/create.go @@ -0,0 +1,45 @@ +package gatewaygrpcapis + +import ( + "k8s.io/apimachinery/pkg/types" + + v1beta1 "github.com/formancehq/operator/v3/api/formance.com/v1beta1" + "github.com/formancehq/operator/v3/internal/core" +) + +type option func(spec *v1beta1.GatewayGRPCAPI) + +func Create(ctx core.Context, owner v1beta1.Module, options ...option) error { + objectName := core.LowerCaseKind(ctx, owner) + _, _, err := core.CreateOrUpdate[*v1beta1.GatewayGRPCAPI](ctx, types.NamespacedName{ + Name: core.GetObjectName(owner.GetStack(), core.LowerCaseKind(ctx, owner)), + }, + func(t *v1beta1.GatewayGRPCAPI) error { + t.Spec = v1beta1.GatewayGRPCAPISpec{ + StackDependency: v1beta1.StackDependency{ + Stack: owner.GetStack(), + }, + Name: objectName, + } + for _, option := range options { + option(t) + } + + return nil + }, + core.WithController[*v1beta1.GatewayGRPCAPI](ctx.GetScheme(), owner), + ) + return err +} + +func WithGRPCServices(services ...string) func(grpcapi *v1beta1.GatewayGRPCAPI) { + return func(grpcapi *v1beta1.GatewayGRPCAPI) { + grpcapi.Spec.GRPCServices = services + } +} + +func WithPort(port int32) func(grpcapi *v1beta1.GatewayGRPCAPI) { + return func(grpcapi *v1beta1.GatewayGRPCAPI) { + grpcapi.Spec.Port = port + } +} diff --git a/internal/resources/gatewaygrpcapis/init.go b/internal/resources/gatewaygrpcapis/init.go new file mode 100644 index 000000000..1f577a0a1 --- /dev/null +++ b/internal/resources/gatewaygrpcapis/init.go @@ -0,0 +1,54 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gatewaygrpcapis + +import ( + corev1 "k8s.io/api/core/v1" + + v1beta1 "github.com/formancehq/operator/v3/api/formance.com/v1beta1" + . "github.com/formancehq/operator/v3/internal/core" + "github.com/formancehq/operator/v3/internal/resources/services" +) + +//+kubebuilder:rbac:groups=formance.com,resources=gatewaygrpcapis,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=formance.com,resources=gatewaygrpcapis/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=formance.com,resources=gatewaygrpcapis/finalizers,verbs=update + +func Reconcile(ctx Context, _ *v1beta1.Stack, grpcAPI *v1beta1.GatewayGRPCAPI) error { + _, err := services.Create(ctx, grpcAPI, grpcAPI.Spec.Name+"-grpc", + services.WithConfig(services.PortConfig{ + ServiceName: grpcAPI.Spec.Name, + PortName: "grpc", + Port: grpcAPI.Spec.Port, + TargetPort: "grpc", + }), + ) + if err != nil { + return err + } + + return nil +} + +func init() { + Init( + WithStackDependencyReconciler(Reconcile, + WithOwn[*v1beta1.GatewayGRPCAPI](&corev1.Service{}), + WithWatchSettings[*v1beta1.GatewayGRPCAPI](), + ), + ) +} diff --git a/internal/resources/gateways/Caddyfile.gotpl b/internal/resources/gateways/Caddyfile.gotpl index 32913fc25..b074fd2b6 100644 --- a/internal/resources/gateways/Caddyfile.gotpl +++ b/internal/resources/gateways/Caddyfile.gotpl @@ -58,6 +58,9 @@ {{- end }} servers { + {{- if .GRPCServices }} + protocols h1 h2c + {{- end }} {{- if and .TrustedProxies (gt (len .TrustedProxies) 0) }} trusted_proxies {{ .TrustedProxies }} {{- end }} @@ -143,6 +146,18 @@ } } + {{- range $i, $service := .GRPCServices }} + {{- range $j, $svcName := $service.GRPCServices }} + handle /{{ $svcName }}/* { + reverse_proxy {{ $service.Name }}-grpc:{{ $service.Port }} { + transport http { + versions h2c 2 + } + } + } + {{- end }} + {{- end }} + # Respond 404 if service does not exists handle /api/* { respond "Not Found" 404 diff --git a/internal/resources/gateways/caddyfile.go b/internal/resources/gateways/caddyfile.go index 629abb747..3ff8652ad 100644 --- a/internal/resources/gateways/caddyfile.go +++ b/internal/resources/gateways/caddyfile.go @@ -14,12 +14,16 @@ import ( type CaddyOptions func(data map[string]any) error func CreateCaddyfile(ctx core.Context, stack *v1beta1.Stack, - gateway *v1beta1.Gateway, httpAPIs []*v1beta1.GatewayHTTPAPI, broker *v1beta1.Broker, options ...CaddyOptions) (string, error) { + gateway *v1beta1.Gateway, httpAPIs []*v1beta1.GatewayHTTPAPI, + grpcAPIs []*v1beta1.GatewayGRPCAPI, broker *v1beta1.Broker, options ...CaddyOptions) (string, error) { data := map[string]any{ "Services": collectionutils.Map(httpAPIs, func(from *v1beta1.GatewayHTTPAPI) v1beta1.GatewayHTTPAPISpec { return from.Spec }), + "GRPCServices": collectionutils.Map(grpcAPIs, func(from *v1beta1.GatewayGRPCAPI) v1beta1.GatewayGRPCAPISpec { + return from.Spec + }), "Platform": ctx.GetPlatform(), "Debug": stack.Spec.Debug, "Port": 8080, diff --git a/internal/resources/gateways/configuration.go b/internal/resources/gateways/configuration.go index 1902ad650..62b39323a 100644 --- a/internal/resources/gateways/configuration.go +++ b/internal/resources/gateways/configuration.go @@ -10,7 +10,8 @@ import ( ) func createConfigMap(ctx core.Context, stack *v1beta1.Stack, - gateway *v1beta1.Gateway, httpAPIs []*v1beta1.GatewayHTTPAPI, broker *v1beta1.Broker) (*v1.ConfigMap, error) { + gateway *v1beta1.Gateway, httpAPIs []*v1beta1.GatewayHTTPAPI, + grpcAPIs []*v1beta1.GatewayGRPCAPI, broker *v1beta1.Broker) (*v1.ConfigMap, error) { options := []CaddyOptions{} @@ -54,7 +55,7 @@ func createConfigMap(ctx core.Context, stack *v1beta1.Stack, options = append(options, withIdleTimeout(*idleTimeout)) } - caddyfile, err := CreateCaddyfile(ctx, stack, gateway, httpAPIs, broker, options...) + caddyfile, err := CreateCaddyfile(ctx, stack, gateway, httpAPIs, grpcAPIs, broker, options...) if err != nil { return nil, err } diff --git a/internal/resources/gateways/init.go b/internal/resources/gateways/init.go index 347964b8c..d0cd250c2 100644 --- a/internal/resources/gateways/init.go +++ b/internal/resources/gateways/init.go @@ -60,6 +60,20 @@ func Reconcile(ctx Context, stack *v1beta1.Stack, gateway *v1beta1.Gateway, vers return from.Spec.Name }) + grpcAPIs := make([]*v1beta1.GatewayGRPCAPI, 0) + err = GetAllStackDependencies(ctx, gateway.Spec.Stack, &grpcAPIs) + if err != nil { + return err + } + + sort.Slice(grpcAPIs, func(i, j int) bool { + return grpcAPIs[i].Spec.Name < grpcAPIs[j].Spec.Name + }) + + gateway.Status.SyncGRPCAPIs = Map(grpcAPIs, func(from *v1beta1.GatewayGRPCAPI) string { + return from.Spec.Name + }) + var broker *v1beta1.Broker if t, err := brokertopics.Find(ctx, stack, "gateway"); err != nil { return err @@ -78,7 +92,7 @@ func Reconcile(ctx Context, stack *v1beta1.Stack, gateway *v1beta1.Gateway, vers } } - configMap, err := createConfigMap(ctx, stack, gateway, httpAPIs, broker) + configMap, err := createConfigMap(ctx, stack, gateway, httpAPIs, grpcAPIs, broker) if err != nil { return err } @@ -160,6 +174,7 @@ func init() { }), WithWatchSettings[*v1beta1.Gateway](), WithWatchDependency[*v1beta1.Gateway](&v1beta1.GatewayHTTPAPI{}), + WithWatchDependency[*v1beta1.Gateway](&v1beta1.GatewayGRPCAPI{}), WithWatchDependency[*v1beta1.Gateway](&v1beta1.Auth{}), brokertopics.Watch[*v1beta1.Gateway]("gateway"), ), diff --git a/internal/tests/gateway_controller_test.go b/internal/tests/gateway_controller_test.go index 19eb9eae6..a7228b2fa 100644 --- a/internal/tests/gateway_controller_test.go +++ b/internal/tests/gateway_controller_test.go @@ -267,6 +267,38 @@ var _ = Describe("GatewayController", func() { }).Should(MatchGoldenFile("gateway-controller", "configmap-with-ledger-and-another-service.yaml")) }) }) + Context("Then adding a GRPCService", func() { + var grpcAPI *v1beta1.GatewayGRPCAPI + BeforeEach(func() { + grpcAPI = &v1beta1.GatewayGRPCAPI{ + ObjectMeta: RandObjectMeta(), + Spec: v1beta1.GatewayGRPCAPISpec{ + StackDependency: v1beta1.StackDependency{ + Stack: stack.Name, + }, + Name: "mymodule", + GRPCServices: []string{"formance.mymodule.v1.MyService"}, + Port: 8081, + }, + } + Expect(Create(grpcAPI)).To(Succeed()) + }) + AfterEach(func() { + Expect(Delete(grpcAPI)).To(Succeed()) + }) + It("Should update the gateway status and Caddyfile with gRPC service", func() { + Eventually(func(g Gomega) []string { + g.Expect(LoadResource("", gateway.Name, gateway)) + return gateway.Status.SyncGRPCAPIs + }).Should(ContainElements(grpcAPI.Spec.Name)) + + Eventually(func(g Gomega) string { + cm := &corev1.ConfigMap{} + g.Expect(LoadResource(stack.Name, "gateway", cm)).To(Succeed()) + return cm.Data["Caddyfile"] + }).Should(MatchGoldenFile("gateway-controller", "configmap-with-ledger-and-grpc.yaml")) + }) + }) Context("With a consumer on gateway", func() { var ( brokerNatsDSNSettings *v1beta1.Settings diff --git a/internal/tests/gatewaygrpcapi_controller_test.go b/internal/tests/gatewaygrpcapi_controller_test.go new file mode 100644 index 000000000..a4ced33e0 --- /dev/null +++ b/internal/tests/gatewaygrpcapi_controller_test.go @@ -0,0 +1,78 @@ +package tests_test + +import ( + "github.com/google/uuid" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + + v1beta1 "github.com/formancehq/operator/v3/api/formance.com/v1beta1" + "github.com/formancehq/operator/v3/internal/resources/settings" + . "github.com/formancehq/operator/v3/internal/tests/internal" +) + +var _ = Describe("GatewayGRPCAPI", func() { + Context("When creating a GatewayGRPCAPI", func() { + var ( + stack *v1beta1.Stack + grpcAPI *v1beta1.GatewayGRPCAPI + ) + BeforeEach(func() { + stack = &v1beta1.Stack{ + ObjectMeta: RandObjectMeta(), + Spec: v1beta1.StackSpec{Version: "v99.0.0"}, + } + grpcAPI = &v1beta1.GatewayGRPCAPI{ + ObjectMeta: RandObjectMeta(), + Spec: v1beta1.GatewayGRPCAPISpec{ + StackDependency: v1beta1.StackDependency{ + Stack: stack.Name, + }, + Name: "mymodule", + GRPCServices: []string{"formance.mymodule.v1.MyService"}, + Port: 8081, + }, + } + }) + JustBeforeEach(func() { + Expect(Create(stack)).To(BeNil()) + Expect(Create(grpcAPI)).To(Succeed()) + }) + AfterEach(func() { + Expect(Delete(grpcAPI)).To(Succeed()) + Expect(Delete(stack)).To(BeNil()) + }) + It("Should create a k8s service with -grpc suffix", func() { + service := &corev1.Service{} + Eventually(func() error { + return LoadResource(stack.Name, "mymodule-grpc", service) + }).Should(BeNil()) + Expect(service).To(BeControlledBy(grpcAPI)) + Expect(service.Spec.Selector).To(Equal(map[string]string{ + "app.kubernetes.io/name": grpcAPI.Spec.Name, + })) + Expect(service.Spec.Ports).To(HaveLen(1)) + Expect(service.Spec.Ports[0].Name).To(Equal("grpc")) + Expect(service.Spec.Ports[0].Port).To(Equal(int32(8081))) + }) + Context("With user defined annotations", func() { + var ( + annotationsSettings *v1beta1.Settings + ) + JustBeforeEach(func() { + annotationsSettings = settings.New(uuid.NewString(), "services.*.annotations", "foo=bar", stack.Name) + Expect(Create(annotationsSettings)).To(Succeed()) + }) + JustAfterEach(func() { + Expect(Delete(annotationsSettings)).To(Succeed()) + }) + It("should add annotations to the service", func() { + Eventually(func(g Gomega) map[string]string { + service := &corev1.Service{} + g.Expect(LoadResource(stack.Name, "mymodule-grpc", service)).To(Succeed()) + return service.Annotations + }).Should(HaveKeyWithValue("foo", "bar")) + }) + }) + }) +}) diff --git a/internal/tests/testdata/resources/gateway-controller/configmap-with-ledger-and-grpc.yaml b/internal/tests/testdata/resources/gateway-controller/configmap-with-ledger-and-grpc.yaml new file mode 100644 index 000000000..507cdb912 --- /dev/null +++ b/internal/tests/testdata/resources/gateway-controller/configmap-with-ledger-and-grpc.yaml @@ -0,0 +1,64 @@ +(cors) { + header { + defer + Access-Control-Allow-Methods "GET,OPTIONS,PUT,POST,DELETE,HEAD,PATCH" + Access-Control-Allow-Headers content-type + Access-Control-Max-Age 100 + Access-Control-Allow-Origin * + } +} + +{ + # Global metrics endpoint (moved from servers block - deprecated location) + metrics + + servers { + protocols h1 h2c + } + + admin :3080 + + # Many directives manipulate the HTTP handler chain and the order in which + # those directives are evaluated matters. So the jwtauth directive must be + # ordered. + # c.f. https://caddyserver.com/docs/caddyfile/directives#directive-order + order versions after metrics +} + +:8080 { + + log { + output stdout + } + handle /api/ledger* { + uri strip_prefix /api/ledger + import cors + reverse_proxy ledger:8080 { + header_up Host {upstream_hostport} + } + } + + handle /versions { + versions { + region "us-west-1" + env "staging" + endpoints { + ledger { + http://ledger:8080/_info http://ledger:8080/ + } + } + } + } + handle /formance.mymodule.v1.MyService/* { + reverse_proxy mymodule-grpc:8081 { + transport http { + versions h2c 2 + } + } + } + + # Respond 404 if service does not exists + handle /api/* { + respond "Not Found" 404 + } +} diff --git a/tests/e2e/chainsaw/26-gatewaygrpcapi-sync/chainsaw-test.yaml b/tests/e2e/chainsaw/26-gatewaygrpcapi-sync/chainsaw-test.yaml new file mode 100644 index 000000000..c2a2e2803 --- /dev/null +++ b/tests/e2e/chainsaw/26-gatewaygrpcapi-sync/chainsaw-test.yaml @@ -0,0 +1,99 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: gateway-grpcapi-sync + labels: + suite: modules + feature: gateway-grpcapi-sync +spec: + concurrent: false + steps: + - name: gateway-renders-grpcapi-routes + try: + - apply: + file: resources/stack.yaml + - apply: + file: resources/gateway.yaml + - apply: + file: resources/httpapi-ledger.yaml + - apply: + file: resources/grpcapi.yaml + - script: + timeout: 2m + content: | + set -eu + # Wait for GatewayGRPCAPI to be ready + kubectl wait --for=jsonpath='{.status.ready}'=true gatewaygrpcapi/chainsaw-grpcapi-mymodule --timeout=2m + + # Verify the -grpc service was created + test "$(kubectl get service mymodule-grpc -n chainsaw-grpcapi-sync -o jsonpath='{.spec.ports[0].port}')" = "8081" + test "$(kubectl get service mymodule-grpc -n chainsaw-grpcapi-sync -o jsonpath='{.spec.ports[0].name}')" = "grpc" + test "$(kubectl get service mymodule-grpc -n chainsaw-grpcapi-sync -o jsonpath='{.spec.selector.app\.kubernetes\.io/name}')" = "mymodule" + + # Verify Gateway synced the GRPCAPIs + kubectl wait --for=jsonpath='{.status.syncGRPCAPIs[0]}'=mymodule gateway/chainsaw-grpcapi-gateway --timeout=2m + + # Verify Caddyfile has h2c protocols and gRPC routes + kubectl get configmap gateway -n chainsaw-grpcapi-sync -o jsonpath='{.data.Caddyfile}' > /tmp/chainsaw-grpcapi-caddyfile + grep -F 'protocols h1 h2c' /tmp/chainsaw-grpcapi-caddyfile + grep -F 'handle /formance.mymodule.v1.MyService/*' /tmp/chainsaw-grpcapi-caddyfile + grep -F 'handle /formance.mymodule.v1.HealthService/*' /tmp/chainsaw-grpcapi-caddyfile + grep -F 'reverse_proxy mymodule-grpc:8081' /tmp/chainsaw-grpcapi-caddyfile + grep -F 'versions h2c 2' /tmp/chainsaw-grpcapi-caddyfile + catch: + - script: + timeout: 2m + content: ../../scripts/dump-diagnostics.sh gateway-grpcapi-routes + - name: grpcapi-mutation-reconfigures-gateway + try: + - apply: + file: resources/grpcapi-updated.yaml + - script: + timeout: 2m + content: | + set -eu + # Wait until new gRPC service name appears in config + until kubectl get configmap gateway -n chainsaw-grpcapi-sync -o jsonpath='{.data.Caddyfile}' | grep -F 'handle /formance.mymodule.v2.MyService/*'; do + sleep 1 + done + kubectl get configmap gateway -n chainsaw-grpcapi-sync -o jsonpath='{.data.Caddyfile}' > /tmp/chainsaw-grpcapi-caddyfile + # New port should be reflected + grep -F 'reverse_proxy mymodule-grpc:9090' /tmp/chainsaw-grpcapi-caddyfile + # Old v1 service should be gone + if grep -F 'formance.mymodule.v1.MyService' /tmp/chainsaw-grpcapi-caddyfile; then + echo "old gRPC service name should not remain after GatewayGRPCAPI mutation" + exit 1 + fi + # Service port should be updated too + test "$(kubectl get service mymodule-grpc -n chainsaw-grpcapi-sync -o jsonpath='{.spec.ports[0].port}')" = "9090" + catch: + - script: + timeout: 2m + content: ../../scripts/dump-diagnostics.sh gateway-grpcapi-mutation + - name: grpcapi-deletion-reconfigures-gateway + try: + - script: + timeout: 2m + content: | + set -eu + kubectl delete gatewaygrpcapi chainsaw-grpcapi-mymodule + # Service should be deleted + kubectl wait --for=delete service/mymodule-grpc -n chainsaw-grpcapi-sync --timeout=2m + # Gateway should no longer list grpc APIs + until test "$(kubectl get gateway chainsaw-grpcapi-gateway -o jsonpath='{.status.syncGRPCAPIs}')" = "" -o "$(kubectl get gateway chainsaw-grpcapi-gateway -o jsonpath='{.status.syncGRPCAPIs}')" = "[]"; do + sleep 1 + done + # Caddyfile should not contain gRPC routes or h2c anymore + kubectl get configmap gateway -n chainsaw-grpcapi-sync -o jsonpath='{.data.Caddyfile}' > /tmp/chainsaw-grpcapi-caddyfile + if grep -F 'mymodule-grpc' /tmp/chainsaw-grpcapi-caddyfile; then + echo "deleted GatewayGRPCAPI should not remain in gateway Caddyfile" + exit 1 + fi + if grep -F 'protocols h1 h2c' /tmp/chainsaw-grpcapi-caddyfile; then + echo "h2c protocols should be removed when no gRPC APIs exist" + exit 1 + fi + catch: + - script: + timeout: 2m + content: ../../scripts/dump-diagnostics.sh gateway-grpcapi-deletion diff --git a/tests/e2e/chainsaw/26-gatewaygrpcapi-sync/resources/gateway.yaml b/tests/e2e/chainsaw/26-gatewaygrpcapi-sync/resources/gateway.yaml new file mode 100644 index 000000000..a4efa7759 --- /dev/null +++ b/tests/e2e/chainsaw/26-gatewaygrpcapi-sync/resources/gateway.yaml @@ -0,0 +1,7 @@ +apiVersion: formance.com/v1beta1 +kind: Gateway +metadata: + name: chainsaw-grpcapi-gateway +spec: + stack: chainsaw-grpcapi-sync + dev: true diff --git a/tests/e2e/chainsaw/26-gatewaygrpcapi-sync/resources/grpcapi-updated.yaml b/tests/e2e/chainsaw/26-gatewaygrpcapi-sync/resources/grpcapi-updated.yaml new file mode 100644 index 000000000..38c1d4422 --- /dev/null +++ b/tests/e2e/chainsaw/26-gatewaygrpcapi-sync/resources/grpcapi-updated.yaml @@ -0,0 +1,10 @@ +apiVersion: formance.com/v1beta1 +kind: GatewayGRPCAPI +metadata: + name: chainsaw-grpcapi-mymodule +spec: + stack: chainsaw-grpcapi-sync + name: mymodule + grpcServices: + - formance.mymodule.v2.MyService + port: 9090 diff --git a/tests/e2e/chainsaw/26-gatewaygrpcapi-sync/resources/grpcapi.yaml b/tests/e2e/chainsaw/26-gatewaygrpcapi-sync/resources/grpcapi.yaml new file mode 100644 index 000000000..36a6448d9 --- /dev/null +++ b/tests/e2e/chainsaw/26-gatewaygrpcapi-sync/resources/grpcapi.yaml @@ -0,0 +1,11 @@ +apiVersion: formance.com/v1beta1 +kind: GatewayGRPCAPI +metadata: + name: chainsaw-grpcapi-mymodule +spec: + stack: chainsaw-grpcapi-sync + name: mymodule + grpcServices: + - formance.mymodule.v1.MyService + - formance.mymodule.v1.HealthService + port: 8081 diff --git a/tests/e2e/chainsaw/26-gatewaygrpcapi-sync/resources/httpapi-ledger.yaml b/tests/e2e/chainsaw/26-gatewaygrpcapi-sync/resources/httpapi-ledger.yaml new file mode 100644 index 000000000..d9dbad967 --- /dev/null +++ b/tests/e2e/chainsaw/26-gatewaygrpcapi-sync/resources/httpapi-ledger.yaml @@ -0,0 +1,8 @@ +apiVersion: formance.com/v1beta1 +kind: GatewayHTTPAPI +metadata: + name: chainsaw-grpcapi-httpapi-ledger +spec: + stack: chainsaw-grpcapi-sync + name: ledger + rules: [] diff --git a/tests/e2e/chainsaw/26-gatewaygrpcapi-sync/resources/stack.yaml b/tests/e2e/chainsaw/26-gatewaygrpcapi-sync/resources/stack.yaml new file mode 100644 index 000000000..a9cce67d7 --- /dev/null +++ b/tests/e2e/chainsaw/26-gatewaygrpcapi-sync/resources/stack.yaml @@ -0,0 +1,6 @@ +apiVersion: formance.com/v1beta1 +kind: Stack +metadata: + name: chainsaw-grpcapi-sync +spec: + version: v0.0.0-e2e From 3971821fc76f4af2b79e8653f5d0e7bc89dff249 Mon Sep 17 00:00:00 2001 From: Geoffrey Ragot Date: Mon, 25 May 2026 16:05:11 +0200 Subject: [PATCH 02/30] fix(e2e): increase chainsaw grpcapi deletion step timeout The deletion step was timing out on K8s 1.34 because the script timeout (2m) was being consumed by kubectl wait --for=delete. Increase the script timeout to 5m and the wait to 3m for more headroom. Co-Authored-By: Claude Opus 4.6 (1M context) --- tests/e2e/chainsaw/26-gatewaygrpcapi-sync/chainsaw-test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/e2e/chainsaw/26-gatewaygrpcapi-sync/chainsaw-test.yaml b/tests/e2e/chainsaw/26-gatewaygrpcapi-sync/chainsaw-test.yaml index c2a2e2803..6e01fa7df 100644 --- a/tests/e2e/chainsaw/26-gatewaygrpcapi-sync/chainsaw-test.yaml +++ b/tests/e2e/chainsaw/26-gatewaygrpcapi-sync/chainsaw-test.yaml @@ -73,12 +73,12 @@ spec: - name: grpcapi-deletion-reconfigures-gateway try: - script: - timeout: 2m + timeout: 5m content: | set -eu kubectl delete gatewaygrpcapi chainsaw-grpcapi-mymodule # Service should be deleted - kubectl wait --for=delete service/mymodule-grpc -n chainsaw-grpcapi-sync --timeout=2m + kubectl wait --for=delete service/mymodule-grpc -n chainsaw-grpcapi-sync --timeout=3m # Gateway should no longer list grpc APIs until test "$(kubectl get gateway chainsaw-grpcapi-gateway -o jsonpath='{.status.syncGRPCAPIs}')" = "" -o "$(kubectl get gateway chainsaw-grpcapi-gateway -o jsonpath='{.status.syncGRPCAPIs}')" = "[]"; do sleep 1 From de1e325891a9fc0946bb21e02ddc98ab75f14a8e Mon Sep 17 00:00:00 2001 From: Geoffrey Ragot Date: Thu, 28 May 2026 17:24:23 +0200 Subject: [PATCH 03/30] feat: add Dockerfile and Pulumi deployment app Extract the operator image build from Earthly into a standard Dockerfile so it can be used by both Earthly (via FROM DOCKERFILE) and the new Pulumi deployment app. The Pulumi app (deployment/operator/) builds the operator image, applies CRDs, and deploys the operator via its Helm chart. Co-Authored-By: Claude Opus 4.6 (1M context) --- Dockerfile | 34 +++ Earthfile | 10 +- deployment/operator/.gitignore | 2 + deployment/operator/Pulumi.yaml | 68 ++++++ deployment/operator/go.mod | 121 +++++++++++ deployment/operator/go.sum | 362 ++++++++++++++++++++++++++++++++ deployment/operator/helpers.go | 296 ++++++++++++++++++++++++++ deployment/operator/main.go | 124 +++++++++++ 8 files changed, 1014 insertions(+), 3 deletions(-) create mode 100644 Dockerfile create mode 100644 deployment/operator/.gitignore create mode 100644 deployment/operator/Pulumi.yaml create mode 100644 deployment/operator/go.mod create mode 100644 deployment/operator/go.sum create mode 100644 deployment/operator/helpers.go create mode 100644 deployment/operator/main.go diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..ac502b218 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,34 @@ +FROM golang:1.26-alpine AS builder + +RUN apk add --no-cache git + +WORKDIR /src + +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . + +ARG VERSION=latest +ARG COMMIT=unknown +ARG LICENCE_PUBLIC_KEY_B64="" + +WORKDIR /src/cmd + +RUN set -e; \ + LDFLAGS="-X github.com/formancehq/operator/v3/cmd.Version=${VERSION} \ + -X github.com/formancehq/operator/v3/cmd.BuildDate=$(date +%s) \ + -X github.com/formancehq/operator/v3/cmd.Commit=${COMMIT}"; \ + if [ -n "$LICENCE_PUBLIC_KEY_B64" ]; then \ + LICENCE_PUBLIC_KEY="$(printf '%s' "$LICENCE_PUBLIC_KEY_B64" | base64 -d)"; \ + LDFLAGS="${LDFLAGS} -X 'github.com/formancehq/go-libs/v5/pkg/authn/licence.formancePublicKey=${LICENCE_PUBLIC_KEY}'"; \ + fi; \ + CGO_ENABLED=0 go build -o /usr/bin/operator -ldflags="${LDFLAGS}" . + +FROM alpine:3.20 + +RUN apk update && apk add --no-cache ca-certificates curl + +ENTRYPOINT ["/usr/bin/operator"] + +COPY --from=builder /usr/bin/operator /usr/bin/operator diff --git a/Earthfile b/Earthfile index bae9f52a9..34b69c286 100644 --- a/Earthfile +++ b/Earthfile @@ -45,9 +45,13 @@ compile: SAVE ARTIFACT main build-image: - FROM core+final-image - ENTRYPOINT ["/usr/bin/operator"] - COPY --pass-args (+compile/main) /usr/bin/operator + ARG LICENCE_PUBLIC_KEY_B64="" + ARG EARTHLY_BUILD_SHA + FROM DOCKERFILE \ + --build-arg LICENCE_PUBLIC_KEY_B64=$LICENCE_PUBLIC_KEY_B64 \ + --build-arg VERSION=$tag \ + --build-arg COMMIT=$EARTHLY_BUILD_SHA \ + -f Dockerfile . ARG REPOSITORY=ghcr.io ARG tag=latest DO --pass-args core+SAVE_IMAGE --COMPONENT=operator --TAG=$tag diff --git a/deployment/operator/.gitignore b/deployment/operator/.gitignore new file mode 100644 index 000000000..7dc315b66 --- /dev/null +++ b/deployment/operator/.gitignore @@ -0,0 +1,2 @@ +deployment/operator/bin/ +operator diff --git a/deployment/operator/Pulumi.yaml b/deployment/operator/Pulumi.yaml new file mode 100644 index 000000000..0b8aac981 --- /dev/null +++ b/deployment/operator/Pulumi.yaml @@ -0,0 +1,68 @@ +name: formance-operator +runtime: + name: go + options: + buildTarget: ./bin/pulumi +description: Formance operator deployment (CRDs, operator) +x-plr-config: + k8s-context: + type: string + required: true + description: Kubernetes context to use + namespace: + type: string + description: Kubernetes namespace (defaults to stack name) + registry: + type: string + default: ghcr.io + description: Docker registry for building images + pull-registry: + type: string + description: Docker registry for pulling images (defaults to registry) + docker-builder-name: + type: string + description: Docker buildx builder name + imageTag: + type: string + description: Image tag (defaults to git version) + arch: + type: string + default: amd64 + description: Target CPU architecture + registry-username: + type: string + secret: true + description: Docker registry username + registry-password: + type: string + secret: true + description: Docker registry password + image-pull-secrets: + type: array + description: "Image pull secrets [{name: string}]" + operator-region: + type: string + default: eu-west-1 + description: Region passed to the operator + operator-env: + type: string + default: staging + description: Environment passed to the operator + operator-dev: + type: boolean + default: false + description: Enable operator dev mode + licence-token: + type: string + secret: true + description: Formance licence token + licence-issuer: + type: string + default: "https://license.formance.cloud/keys" + description: Formance licence issuer URL + node-selector: + type: object + description: "Node selector for the operator pod" + tolerations: + type: array + description: "Tolerations for the operator pod" diff --git a/deployment/operator/go.mod b/deployment/operator/go.mod new file mode 100644 index 000000000..550b43c4b --- /dev/null +++ b/deployment/operator/go.mod @@ -0,0 +1,121 @@ +module github.com/formancehq/operator/v3/deployment/operator + +go 1.26.2 + +require ( + github.com/pulumi/pulumi-docker-build/sdk/go/dockerbuild v0.0.18 + github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.31.1 + github.com/pulumi/pulumi/sdk/v3 v3.243.0 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + dario.cat/mergo v1.0.1 // indirect + github.com/BurntSushi/toml v1.6.0 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/ProtonMail/go-crypto v1.2.0 // indirect + github.com/agext/levenshtein v1.2.3 // indirect + github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect + github.com/atotto/clipboard v0.1.4 // indirect + github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/blang/semver v3.5.1+incompatible // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/charmbracelet/bubbles v1.0.0 // indirect + github.com/charmbracelet/bubbletea v1.3.10 // indirect + github.com/charmbracelet/colorprofile v0.4.3 // indirect + github.com/charmbracelet/lipgloss v1.1.0 // indirect + github.com/charmbracelet/x/ansi v0.11.7 // indirect + github.com/charmbracelet/x/cellbuf v0.0.15 // indirect + github.com/charmbracelet/x/term v0.2.2 // indirect + github.com/cheggaaa/pb v1.0.29 // indirect + github.com/clipperhouse/displaywidth v0.11.0 // indirect + github.com/clipperhouse/uax29/v2 v2.7.0 // indirect + github.com/cloudflare/circl v1.6.3 // indirect + github.com/cyphar/filepath-securejoin v0.6.1 // indirect + github.com/djherbis/times v1.6.0 // indirect + github.com/emirpasic/gods v1.18.1 // indirect + github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect + github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect + github.com/go-git/go-billy/v5 v5.9.0 // indirect + github.com/go-git/go-git/v5 v5.19.1 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/glog v1.2.5 // indirect + github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect + github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/hashicorp/go-version v1.9.0 // indirect + github.com/hashicorp/hcl/v2 v2.24.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/kevinburke/ssh_config v1.2.0 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/lucasb-eyer/go-colorful v1.4.0 // indirect + github.com/mattn/go-isatty v0.0.22 // indirect + github.com/mattn/go-localereader v0.0.1 // indirect + github.com/mattn/go-runewidth v0.0.23 // indirect + github.com/mitchellh/go-ps v1.0.0 // indirect + github.com/mitchellh/go-wordwrap v1.0.1 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect + github.com/muesli/cancelreader v0.2.2 // indirect + github.com/muesli/termenv v0.16.0 // indirect + github.com/opentracing/basictracer-go v1.1.0 // indirect + github.com/opentracing/opentracing-go v1.2.0 // indirect + github.com/pgavlin/fx v0.1.6 // indirect + github.com/pgavlin/fx/v2 v2.0.12 // indirect + github.com/pjbgf/sha1cd v0.6.0 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pkg/term v1.1.0 // indirect + github.com/pulumi/appdash v0.0.0-20231130102222-75f619a67231 // indirect + github.com/pulumi/esc v0.24.0 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect + github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect + github.com/sergi/go-diff v1.4.0 // indirect + github.com/skeema/knownhosts v1.3.1 // indirect + github.com/spf13/cast v1.5.0 // indirect + github.com/spf13/cobra v1.10.2 // indirect + github.com/spf13/pflag v1.0.10 // indirect + github.com/texttheater/golang-levenshtein v1.0.1 // indirect + github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect + github.com/uber/jaeger-lib v2.4.1+incompatible // indirect + github.com/xanzy/ssh-agent v0.3.3 // indirect + github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect + github.com/zclconf/go-cty v1.17.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/collector/featuregate v1.58.0 // indirect + go.opentelemetry.io/collector/pdata v1.58.0 // indirect + go.opentelemetry.io/otel v1.43.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.43.0 // indirect + go.opentelemetry.io/otel/metric v1.43.0 // indirect + go.opentelemetry.io/otel/sdk v1.43.0 // indirect + go.opentelemetry.io/otel/trace v1.43.0 // indirect + go.opentelemetry.io/proto/otlp v1.10.0 // indirect + go.uber.org/atomic v1.11.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/crypto v0.51.0 // indirect + golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect + golang.org/x/mod v0.35.0 // indirect + golang.org/x/net v0.54.0 // indirect + golang.org/x/sync v0.20.0 // indirect + golang.org/x/sys v0.44.0 // indirect + golang.org/x/term v0.43.0 // indirect + golang.org/x/text v0.37.0 // indirect + golang.org/x/tools v0.44.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260519071638-aa98bba5eb94 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260519071638-aa98bba5eb94 // indirect + google.golang.org/grpc v1.81.1 // indirect + google.golang.org/protobuf v1.36.11 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect + lukechampine.com/frand v1.5.1 // indirect +) diff --git a/deployment/operator/go.sum b/deployment/operator/go.sum new file mode 100644 index 000000000..e4ba939bd --- /dev/null +++ b/deployment/operator/go.sum @@ -0,0 +1,362 @@ +dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= +dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= +github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= +github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM= +github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= +github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/ProtonMail/go-crypto v1.2.0 h1:+PhXXn4SPGd+qk76TlEePBfOfivE0zkWFenhGhFLzWs= +github.com/ProtonMail/go-crypto v1.2.0/go.mod h1:9whxjD8Rbs29b4XWbB8irEcE8KHMqaR2e7GWU1R+/PE= +github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo= +github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= +github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY= +github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= +github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= +github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= +github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= +github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= +github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/charmbracelet/bubbles v1.0.0 h1:12J8/ak/uCZEMQ6KU7pcfwceyjLlWsDLAxB5fXonfvc= +github.com/charmbracelet/bubbles v1.0.0/go.mod h1:9d/Zd5GdnauMI5ivUIVisuEm3ave1XwXtD1ckyV6r3E= +github.com/charmbracelet/bubbletea v1.3.10 h1:otUDHWMMzQSB0Pkc87rm691KZ3SWa4KUlvF9nRvCICw= +github.com/charmbracelet/bubbletea v1.3.10/go.mod h1:ORQfo0fk8U+po9VaNvnV95UPWA1BitP1E0N6xJPlHr4= +github.com/charmbracelet/colorprofile v0.4.3 h1:QPa1IWkYI+AOB+fE+mg/5/4HRMZcaXex9t5KX76i20Q= +github.com/charmbracelet/colorprofile v0.4.3/go.mod h1:/zT4BhpD5aGFpqQQqw7a+VtHCzu+zrQtt1zhMt9mR4Q= +github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY= +github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30= +github.com/charmbracelet/x/ansi v0.11.7 h1:kzv1kJvjg2S3r9KHo8hDdHFQLEqn4RBCb39dAYC84jI= +github.com/charmbracelet/x/ansi v0.11.7/go.mod h1:9qGpnAVYz+8ACONkZBUWPtL7lulP9No6p1epAihUZwQ= +github.com/charmbracelet/x/cellbuf v0.0.15 h1:ur3pZy0o6z/R7EylET877CBxaiE1Sp1GMxoFPAIztPI= +github.com/charmbracelet/x/cellbuf v0.0.15/go.mod h1:J1YVbR7MUuEGIFPCaaZ96KDl5NoS0DAWkskup+mOY+Q= +github.com/charmbracelet/x/term v0.2.2 h1:xVRT/S2ZcKdhhOuSP4t5cLi5o+JxklsoEObBSgfgZRk= +github.com/charmbracelet/x/term v0.2.2/go.mod h1:kF8CY5RddLWrsgVwpw4kAa6TESp6EB5y3uxGLeCqzAI= +github.com/cheggaaa/pb v1.0.29 h1:FckUN5ngEk2LpvuG0fw1GEFx6LtyY2pWI/Z2QgCnEYo= +github.com/cheggaaa/pb v1.0.29/go.mod h1:W40334L7FMC5JKWldsTWbdGjLo0RxUKK73K+TuPxX30= +github.com/clipperhouse/displaywidth v0.11.0 h1:lBc6kY44VFw+TDx4I8opi/EtL9m20WSEFgwIwO+UVM8= +github.com/clipperhouse/displaywidth v0.11.0/go.mod h1:bkrFNkf81G8HyVqmKGxsPufD3JhNl3dSqnGhOoSD/o0= +github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= +github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= +github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8= +github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/cyphar/filepath-securejoin v0.6.1 h1:5CeZ1jPXEiYt3+Z6zqprSAgSWiggmpVyciv8syjIpVE= +github.com/cyphar/filepath-securejoin v0.6.1/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c= +github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0= +github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= +github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= +github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= +github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= +github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= +github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= +github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c= +github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= +github.com/go-git/go-billy/v5 v5.9.0 h1:jItGXszUDRtR/AlferWPTMN4j38BQ88XnXKbilmmBPA= +github.com/go-git/go-billy/v5 v5.9.0/go.mod h1:jCnQMLj9eUgGU7+ludSTYoZL/GGmii14RxKFj7ROgHw= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= +github.com/go-git/go-git/v5 v5.19.1 h1:nX27AnaU43/K5bKktKwgBmR9lawoYVe1Ckg0rgzzN00= +github.com/go-git/go-git/v5 v5.19.1/go.mod h1:Pb1v0c7/g8aGQJwx9Us09W85yGoyvSwuhEGMH7zjDKQ= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v1.2.5 h1:DrW6hGnjIhtvhOIiAKT6Psh/Kd/ldepEa81DKeiRJ5I= +github.com/golang/glog v1.2.5/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= +github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 h1:MJG/KsmcqMwFAkh8mTnAwhyKoB+sTAnY4CACC110tbU= +github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-version v1.9.0 h1:CeOIz6k+LoN3qX9Z0tyQrPtiB1DFYRPfCIBtaXPSCnA= +github.com/hashicorp/go-version v1.9.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/hcl/v2 v2.24.0 h1:2QJdZ454DSsYGoaE6QheQZjtKZSUs9Nh2izTWiwQxvE= +github.com/hashicorp/hcl/v2 v2.24.0/go.mod h1:oGoO1FIQYfn/AgyOhlg9qLC6/nOJPX3qGbkZpYAcqfM= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= +github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lucasb-eyer/go-colorful v1.4.0 h1:UtrWVfLdarDgc44HcS7pYloGHJUjHV/4FwW4TvVgFr4= +github.com/lucasb-eyer/go-colorful v1.4.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4= +github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= +github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= +github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= +github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.23 h1:7ykA0T0jkPpzSvMS5i9uoNn2Xy3R383f9HDx3RybWcw= +github.com/mattn/go-runewidth v0.0.23/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc= +github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg= +github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= +github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI= +github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6/go.mod h1:CJlz5H+gyd6CUWT45Oy4q24RdLyn7Md9Vj2/ldJBSIo= +github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= +github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= +github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc= +github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= +github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= +github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= +github.com/opentracing/basictracer-go v1.1.0 h1:Oa1fTSBvAl8pa3U+IJYqrKm0NALwH9OsgwOqDv4xJW0= +github.com/opentracing/basictracer-go v1.1.0/go.mod h1:V2HZueSJEp879yv285Aap1BS69fQMD+MNP1mRs6mBQc= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= +github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/pgavlin/fx v0.1.6 h1:r9jEg69DhNoCd3Xh0+5mIbdbS3PqWrVWujkY76MFRTU= +github.com/pgavlin/fx v0.1.6/go.mod h1:KWZJ6fqBBSh8GxHYqwYCf3rYE7Gp2p0N8tJp8xv9u9M= +github.com/pgavlin/fx/v2 v2.0.12 h1:SjjaJ68Dt8Z4zHwOpY/RPijd7lShs6xYupJbF9ra00M= +github.com/pgavlin/fx/v2 v2.0.12/go.mod h1:M/nF/ooAOy+NUBooYYXl2REARzJ/giPJxfMs8fINfKc= +github.com/pjbgf/sha1cd v0.6.0 h1:3WJ8Wz8gvDz29quX1OcEmkAlUg9diU4GxJHqs0/XiwU= +github.com/pjbgf/sha1cd v0.6.0/go.mod h1:lhpGlyHLpQZoxMv8HcgXvZEhcGs0PG/vsZnEJ7H0iCM= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/term v1.1.0 h1:xIAAdCMh3QIAy+5FrE8Ad8XoDhEU4ufwbaSozViP9kk= +github.com/pkg/term v1.1.0/go.mod h1:E25nymQcrSllhX42Ok8MRm1+hyBdHY0dCeiKZ9jpNGw= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pulumi/appdash v0.0.0-20231130102222-75f619a67231 h1:vkHw5I/plNdTr435cARxCW6q9gc0S/Yxz7Mkd38pOb0= +github.com/pulumi/appdash v0.0.0-20231130102222-75f619a67231/go.mod h1:murToZ2N9hNJzewjHBgfFdXhZKjY3z5cYC1VXk+lbFE= +github.com/pulumi/esc v0.24.0 h1:sCtiB0qbyrlU1ZNzJn4dTLYiChl8xeCBFbHWl1YoXJg= +github.com/pulumi/esc v0.24.0/go.mod h1:eCOOkcDJS6eooGwdE4/E0+pOsvUWG254+KBmPCFwJpA= +github.com/pulumi/pulumi-docker-build/sdk/go/dockerbuild v0.0.18 h1:emkSEfjXfz7i2vNDi43WTqABhP9TY2mQnO2zdL683hw= +github.com/pulumi/pulumi-docker-build/sdk/go/dockerbuild v0.0.18/go.mod h1:BriBqoV2I/58/AZy4/4oJfoiJYX7Nf/NxsAmGXDgvgo= +github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.31.1 h1:Hg9RK9zqIU9kFbD5KeiON06gPP7cLgS68jvsgMBmPgw= +github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.31.1/go.mod h1:BAWI9R3JEEGOp1JlXLPSZKwBGANSrPGUWKtMnS5w5qw= +github.com/pulumi/pulumi/sdk/v3 v3.243.0 h1:pZaMx58nXrdh4XB0cgTlHnL3EMy3/JQwuin3aDuWyRM= +github.com/pulumi/pulumi/sdk/v3 v3.243.0/go.mod h1:BPWWuYPXcPH5YbXGoyy9Rrfa+evrh6IdM51AjDhcDpM= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= +github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= +github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= +github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnBY8= +github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY= +github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= +github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/texttheater/golang-levenshtein v1.0.1 h1:+cRNoVrfiwufQPhoMzB6N0Yf/Mqajr6t1lOv8GyGE2U= +github.com/texttheater/golang-levenshtein v1.0.1/go.mod h1:PYAKrbF5sAiq9wd+H82hs7gNaen0CplQ9uvm6+enD/8= +github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o= +github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= +github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg= +github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= +github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= +github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/zclconf/go-cty v1.17.0 h1:seZvECve6XX4tmnvRzWtJNHdscMtYEx5R7bnnVyd/d0= +github.com/zclconf/go-cty v1.17.0/go.mod h1:wqFzcImaLTI6A5HfsRwB0nj5n0MRZFwmey8YoFPPs3U= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/collector/featuregate v1.58.0 h1:Kh6Dpgbxywv/Q3D6qPehaSxNCxvr/U/ki7CL4y3udCo= +go.opentelemetry.io/collector/featuregate v1.58.0/go.mod h1:4ga1QBMPEejXXmpyJS8lmaRpknJ3Lb9Bvk6e420bUFU= +go.opentelemetry.io/collector/internal/testutil v0.152.0 h1:8LGwekR7mLcUDhT1ofLmdnrHRFuUa3U7PBd95ZvJEjQ= +go.opentelemetry.io/collector/internal/testutil v0.152.0/go.mod h1:Jkjs6rkqs973LqgZ0Fe3zrokQRKULYXPIf4HuqStiEE= +go.opentelemetry.io/collector/pdata v1.58.0 h1:5Lxut3NxKp87066Pzt+3q7+JUuFI5B3teCyLZIF8wIs= +go.opentelemetry.io/collector/pdata v1.58.0/go.mod h1:4vZtODINbC/JF3eGocnatdImzbRHseOywIcr+aULjCg= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 h1:88Y4s2C8oTui1LGM6bTWkw0ICGcOLCAI5l6zsD1j20k= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0/go.mod h1:Vl1/iaggsuRlrHf/hfPJPvVag77kKyvrLeD10kpMl+A= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.43.0 h1:RAE+JPfvEmvy+0LzyUA25/SGawPwIUbZ6u0Wug54sLc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.43.0/go.mod h1:AGmbycVGEsRx9mXMZ75CsOyhSP6MFIcj/6dnG+vhVjk= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= +go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= +go.opentelemetry.io/proto/slim/otlp v1.10.0 h1:iR97Vs/ZDR+y9TfuP9b1XBtdPWeC+OMslIBmhcLU7jM= +go.opentelemetry.io/proto/slim/otlp v1.10.0/go.mod h1:lV9250stpjYLPNA5viFabIgP2QlUGRT1GdTgAf8SIUk= +go.opentelemetry.io/proto/slim/otlp/collector/profiles/v1development v0.3.0 h1:RUF5rO0hAlgiJt1fzQVzcVs3vZVNHIcMLgOgG4rWNcQ= +go.opentelemetry.io/proto/slim/otlp/collector/profiles/v1development v0.3.0/go.mod h1:I89cynRj8y+383o7tEQVg2SVA6SRgDVIouWPUVXjx0U= +go.opentelemetry.io/proto/slim/otlp/profiles/v1development v0.3.0 h1:CQvJSldHRUN6Z8jsUeYv8J0lXRvygALXIzsmAeCcZE0= +go.opentelemetry.io/proto/slim/otlp/profiles/v1development v0.3.0/go.mod h1:xSQ+mEfJe/GjK1LXEyVOoSI1N9JV9ZI923X5kup43W4= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= +golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= +golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= +golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= +golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w= +golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= +golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= +golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= +golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/api v0.0.0-20260519071638-aa98bba5eb94 h1:DddG61lE5LkX6144z22i0gma9BMBs5aZ9B8lZLobxyw= +google.golang.org/genproto/googleapis/api v0.0.0-20260519071638-aa98bba5eb94/go.mod h1:1dCETSCY2YKZNXQE3h4fun3TYwF5p8jejRKZgfWAgAY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260519071638-aa98bba5eb94 h1:eZCjr/aAF8c5ccm5pb6T4EXgIei5MlAAPWPJk+5ArfY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260519071638-aa98bba5eb94/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.81.1 h1:VnnIIZ88UzOOKLukQi+ImGz8O1Wdp8nAGGnvOfEIWQQ= +google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +lukechampine.com/frand v1.5.1 h1:fg0eRtdmGFIxhP5zQJzM1lFDbD6CUfu/f+7WgAZd5/w= +lukechampine.com/frand v1.5.1/go.mod h1:4VstaWc2plN4Mjr10chUD46RAVGWhpkZ5Nja8+Azp0Q= +pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= +pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= diff --git a/deployment/operator/helpers.go b/deployment/operator/helpers.go new file mode 100644 index 000000000..480d61281 --- /dev/null +++ b/deployment/operator/helpers.go @@ -0,0 +1,296 @@ +package main + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/pulumi/pulumi-docker-build/sdk/go/dockerbuild" + "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes" + v1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/core/v1" + metav1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/meta/v1" + "github.com/pulumi/pulumi/sdk/v3/go/pulumi" + "github.com/pulumi/pulumi/sdk/v3/go/pulumi/config" + "gopkg.in/yaml.v3" +) + +func getBuildVersion(gitDir string) string { + cmd := exec.Command("git", "rev-parse", "--short", "HEAD") + cmd.Dir = gitDir + output, err := cmd.Output() + + timestamp := time.Now().Format("20060102-150405") + + if err != nil { + return timestamp + } + + commit := strings.TrimSpace(string(output)) + + cmd = exec.Command("git", "status", "--porcelain") + cmd.Dir = gitDir + statusOutput, _ := cmd.Output() + + if len(statusOutput) > 0 { + return fmt.Sprintf("%s-dirty-%s", commit, timestamp) + } + + return fmt.Sprintf("%s-%s", commit, timestamp) +} + +func getConfigBool(cfg *config.Config, key string, fallback bool) bool { + value := cfg.GetBool(key) + if value { + return true + } + if cfg.Get(key) == "false" { + return false + } + return fallback +} + +type k8sSetup struct { + Provider pulumi.ProviderResource + Namespace *v1.Namespace +} + +func newK8sSetup(ctx *pulumi.Context, cfg *config.Config) (*k8sSetup, error) { + kubeContext := cfg.Require("k8s-context") + + namespaceName := cfg.Get("namespace") + if namespaceName == "" { + namespaceName = ctx.Stack() + } + + k8sProvider, err := kubernetes.NewProvider(ctx, "k8s", &kubernetes.ProviderArgs{ + Context: pulumi.StringPtr(kubeContext), + }) + if err != nil { + return nil, fmt.Errorf("failed to create k8s provider: %w", err) + } + + namespace, err := v1.NewNamespace(ctx, "namespace", &v1.NamespaceArgs{ + Metadata: &metav1.ObjectMetaArgs{ + Name: pulumi.String(namespaceName), + }, + }, pulumi.Provider(k8sProvider), pulumi.RetainOnDelete(true)) + if err != nil { + return nil, fmt.Errorf("failed to create namespace: %w", err) + } + + return &k8sSetup{ + Provider: k8sProvider, + Namespace: namespace, + }, nil +} + +type dockerConfig struct { + Registry string + PullRegistry string + BuilderName string + ImageTag string + Platforms []string + RegistryAuth dockerbuild.RegistryArray +} + +var allPlatforms = []string{"linux-amd64", "linux-arm64"} + +func newDockerConfig(ctx *pulumi.Context, cfg *config.Config) *dockerConfig { + registry := cfg.Get("registry") + if registry == "" { + registry = "ghcr.io" + } + pullRegistry := cfg.Get("pull-registry") + if pullRegistry == "" { + pullRegistry = registry + } + builderName := cfg.Get("docker-builder-name") + + buildVersion := getBuildVersion("../..") + imageTag := cfg.Get("imageTag") + if imageTag == "" { + imageTag = buildVersion + } + + arch := cfg.Get("arch") + if arch == "" { + arch = "amd64" + } + platforms := make([]string, 0, len(allPlatforms)) + for _, p := range allPlatforms { + if strings.HasSuffix(p, arch) { + platforms = append(platforms, p) + } + } + if len(platforms) == 0 { + platforms = []string{"linux-" + arch} + } + + return &dockerConfig{ + Registry: registry, + PullRegistry: pullRegistry, + BuilderName: builderName, + ImageTag: imageTag, + Platforms: platforms, + RegistryAuth: dockerbuild.RegistryArray{ + dockerbuild.RegistryArgs{ + Address: pulumi.String(registry), + Username: config.GetSecret(ctx, "registry-username"), + Password: config.GetSecret(ctx, "registry-password"), + }, + }, + } +} + +type multiArchImage struct { + Index *dockerbuild.Index + Images []*dockerbuild.Image + Ref pulumi.StringOutput + Digest pulumi.StringOutput +} + +func (m *multiArchImage) Resource() pulumi.Resource { + return m.Index +} + +func (dc *dockerConfig) buildImage( + ctx *pulumi.Context, + name string, + contextPath string, + dockerfilePath string, +) (*multiArchImage, error) { + var sources pulumi.StringArray + var images []*dockerbuild.Image + + for _, platform := range dc.Platforms { + img, err := dockerbuild.NewImage(ctx, fmt.Sprintf("%s-%s", name, platform), &dockerbuild.ImageArgs{ + Context: dockerbuild.BuildContextArgs{ + Location: pulumi.String(contextPath), + }, + Builder: dockerbuild.BuilderConfigArgs{ + Name: pulumi.String(dc.BuilderName), + }, + CacheFrom: dockerbuild.CacheFromArray{ + dockerbuild.CacheFromArgs{ + Registry: dockerbuild.CacheFromRegistryArgs{ + Ref: pulumi.Sprintf("%s/%s:buildcache-%s", dc.Registry, name, platform), + }, + }, + }, + CacheTo: dockerbuild.CacheToArray{ + dockerbuild.CacheToArgs{ + Registry: dockerbuild.CacheToRegistryArgs{ + Ref: pulumi.Sprintf("%s/%s:buildcache-%s,mode=max", dc.Registry, name, platform), + }, + }, + }, + Dockerfile: dockerbuild.DockerfileArgs{ + Location: pulumi.String(dockerfilePath), + }, + Platforms: dockerbuild.PlatformArray{ + dockerbuild.Platform(strings.ReplaceAll(platform, "-", "/")), + }, + Push: pulumi.Bool(true), + Registries: dc.RegistryAuth, + Tags: pulumi.StringArray{ + pulumi.Sprintf("%s/%s:%s-%s", dc.Registry, name, dc.ImageTag, platform), + }, + }) + if err != nil { + return nil, fmt.Errorf("failed to build %s for %s: %w", name, platform, err) + } + sources = append(sources, img.Ref) + images = append(images, img) + } + + idx, err := dockerbuild.NewIndex(ctx, name, &dockerbuild.IndexArgs{ + Sources: sources, + Tag: pulumi.Sprintf("%s/%s:%s", dc.Registry, name, dc.ImageTag), + Push: pulumi.Bool(true), + Registry: dockerbuild.RegistryArgs{ + Address: pulumi.String(dc.Registry), + Username: dc.RegistryAuth[0].(dockerbuild.RegistryArgs).Username, + Password: dc.RegistryAuth[0].(dockerbuild.RegistryArgs).Password, + }, + }) + if err != nil { + return nil, fmt.Errorf("failed to create index for %s: %w", name, err) + } + + digest := idx.Ref.ApplyT(func(ref string) string { + if i := strings.Index(ref, "@"); i >= 0 { + return ref[i+1:] + } + return ref + }).(pulumi.StringOutput) + + return &multiArchImage{ + Index: idx, + Images: images, + Ref: idx.Ref, + Digest: digest, + }, nil +} + +func getConfigMap(cfg *config.Config, key string) pulumi.Map { + var obj map[string]any + if err := cfg.GetObject(key, &obj); err != nil || obj == nil { + return pulumi.Map{} + } + return pulumi.ToMap(obj) +} + +func getConfigArray(cfg *config.Config, key string) pulumi.Array { + var arr []map[string]any + if err := cfg.GetObject(key, &arr); err != nil || arr == nil { + return pulumi.Array{} + } + result := make(pulumi.Array, len(arr)) + for i, v := range arr { + result[i] = pulumi.ToMap(v) + } + return result +} + +func getImagePullSecrets(cfg *config.Config) pulumi.Array { + var secrets []map[string]any + if err := cfg.GetObject("image-pull-secrets", &secrets); err != nil || len(secrets) == 0 { + return pulumi.Array{} + } + var result pulumi.Array + for _, s := range secrets { + if name, ok := s["name"].(string); ok && name != "" { + result = append(result, pulumi.Map{ + "name": pulumi.String(name), + }) + } + } + return result +} + +func getConfigObject(cfg *config.Config, key string, basePath string) (map[string]any, error) { + var configObj map[string]any + if err := cfg.GetObject(key, &configObj); err != nil { + return nil, fmt.Errorf("failed to get config object %s: %w", key, err) + } + + if filePath, ok := configObj["file"].(string); ok { + fullPath := filepath.Join(basePath, filePath) + data, err := os.ReadFile(fullPath) + if err != nil { + return nil, fmt.Errorf("failed to read values file %s: %w", fullPath, err) + } + + var result map[string]any + if err := yaml.Unmarshal(data, &result); err != nil { + return nil, fmt.Errorf("failed to parse YAML file %s: %w", fullPath, err) + } + + return result, nil + } + + return configObj, nil +} diff --git a/deployment/operator/main.go b/deployment/operator/main.go new file mode 100644 index 000000000..5ae98fec4 --- /dev/null +++ b/deployment/operator/main.go @@ -0,0 +1,124 @@ +package main + +import ( + "fmt" + "path/filepath" + "strings" + + k8syaml "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/yaml" + "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/helm/v3" + "github.com/pulumi/pulumi/sdk/v3/go/pulumi" + "github.com/pulumi/pulumi/sdk/v3/go/pulumi/config" +) + +func main() { + pulumi.Run(func(ctx *pulumi.Context) error { + cfg := config.New(ctx, "") + + k8s, err := newK8sSetup(ctx, cfg) + if err != nil { + return err + } + namespace := k8s.Namespace + k8sProvider := k8s.Provider + + dc := newDockerConfig(ctx, cfg) + + // Build operator image + operatorImage, err := dc.buildImage(ctx, "formancehq/operator", "../..", "../../Dockerfile") + if err != nil { + return fmt.Errorf("failed to build operator image: %w", err) + } + + // Apply CRDs + crdFiles, err := filepath.Glob(filepath.Join("..", "..", "config", "crd", "bases", "*.yaml")) + if err != nil { + return fmt.Errorf("failed to glob CRD files: %w", err) + } + + var crds []pulumi.Resource + for _, crdFile := range crdFiles { + name := strings.TrimSuffix(filepath.Base(crdFile), filepath.Ext(crdFile)) + crd, crdErr := k8syaml.NewConfigFile(ctx, name+"-crd", &k8syaml.ConfigFileArgs{ + File: crdFile, + }, pulumi.Provider(k8sProvider)) + if crdErr != nil { + return fmt.Errorf("failed to apply CRD %s: %w", name, crdErr) + } + crds = append(crds, crd) + } + + // Operator configuration + region := cfg.Get("operator-region") + if region == "" { + region = "eu-west-1" + } + env := cfg.Get("operator-env") + if env == "" { + env = "staging" + } + + // Licence configuration + licenceValues := pulumi.Map{} + licenceToken := cfg.Get("licence-token") + if licenceToken != "" { + licenceIssuer := cfg.Get("licence-issuer") + if licenceIssuer == "" { + licenceIssuer = "https://license.formance.cloud/keys" + } + licenceValues = pulumi.Map{ + "createSecret": pulumi.Bool(true), + "token": pulumi.String(licenceToken), + "issuer": pulumi.String(licenceIssuer), + } + } else { + licenceValues = pulumi.Map{ + "createSecret": pulumi.Bool(false), + } + } + + // Deploy operator via Helm + operatorChartPath := filepath.Join("..", "..", "helm", "operator") + + operatorRelease, err := helm.NewRelease(ctx, "formance-operator", &helm.ReleaseArgs{ + Name: pulumi.String("formance-operator"), + Chart: pulumi.String(operatorChartPath), + Namespace: namespace.Metadata.Name(), + Values: pulumi.Map{ + "operator-crds": pulumi.Map{ + "create": pulumi.Bool(false), + }, + "image": pulumi.Map{ + "repository": pulumi.Sprintf("%s/formancehq/operator", dc.PullRegistry), + "tag": pulumi.Sprintf("latest@%s", operatorImage.Digest), + }, + "imagePullSecrets": getImagePullSecrets(cfg), + "global": pulumi.Map{ + "licence": licenceValues, + }, + "operator": pulumi.Map{ + "region": pulumi.String(region), + "env": pulumi.String(env), + "dev": pulumi.Bool(getConfigBool(cfg, "operator-dev", false)), + "enableLeaderElection": pulumi.Bool(true), + }, + "nodeSelector": getConfigMap(cfg, "node-selector"), + "tolerations": getConfigArray(cfg, "tolerations"), + }, + ForceUpdate: pulumi.Bool(true), + }, + pulumi.DependsOn(append([]pulumi.Resource{namespace, operatorImage.Resource()}, crds...)), + pulumi.Provider(k8sProvider), + ) + if err != nil { + return fmt.Errorf("failed to deploy operator: %w", err) + } + + // Exports + ctx.Export("namespace", namespace.Metadata.Name()) + ctx.Export("operatorImage", pulumi.Sprintf("%s/formancehq/operator:latest@%s", dc.PullRegistry, operatorImage.Digest)) + ctx.Export("operatorRelease", operatorRelease.Name) + + return nil + }) +} From c0fdbff965fd45264fba5f571e7d8efc4f3895d5 Mon Sep 17 00:00:00 2001 From: Geoffrey Ragot Date: Thu, 28 May 2026 17:27:59 +0200 Subject: [PATCH 04/30] refactor(pulumi): take namespace as parameter instead of creating it The Helm release creates the namespace itself via createNamespace. Defaults to formance-system. Co-Authored-By: Claude Opus 4.6 (1M context) --- deployment/operator/helpers.go | 28 ++-------------------------- deployment/operator/main.go | 16 ++++++++++------ 2 files changed, 12 insertions(+), 32 deletions(-) diff --git a/deployment/operator/helpers.go b/deployment/operator/helpers.go index 480d61281..44effc7eb 100644 --- a/deployment/operator/helpers.go +++ b/deployment/operator/helpers.go @@ -10,8 +10,6 @@ import ( "github.com/pulumi/pulumi-docker-build/sdk/go/dockerbuild" "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes" - v1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/core/v1" - metav1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/meta/v1" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" "github.com/pulumi/pulumi/sdk/v3/go/pulumi/config" "gopkg.in/yaml.v3" @@ -52,19 +50,9 @@ func getConfigBool(cfg *config.Config, key string, fallback bool) bool { return fallback } -type k8sSetup struct { - Provider pulumi.ProviderResource - Namespace *v1.Namespace -} - -func newK8sSetup(ctx *pulumi.Context, cfg *config.Config) (*k8sSetup, error) { +func newK8sProvider(ctx *pulumi.Context, cfg *config.Config) (pulumi.ProviderResource, error) { kubeContext := cfg.Require("k8s-context") - namespaceName := cfg.Get("namespace") - if namespaceName == "" { - namespaceName = ctx.Stack() - } - k8sProvider, err := kubernetes.NewProvider(ctx, "k8s", &kubernetes.ProviderArgs{ Context: pulumi.StringPtr(kubeContext), }) @@ -72,19 +60,7 @@ func newK8sSetup(ctx *pulumi.Context, cfg *config.Config) (*k8sSetup, error) { return nil, fmt.Errorf("failed to create k8s provider: %w", err) } - namespace, err := v1.NewNamespace(ctx, "namespace", &v1.NamespaceArgs{ - Metadata: &metav1.ObjectMetaArgs{ - Name: pulumi.String(namespaceName), - }, - }, pulumi.Provider(k8sProvider), pulumi.RetainOnDelete(true)) - if err != nil { - return nil, fmt.Errorf("failed to create namespace: %w", err) - } - - return &k8sSetup{ - Provider: k8sProvider, - Namespace: namespace, - }, nil + return k8sProvider, nil } type dockerConfig struct { diff --git a/deployment/operator/main.go b/deployment/operator/main.go index 5ae98fec4..0c8ddb1c0 100644 --- a/deployment/operator/main.go +++ b/deployment/operator/main.go @@ -15,12 +15,15 @@ func main() { pulumi.Run(func(ctx *pulumi.Context) error { cfg := config.New(ctx, "") - k8s, err := newK8sSetup(ctx, cfg) + k8sProvider, err := newK8sProvider(ctx, cfg) if err != nil { return err } - namespace := k8s.Namespace - k8sProvider := k8s.Provider + + namespace := cfg.Get("namespace") + if namespace == "" { + namespace = "formance-system" + } dc := newDockerConfig(ctx, cfg) @@ -83,7 +86,8 @@ func main() { operatorRelease, err := helm.NewRelease(ctx, "formance-operator", &helm.ReleaseArgs{ Name: pulumi.String("formance-operator"), Chart: pulumi.String(operatorChartPath), - Namespace: namespace.Metadata.Name(), + Namespace: pulumi.String(namespace), + CreateNamespace: pulumi.Bool(true), Values: pulumi.Map{ "operator-crds": pulumi.Map{ "create": pulumi.Bool(false), @@ -107,7 +111,7 @@ func main() { }, ForceUpdate: pulumi.Bool(true), }, - pulumi.DependsOn(append([]pulumi.Resource{namespace, operatorImage.Resource()}, crds...)), + pulumi.DependsOn(append([]pulumi.Resource{operatorImage.Resource()}, crds...)), pulumi.Provider(k8sProvider), ) if err != nil { @@ -115,7 +119,7 @@ func main() { } // Exports - ctx.Export("namespace", namespace.Metadata.Name()) + ctx.Export("namespace", pulumi.String(namespace)) ctx.Export("operatorImage", pulumi.Sprintf("%s/formancehq/operator:latest@%s", dc.PullRegistry, operatorImage.Digest)) ctx.Export("operatorRelease", operatorRelease.Name) From 6aa2687ebd98684755eee2e3f293deeda07d19c6 Mon Sep 17 00:00:00 2001 From: Geoffrey Ragot Date: Thu, 28 May 2026 17:40:08 +0200 Subject: [PATCH 05/30] fix: add -buildvcs=false to Dockerfile go build The Docker build context may not include .git, causing go build to fail with "error obtaining VCS status". Disable VCS stamping since version info is already passed via ldflags. Co-Authored-By: Claude Opus 4.6 (1M context) --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ac502b218..6733ff7d3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,7 @@ RUN set -e; \ LICENCE_PUBLIC_KEY="$(printf '%s' "$LICENCE_PUBLIC_KEY_B64" | base64 -d)"; \ LDFLAGS="${LDFLAGS} -X 'github.com/formancehq/go-libs/v5/pkg/authn/licence.formancePublicKey=${LICENCE_PUBLIC_KEY}'"; \ fi; \ - CGO_ENABLED=0 go build -o /usr/bin/operator -ldflags="${LDFLAGS}" . + CGO_ENABLED=0 go build -buildvcs=false -o /usr/bin/operator -ldflags="${LDFLAGS}" . FROM alpine:3.20 From 7f56f8c7723f5c4aac18b54eafe6b21973774c44 Mon Sep 17 00:00:00 2001 From: Maxence Maireaux Date: Tue, 14 Jul 2026 18:20:20 +0200 Subject: [PATCH 06/30] feat(ledger): delegate v3 provisioning to ledger operator --- config/rbac/role.yaml | 12 + docs/04-Modules/03-Ledger.md | 40 ++- .../09-Configuration reference/01-Settings.md | 1 + .../settings.catalog.json | 8 + ..._v1_clusterrole_formance-manager-role.yaml | 12 + internal/resources/ledgers/init.go | 22 ++ internal/resources/ledgers/v3.go | 320 ++++++++++++++++++ internal/resources/ledgers/v3_test.go | 29 ++ internal/tests/application_test.go | 2 +- internal/tests/auth_scopes_settings_test.go | 2 +- .../crds/ledger.formance.com_clusters.yaml | 34 ++ internal/tests/jobs_controller_test.go | 2 +- internal/tests/ledger_controller_test.go | 2 +- internal/tests/ledger_v3_controller_test.go | 194 +++++++++++ .../tests/orchestration_controller_test.go | 2 +- internal/tests/registries_test.go | 2 +- .../tests/transactionplane_controller_test.go | 6 +- internal/tests/wallets_controller_test.go | 2 +- 18 files changed, 681 insertions(+), 11 deletions(-) create mode 100644 internal/resources/ledgers/v3.go create mode 100644 internal/resources/ledgers/v3_test.go create mode 100644 internal/tests/crds/ledger.formance.com_clusters.yaml create mode 100644 internal/tests/ledger_v3_controller_test.go diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index bf8876117..ab2aa150b 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -189,6 +189,18 @@ rules: - patch - update - watch +- apiGroups: + - ledger.formance.com + resources: + - clusters + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - networking.k8s.io resources: diff --git a/docs/04-Modules/03-Ledger.md b/docs/04-Modules/03-Ledger.md index 2a78816e0..b31499b18 100644 --- a/docs/04-Modules/03-Ledger.md +++ b/docs/04-Modules/03-Ledger.md @@ -2,10 +2,15 @@ Formance Ledger is a real-time money tracking microservice that lets you model a ## Requirements -Formance Ledger requires: +Ledger versions up to and including `v3.0.0-alpha` require: + - **PostgreSQL**: See configuration guide [here](../05-Infrastructure%20services/01-PostgreSQL.md). - (Optional) **Broker**: See configuration guide [here](../05-Infrastructure%20services/02-Message%20broker.md). +Ledger versions newer than `v3.0.0-alpha` require the Ledger Operator and its +`ledger.formance.com/v1alpha1` CRDs to be installed in the cluster. They use the +Ledger v3 native storage and do not require a PostgreSQL `Database` resource. + ## Ledger Object :::info @@ -21,6 +26,39 @@ spec: stack: formance-dev ``` +## Ledger v3 delegation + +When the stack version is strictly newer than `v3.0.0-alpha`, the Formance +Operator delegates Ledger provisioning to the Ledger Operator. It creates a +`ledger.formance.com/v1alpha1` `Cluster` with the same name and namespace as the +stack instead of creating the legacy Ledger Deployments, Database, migration +jobs, and CronJobs. + +The Ledger Operator must be installed before the Formance Operator starts so +that the latter can watch `Cluster` resources. If the CRD is unavailable, the +Ledger remains pending and no legacy resources are created. + +Automatic in-place migration is intentionally not supported. If legacy Ledger +Deployments or a Database already exist when switching a stack to v3, the +Ledger reports that an explicit migration is required and does not create the +v3 `Cluster`. The reverse transition is guarded in the same way: legacy +resources are not created while a v3 `Cluster` still exists. + +The v3 cluster size defaults to three replicas and can be configured per stack: + +```yaml +apiVersion: formance.com/v1beta1 +kind: Settings +metadata: + name: ledger-v3-replicas +spec: + stacks: ["formance-dev"] + key: module.ledger.v3.replicas + value: "3" +``` + +The replica count must be a positive odd number. + ## Settings (v2.4+) ### Schema Enforcement Mode diff --git a/docs/09-Configuration reference/01-Settings.md b/docs/09-Configuration reference/01-Settings.md index 8a3ec2044..af7c5276a 100644 --- a/docs/09-Configuration reference/01-Settings.md +++ b/docs/09-Configuration reference/01-Settings.md @@ -33,6 +33,7 @@ While we have some basic types (string, number, bool ...), we also have some com | ledger.worker.async-block-hasher | Map | max-block-size=1000, schedule="0 * * * * *" | Configure async block hasher for the Ledger worker (v2.3+). Fields: `max-block-size`, `schedule` | | ledger.worker.bucket-cleanup | Map | retention-period=720h, schedule="0 0 * * *" | Configure bucket cleanup for the Ledger worker (v2.4+). Fields: `retention-period`, `schedule` | | ledger.worker.pipelines | Map | pull-interval=5s, push-retry-period=10s, sync-period=1m, logs-page-size=100 | Configure pipelines for the Ledger worker (v2.3+). Fields: `pull-interval`, `push-retry-period`, `sync-period`, `logs-page-size` | +| module.ledger.v3.replicas | Int | 3 | Number of Ledger v3 replicas. Must be a positive odd number. | | payments.encryption-key | string | | Payments data encryption key | | payments.clear-temporal | bool | true | Whether to clean up Temporal workflows when the Payments module is deleted (default: true). Set to `false` to skip the cleanup, e.g. when migrating a stack across clusters. | | payments.worker.temporal-max-concurrent-workflow-task-pollers | Int | | Payments worker max concurrent workflow task pollers configuration | diff --git a/docs/09-Configuration reference/settings.catalog.json b/docs/09-Configuration reference/settings.catalog.json index 5d4cd3895..b7a93c556 100644 --- a/docs/09-Configuration reference/settings.catalog.json +++ b/docs/09-Configuration reference/settings.catalog.json @@ -496,6 +496,14 @@ "internal/resources/applications/application.go:392" ] }, + { + "key": "module.ledger.v3.replicas", + "valueType": "int32", + "default": "3", + "sources": [ + "internal/resources/ledgers/v3.go:182" + ] + }, { "key": "modules.\u003cmodule-name\u003e.database.connection-pool", "valueType": "object", diff --git a/helm/operator/templates/gen/rbac.authorization.k8s.io_v1_clusterrole_formance-manager-role.yaml b/helm/operator/templates/gen/rbac.authorization.k8s.io_v1_clusterrole_formance-manager-role.yaml index 16f6c6e34..40f546122 100644 --- a/helm/operator/templates/gen/rbac.authorization.k8s.io_v1_clusterrole_formance-manager-role.yaml +++ b/helm/operator/templates/gen/rbac.authorization.k8s.io_v1_clusterrole_formance-manager-role.yaml @@ -188,6 +188,18 @@ rules: - patch - update - watch +- apiGroups: + - ledger.formance.com + resources: + - clusters + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - networking.k8s.io resources: diff --git a/internal/resources/ledgers/init.go b/internal/resources/ledgers/init.go index 3282bf337..15b3455d0 100644 --- a/internal/resources/ledgers/init.go +++ b/internal/resources/ledgers/init.go @@ -23,6 +23,7 @@ import ( appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/formancehq/operator/v3/api/formance.com/v1beta1" . "github.com/formancehq/operator/v3/internal/core" @@ -39,6 +40,26 @@ import ( //+kubebuilder:rbac:groups=batch,resources=cronjobs,verbs=get;list;watch;create;update;patch;delete func Reconcile(ctx Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, version string) error { + if isLedgerV3(version) { + return reconcileV3(ctx, stack, ledger, version) + } + + if ledgerV3ClusterAvailable { + exists, err := v3ClusterExists(ctx, stack) + if err != nil { + return err + } + if exists { + setLedgerV3Condition(ledger, metav1.ConditionFalse, "MigrationRequired", "A Ledger v3 Cluster exists; an explicit v3 to v2 migration is required") + return NewPendingError().WithMessage("migration required before switching Ledger from v3 to v2") + } + } + + ledger.GetConditions().Delete(v1beta1.ConditionTypeMatch(ledgerV3ClusterReadyCondition)) + return reconcileLegacy(ctx, stack, ledger, version) +} + +func reconcileLegacy(ctx Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, version string) error { database, err := databases.Create(ctx, stack, ledger) if err != nil { return err @@ -112,6 +133,7 @@ func init() { WithOwn[*v1beta1.Ledger](&batchv1.CronJob{}), WithOwn[*v1beta1.Ledger](&corev1.ConfigMap{}), WithOwn[*v1beta1.Ledger](&v1beta1.BenthosStream{}), + withLedgerV3ClusterWatch(), WithWatchSettings[*v1beta1.Ledger](), WithWatchDependency[*v1beta1.Ledger](&v1beta1.Search{}), brokertopics.Watch[*v1beta1.Ledger]("ledger"), diff --git a/internal/resources/ledgers/v3.go b/internal/resources/ledgers/v3.go new file mode 100644 index 000000000..152d36e45 --- /dev/null +++ b/internal/resources/ledgers/v3.go @@ -0,0 +1,320 @@ +package ledgers + +import ( + "fmt" + "strings" + + "golang.org/x/mod/semver" + appsv1 "k8s.io/api/apps/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" + + "github.com/formancehq/operator/v3/api/formance.com/v1beta1" + "github.com/formancehq/operator/v3/internal/core" + "github.com/formancehq/operator/v3/internal/resources/auths" + "github.com/formancehq/operator/v3/internal/resources/gatewayhttpapis" + "github.com/formancehq/operator/v3/internal/resources/registries" + "github.com/formancehq/operator/v3/internal/resources/settings" +) + +const ( + ledgerV3Threshold = "v3.0.0-alpha" + ledgerV3ClusterReadyCondition = "LedgerV3ClusterReady" +) + +var ( + ledgerV3ClusterGVK = schema.GroupVersionKind{ + Group: "ledger.formance.com", + Version: "v1alpha1", + Kind: "Cluster", + } + ledgerV3ClusterAvailable bool +) + +//+kubebuilder:rbac:groups=ledger.formance.com,resources=clusters,verbs=get;list;watch;create;update;patch;delete + +func isLedgerV3(version string) bool { + normalizedVersion := version + if !strings.HasPrefix(normalizedVersion, "v") { + normalizedVersion = "v" + normalizedVersion + } + return semver.IsValid(normalizedVersion) && semver.Compare(normalizedVersion, ledgerV3Threshold) > 0 +} + +func newV3Cluster() *unstructured.Unstructured { + cluster := &unstructured.Unstructured{} + cluster.SetGroupVersionKind(ledgerV3ClusterGVK) + return cluster +} + +func withLedgerV3ClusterWatch() core.ReconcilerOption[*v1beta1.Ledger] { + return func(options *core.ReconcilerOptions[*v1beta1.Ledger]) { + options.Raws = append(options.Raws, func(ctx core.Context, b *builder.Builder) error { + crds := &apiextensionsv1.CustomResourceDefinitionList{} + if err := ctx.GetAPIReader().List(ctx, crds); err != nil { + return err + } + + for _, crd := range crds.Items { + if crd.Spec.Group != ledgerV3ClusterGVK.Group || crd.Spec.Names.Kind != ledgerV3ClusterGVK.Kind { + continue + } + for _, version := range crd.Spec.Versions { + if version.Name == ledgerV3ClusterGVK.Version && version.Served { + ledgerV3ClusterAvailable = true + cluster := newV3Cluster() + options.Owns[cluster] = nil + b.Owns(cluster) + log.FromContext(ctx).Info("Ledger v3 Cluster CRD is available", "gvk", ledgerV3ClusterGVK) + return nil + } + } + } + + log.FromContext(ctx).Info("Ledger v3 Cluster CRD is not available", "gvk", ledgerV3ClusterGVK) + return nil + }) + } +} + +func reconcileV3(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, version string) error { + if !ledgerV3ClusterAvailable { + setLedgerV3Condition(ledger, metav1.ConditionFalse, "OperatorUnavailable", "Ledger v3 Cluster CRD is not installed") + return core.NewPendingError().WithMessage("Ledger v3 operator unavailable: Cluster CRD is not installed") + } + + cluster := newV3Cluster() + clusterKey := types.NamespacedName{Namespace: stack.Name, Name: stack.Name} + err := ctx.GetClient().Get(ctx, clusterKey, cluster) + if apierrors.IsNotFound(err) { + hasLegacyResources, err := legacyLedgerResourcesExist(ctx, stack) + if err != nil { + return err + } + if hasLegacyResources { + setLedgerV3Condition(ledger, metav1.ConditionFalse, "MigrationRequired", "Legacy Ledger resources exist; an explicit v2 to v3 migration is required") + return core.NewPendingError().WithMessage("migration required before switching Ledger from v2 to v3") + } + } else if err != nil { + return err + } + + if err := gatewayhttpapis.Create(ctx, ledger, gatewayhttpapis.WithHealthCheckEndpoint("livez")); err != nil { + return err + } + + cluster, err = createOrUpdateV3Cluster(ctx, stack, ledger, version) + if err != nil { + setLedgerV3Condition(ledger, metav1.ConditionFalse, "ReconcileFailed", err.Error()) + return err + } + + ready, message, err := isV3ClusterReady(cluster) + if err != nil { + return err + } + if !ready { + setLedgerV3Condition(ledger, metav1.ConditionFalse, "Pending", message) + return core.NewPendingError().WithMessage("Ledger v3 Cluster is not ready: %s", message) + } + + setLedgerV3Condition(ledger, metav1.ConditionTrue, "Running", message) + return nil +} + +func setLedgerV3Condition(ledger *v1beta1.Ledger, status metav1.ConditionStatus, reason, message string) { + ledger.GetConditions().AppendOrReplace(v1beta1.Condition{ + Type: ledgerV3ClusterReadyCondition, + Status: status, + ObservedGeneration: ledger.GetGeneration(), + LastTransitionTime: metav1.Now(), + Reason: reason, + Message: message, + }, v1beta1.ConditionTypeMatch(ledgerV3ClusterReadyCondition)) +} + +func legacyLedgerResourcesExist(ctx core.Context, stack *v1beta1.Stack) (bool, error) { + for _, name := range []string{"ledger", "ledger-worker", "ledger-write", "ledger-read", "ledger-gateway"} { + deployment := &appsv1.Deployment{} + err := ctx.GetClient().Get(ctx, types.NamespacedName{Namespace: stack.Name, Name: name}, deployment) + if err == nil { + return true, nil + } + if !apierrors.IsNotFound(err) { + return false, err + } + } + + database := &v1beta1.Database{} + err := ctx.GetClient().Get(ctx, types.NamespacedName{Name: core.GetObjectName(stack.Name, "ledger")}, database) + if err == nil { + return true, nil + } + if !apierrors.IsNotFound(err) { + return false, err + } + return false, nil +} + +func v3ClusterExists(ctx core.Context, stack *v1beta1.Stack) (bool, error) { + cluster := newV3Cluster() + err := ctx.GetClient().Get(ctx, types.NamespacedName{Namespace: stack.Name, Name: stack.Name}, cluster) + if apierrors.IsNotFound(err) { + return false, nil + } + return err == nil, err +} + +func createOrUpdateV3Cluster(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, version string) (*unstructured.Unstructured, error) { + image, err := registries.GetFormanceImage(ctx, stack, "ledger", version) + if err != nil { + return nil, err + } + + replicas, err := settings.GetInt32OrDefault(ctx, stack.Name, 3, "module", "ledger", "v3", "replicas") + if err != nil { + return nil, err + } + if replicas < 1 || replicas%2 == 0 { + return nil, fmt.Errorf("module.ledger.v3.replicas must be a positive odd number, got %d", replicas) + } + + extraEnv, err := buildV3ExtraEnv(ctx, stack, ledger) + if err != nil { + return nil, err + } + + serviceAccountName, err := settings.GetAWSServiceAccount(ctx, stack.Name) + if err != nil { + return nil, err + } + + cluster := newV3Cluster() + cluster.SetNamespace(stack.Name) + cluster.SetName(stack.Name) + _, err = controllerutil.CreateOrUpdate(ctx, ctx.GetClient(), cluster, func() error { + labels := cluster.GetLabels() + if labels == nil { + labels = map[string]string{} + } + labels[v1beta1.StackLabel] = stack.Name + cluster.SetLabels(labels) + + if err := controllerutil.SetControllerReference(ledger, cluster, ctx.GetScheme()); err != nil { + return err + } + + if err := unstructured.SetNestedField(cluster.Object, imageRepository(image), "spec", "image", "repository"); err != nil { + return err + } + if err := unstructured.SetNestedField(cluster.Object, image.Version, "spec", "image", "tag"); err != nil { + return err + } + if err := unstructured.SetNestedField(cluster.Object, int64(replicas), "spec", "replicas"); err != nil { + return err + } + if err := unstructured.SetNestedField(cluster.Object, stack.Name, "spec", "clusterID"); err != nil { + return err + } + if err := unstructured.SetNestedField(cluster.Object, stack.Spec.Debug || ledger.Spec.Debug, "spec", "debug"); err != nil { + return err + } + + if len(image.PullSecrets) > 0 { + pullSecrets := make([]any, 0, len(image.PullSecrets)) + for _, pullSecret := range image.PullSecrets { + pullSecrets = append(pullSecrets, map[string]any{"name": pullSecret.Name}) + } + if err := unstructured.SetNestedSlice(cluster.Object, pullSecrets, "spec", "imagePullSecrets"); err != nil { + return err + } + } else { + unstructured.RemoveNestedField(cluster.Object, "spec", "imagePullSecrets") + } + + if len(extraEnv) > 0 { + if err := unstructured.SetNestedSlice(cluster.Object, extraEnv, "spec", "extraEnv"); err != nil { + return err + } + } else { + unstructured.RemoveNestedField(cluster.Object, "spec", "extraEnv") + } + + if serviceAccountName != "" { + if err := unstructured.SetNestedField(cluster.Object, false, "spec", "serviceAccount", "create"); err != nil { + return err + } + if err := unstructured.SetNestedField(cluster.Object, serviceAccountName, "spec", "serviceAccount", "name"); err != nil { + return err + } + } else { + unstructured.RemoveNestedField(cluster.Object, "spec", "serviceAccount") + } + + return nil + }) + return cluster, err +} + +func imageRepository(image *registries.ImageConfiguration) string { + if image.Registry == "" { + return image.Image + } + return strings.TrimSuffix(image.Registry, "/") + "/" + image.Image +} + +func buildV3ExtraEnv(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger) ([]any, error) { + env, err := settings.GetOTELEnvVars(ctx, stack.Name, "ledger", " ") + if err != nil { + return nil, err + } + authEnv, err := auths.ProtectedEnvVars(ctx, stack, "ledger", ledger.Spec.Auth) + if err != nil { + return nil, err + } + env = append(env, authEnv...) + env = append(env, core.GetDevEnvVars(stack, ledger)...) + + ret := make([]any, 0, len(env)) + for i := range env { + value, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&env[i]) + if err != nil { + return nil, err + } + ret = append(ret, value) + } + return ret, nil +} + +func isV3ClusterReady(cluster *unstructured.Unstructured) (bool, string, error) { + phase, _, err := unstructured.NestedString(cluster.Object, "status", "phase") + if err != nil { + return false, "", err + } + readyReplicas, _, err := unstructured.NestedInt64(cluster.Object, "status", "readyReplicas") + if err != nil { + return false, "", err + } + observedGeneration, _, err := unstructured.NestedInt64(cluster.Object, "status", "observedGeneration") + if err != nil { + return false, "", err + } + replicas, found, err := unstructured.NestedInt64(cluster.Object, "spec", "replicas") + if err != nil { + return false, "", err + } + if !found { + replicas = 3 + } + + message := fmt.Sprintf("phase=%s readyReplicas=%d/%d observedGeneration=%d/%d", phase, readyReplicas, replicas, observedGeneration, cluster.GetGeneration()) + return phase == "Running" && readyReplicas == replicas && observedGeneration == cluster.GetGeneration(), message, nil +} diff --git a/internal/resources/ledgers/v3_test.go b/internal/resources/ledgers/v3_test.go new file mode 100644 index 000000000..f2d2f40bb --- /dev/null +++ b/internal/resources/ledgers/v3_test.go @@ -0,0 +1,29 @@ +package ledgers + +import "testing" + +func TestIsLedgerV3(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + version string + want bool + }{ + {name: "legacy release", version: "v2.99.0", want: false}, + {name: "threshold", version: "v3.0.0-alpha", want: false}, + {name: "first alpha release", version: "v3.0.0-alpha.1", want: true}, + {name: "without v prefix", version: "3.0.0-alpha.1", want: true}, + {name: "stable v3", version: "v3.0.0", want: true}, + {name: "development tag remains legacy", version: "develop", want: false}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + if got := isLedgerV3(test.version); got != test.want { + t.Fatalf("isLedgerV3(%q) = %t, want %t", test.version, got, test.want) + } + }) + } +} diff --git a/internal/tests/application_test.go b/internal/tests/application_test.go index 770cfaaf7..0f2bacc0d 100644 --- a/internal/tests/application_test.go +++ b/internal/tests/application_test.go @@ -20,7 +20,7 @@ var _ = Context("When creating a Application", func() { BeforeEach(func() { stack = &v1beta1.Stack{ ObjectMeta: RandObjectMeta(), - Spec: v1beta1.StackSpec{Version: "v99.0.0"}, + Spec: v1beta1.StackSpec{Version: "v2.99.0"}, } settings = []*v1beta1.Settings{ coresettings.New(uuid.NewString(), "postgres.*.uri", "postgresql://localhost", stack.Name), diff --git a/internal/tests/auth_scopes_settings_test.go b/internal/tests/auth_scopes_settings_test.go index 3cb3b4c10..d765bfc11 100644 --- a/internal/tests/auth_scopes_settings_test.go +++ b/internal/tests/auth_scopes_settings_test.go @@ -26,7 +26,7 @@ var _ = Describe("AuthScopesSettings", func() { BeforeEach(func() { stack = &v1beta1.Stack{ ObjectMeta: RandObjectMeta(), - Spec: v1beta1.StackSpec{Version: "v99.0.0"}, + Spec: v1beta1.StackSpec{Version: "v2.99.0"}, } databaseSettings = settings.New(uuid.NewString(), "postgres.*.uri", "postgresql://localhost", stack.Name) auth = &v1beta1.Auth{ diff --git a/internal/tests/crds/ledger.formance.com_clusters.yaml b/internal/tests/crds/ledger.formance.com_clusters.yaml new file mode 100644 index 000000000..ae8a95691 --- /dev/null +++ b/internal/tests/crds/ledger.formance.com_clusters.yaml @@ -0,0 +1,34 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clusters.ledger.formance.com +spec: + group: ledger.formance.com + names: + kind: Cluster + listKind: ClusterList + plural: clusters + singular: cluster + scope: Namespaced + versions: + - name: v1alpha1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + status: + type: object + x-kubernetes-preserve-unknown-fields: true + subresources: + status: {} diff --git a/internal/tests/jobs_controller_test.go b/internal/tests/jobs_controller_test.go index 6b723bdfd..3d30c03d3 100644 --- a/internal/tests/jobs_controller_test.go +++ b/internal/tests/jobs_controller_test.go @@ -41,7 +41,7 @@ var _ = Describe("Job", func() { ObjectMeta: metav1.ObjectMeta{ Name: uuid.NewString()[:8], }, - Spec: v1beta1.StackSpec{Version: "v99.0.0"}, + Spec: v1beta1.StackSpec{Version: "v2.99.0"}, } runAS = &runAs{ user: rand.IntN(65534), diff --git a/internal/tests/ledger_controller_test.go b/internal/tests/ledger_controller_test.go index 84ffb5bb3..91aa4737b 100644 --- a/internal/tests/ledger_controller_test.go +++ b/internal/tests/ledger_controller_test.go @@ -25,7 +25,7 @@ var _ = Describe("LedgerController", func() { BeforeEach(func() { stack = &v1beta1.Stack{ ObjectMeta: RandObjectMeta(), - Spec: v1beta1.StackSpec{Version: "v99.0.0"}, + Spec: v1beta1.StackSpec{Version: "v2.99.0"}, } databaseSettings = settings.New(uuid.NewString(), "postgres.*.uri", "postgresql://localhost", stack.Name) ledger = &v1beta1.Ledger{ diff --git a/internal/tests/ledger_v3_controller_test.go b/internal/tests/ledger_v3_controller_test.go new file mode 100644 index 000000000..16fd84456 --- /dev/null +++ b/internal/tests/ledger_v3_controller_test.go @@ -0,0 +1,194 @@ +package tests_test + +import ( + "github.com/google/uuid" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/formancehq/operator/v3/api/formance.com/v1beta1" + "github.com/formancehq/operator/v3/internal/core" + "github.com/formancehq/operator/v3/internal/resources/settings" + . "github.com/formancehq/operator/v3/internal/tests/internal" +) + +var ledgerV3ClusterGVK = schema.GroupVersionKind{ + Group: "ledger.formance.com", + Version: "v1alpha1", + Kind: "Cluster", +} + +func newLedgerV3Cluster() *unstructured.Unstructured { + cluster := &unstructured.Unstructured{} + cluster.SetGroupVersionKind(ledgerV3ClusterGVK) + return cluster +} + +var _ = Describe("Ledger v3 controller", func() { + var ( + stack *v1beta1.Stack + ledger *v1beta1.Ledger + ) + + BeforeEach(func() { + stack = &v1beta1.Stack{ + ObjectMeta: RandObjectMeta(), + Spec: v1beta1.StackSpec{Version: "v3.0.0-alpha.1"}, + } + ledger = &v1beta1.Ledger{ + ObjectMeta: RandObjectMeta(), + Spec: v1beta1.LedgerSpec{ + StackDependency: v1beta1.StackDependency{Stack: stack.Name}, + }, + } + }) + + JustBeforeEach(func() { + Expect(Create(stack)).To(Succeed()) + Expect(Create(ledger)).To(Succeed()) + }) + + AfterEach(func() { + Expect(client.IgnoreNotFound(Delete(ledger))).To(Succeed()) + Expect(client.IgnoreNotFound(Delete(stack))).To(Succeed()) + }) + + It("delegates runtime provisioning to a Ledger v3 Cluster", func() { + cluster := newLedgerV3Cluster() + Eventually(func(g Gomega) error { + err := Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name}, cluster) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(cluster).To(BeControlledBy(ledger)) + return nil + }).Should(Succeed()) + + repository, found, err := unstructured.NestedString(cluster.Object, "spec", "image", "repository") + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(repository).To(Equal("ghcr.io/formancehq/ledger")) + + tag, found, err := unstructured.NestedString(cluster.Object, "spec", "image", "tag") + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(tag).To(Equal("v3.0.0-alpha.1")) + + deployment := &appsv1.Deployment{} + Consistently(func() bool { + err := Get(core.GetNamespacedResourceName(stack.Name, "ledger"), deployment) + return apierrors.IsNotFound(err) + }).Should(BeTrue()) + + database := &v1beta1.Database{} + Consistently(func() bool { + err := Get(core.GetResourceName(core.GetObjectName(stack.Name, "ledger")), database) + return apierrors.IsNotFound(err) + }).Should(BeTrue()) + }) + + It("mirrors Cluster readiness on the Formance Ledger", func() { + cluster := newLedgerV3Cluster() + Eventually(func() error { + return Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name}, cluster) + }).Should(Succeed()) + + Eventually(func(g Gomega) error { + g.Expect(Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name}, cluster)).To(Succeed()) + cluster.Object["status"] = map[string]any{ + "phase": "Running", + "readyReplicas": int64(3), + "observedGeneration": cluster.GetGeneration(), + } + return TestContext().GetClient().Status().Update(TestContext(), cluster) + }).Should(Succeed()) + + Eventually(func(g Gomega) bool { + g.Expect(LoadResource("", ledger.Name, ledger)).To(Succeed()) + return ledger.Status.Ready + }).Should(BeTrue()) + }) + + It("removes the Cluster when the stack is disabled", func() { + cluster := newLedgerV3Cluster() + Eventually(func() error { + return Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name}, cluster) + }).Should(Succeed()) + + Expect(LoadResource("", stack.Name, stack)).To(Succeed()) + patch := client.MergeFrom(stack.DeepCopy()) + stack.Spec.Disabled = true + Expect(Patch(stack, patch)).To(Succeed()) + + Eventually(func() bool { + err := Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name}, cluster) + return apierrors.IsNotFound(err) + }).Should(BeTrue()) + }) + + It("requires an explicit migration before switching back to v2", func() { + cluster := newLedgerV3Cluster() + Eventually(func() error { + return Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name}, cluster) + }).Should(Succeed()) + + Expect(LoadResource("", ledger.Name, ledger)).To(Succeed()) + patch := client.MergeFrom(ledger.DeepCopy()) + ledger.Spec.Version = "v2.99.0" + Expect(Patch(ledger, patch)).To(Succeed()) + + Consistently(func() bool { + err := Get(core.GetNamespacedResourceName(stack.Name, "ledger"), &appsv1.Deployment{}) + return apierrors.IsNotFound(err) + }).Should(BeTrue()) + + Consistently(func() bool { + err := Get(core.GetResourceName(core.GetObjectName(stack.Name, "ledger")), &v1beta1.Database{}) + return apierrors.IsNotFound(err) + }).Should(BeTrue()) + + Eventually(func(g Gomega) string { + g.Expect(LoadResource("", ledger.Name, ledger)).To(Succeed()) + return ledger.Status.Info + }).Should(ContainSubstring("migration required")) + }) + + Context("when legacy resources already exist", func() { + var databaseSettings *v1beta1.Settings + + BeforeEach(func() { + stack.Spec.Version = "v2.99.0" + databaseSettings = settings.New(uuid.NewString(), "postgres.*.uri", "postgresql://localhost", stack.Name) + Expect(Create(databaseSettings)).To(Succeed()) + }) + + AfterEach(func() { + Expect(client.IgnoreNotFound(Delete(databaseSettings))).To(Succeed()) + }) + + It("requires an explicit migration before switching to v3", func() { + Eventually(func() error { + return Get(core.GetNamespacedResourceName(stack.Name, "ledger"), &appsv1.Deployment{}) + }).Should(Succeed()) + + Expect(LoadResource("", ledger.Name, ledger)).To(Succeed()) + patch := client.MergeFrom(ledger.DeepCopy()) + ledger.Spec.Version = "v3.0.0-alpha.1" + Expect(Patch(ledger, patch)).To(Succeed()) + + cluster := newLedgerV3Cluster() + Consistently(func() bool { + err := Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name}, cluster) + return apierrors.IsNotFound(err) + }).Should(BeTrue()) + + Eventually(func(g Gomega) string { + g.Expect(LoadResource("", ledger.Name, ledger)).To(Succeed()) + return ledger.Status.Info + }).Should(ContainSubstring("migration required")) + }) + }) +}) diff --git a/internal/tests/orchestration_controller_test.go b/internal/tests/orchestration_controller_test.go index b52dcb3d9..0bdf851c4 100644 --- a/internal/tests/orchestration_controller_test.go +++ b/internal/tests/orchestration_controller_test.go @@ -29,7 +29,7 @@ var _ = Describe("OrchestrationController", func() { BeforeEach(func() { stack = &v1beta1.Stack{ ObjectMeta: RandObjectMeta(), - Spec: v1beta1.StackSpec{Version: "v99.0.0"}, + Spec: v1beta1.StackSpec{Version: "v2.99.0"}, } databaseSettings = settings.New(uuid.NewString(), "postgres.*.uri", "postgresql://localhost", stack.Name) brokerDSNSettings = settings.New(uuid.NewString(), "broker.dsn", "nats://localhost:1234", stack.Name) diff --git a/internal/tests/registries_test.go b/internal/tests/registries_test.go index c6890ba2d..881f10d7a 100644 --- a/internal/tests/registries_test.go +++ b/internal/tests/registries_test.go @@ -22,7 +22,7 @@ var _ = Describe("Registries", func() { BeforeEach(func() { stack = &v1beta1.Stack{ ObjectMeta: RandObjectMeta(), - Spec: v1beta1.StackSpec{Version: "v99.0.0"}, + Spec: v1beta1.StackSpec{Version: "v2.99.0"}, } databaseSettings = settings.New(uuid.NewString(), "postgres.*.uri", "postgresql://localhost", stack.Name) ledger = &v1beta1.Ledger{ diff --git a/internal/tests/transactionplane_controller_test.go b/internal/tests/transactionplane_controller_test.go index c79012f08..c1ddd0601 100644 --- a/internal/tests/transactionplane_controller_test.go +++ b/internal/tests/transactionplane_controller_test.go @@ -28,7 +28,7 @@ var _ = Describe("TransactionPlaneController", func() { BeforeEach(func() { stack = &v1beta1.Stack{ ObjectMeta: RandObjectMeta(), - Spec: v1beta1.StackSpec{Version: "v99.0.0"}, + Spec: v1beta1.StackSpec{Version: "v2.99.0"}, } databaseSettings = settings.New(uuid.NewString(), "postgres.*.uri", "postgresql://localhost", stack.Name) brokerDSNSettings = settings.New(uuid.NewString(), "broker.dsn", "nats://localhost:1234", stack.Name) @@ -152,7 +152,7 @@ var _ = Describe("TransactionPlaneController", func() { BeforeEach(func() { stack = &v1beta1.Stack{ ObjectMeta: RandObjectMeta(), - Spec: v1beta1.StackSpec{Version: "v99.0.0"}, + Spec: v1beta1.StackSpec{Version: "v2.99.0"}, } databaseSettings = settings.New(uuid.NewString(), "postgres.*.uri", "postgresql://localhost", stack.Name) brokerDSNSettings = settings.New(uuid.NewString(), "broker.dsn", "nats://localhost:1234", stack.Name) @@ -253,7 +253,7 @@ var _ = Describe("TransactionPlaneController", func() { BeforeEach(func() { stack = &v1beta1.Stack{ ObjectMeta: RandObjectMeta(), - Spec: v1beta1.StackSpec{Version: "v99.0.0"}, + Spec: v1beta1.StackSpec{Version: "v2.99.0"}, } databaseSettings = settings.New(uuid.NewString(), "postgres.*.uri", "postgresql://localhost", stack.Name) brokerDSNSettings = settings.New(uuid.NewString(), "broker.dsn", "nats://localhost:1234", stack.Name) diff --git a/internal/tests/wallets_controller_test.go b/internal/tests/wallets_controller_test.go index dbb4b6d45..d3396785a 100644 --- a/internal/tests/wallets_controller_test.go +++ b/internal/tests/wallets_controller_test.go @@ -27,7 +27,7 @@ var _ = Describe("WalletsController", func() { BeforeEach(func() { stack = &v1beta1.Stack{ ObjectMeta: RandObjectMeta(), - Spec: v1beta1.StackSpec{Version: "v99.0.0"}, + Spec: v1beta1.StackSpec{Version: "v2.99.0"}, } databaseSettings = settings.New(uuid.NewString(), "postgres.*.uri", "postgresql://localhost", stack.Name) resourceLimitsSettings = settings.New(uuid.NewString(), From ba358fc5b938ed5051756afeabdac3f0681aa154 Mon Sep 17 00:00:00 2001 From: Maxence Maireaux Date: Tue, 14 Jul 2026 18:38:19 +0200 Subject: [PATCH 07/30] fix(ledger): preserve replica settings for v3 --- docs/04-Modules/03-Ledger.md | 9 ++-- .../09-Configuration reference/01-Settings.md | 1 - .../settings.catalog.json | 16 +++---- internal/resources/ledgers/v3.go | 23 ++++++++-- internal/resources/ledgers/v3_test.go | 42 +++++++++++++++++++ internal/tests/ledger_v3_controller_test.go | 17 ++++++++ 6 files changed, 93 insertions(+), 15 deletions(-) diff --git a/docs/04-Modules/03-Ledger.md b/docs/04-Modules/03-Ledger.md index b31499b18..7b4a2cf28 100644 --- a/docs/04-Modules/03-Ledger.md +++ b/docs/04-Modules/03-Ledger.md @@ -53,11 +53,14 @@ metadata: name: ledger-v3-replicas spec: stacks: ["formance-dev"] - key: module.ledger.v3.replicas - value: "3" + key: deployments.ledger.replicas + value: "2" ``` -The replica count must be a positive odd number. +This is the same setting used by legacy Ledger deployments. Ledger v3 requires +an odd replica count for its quorum: a positive even value is rounded up to the +next odd value (`2` becomes `3`, `4` becomes `5`). If the setting is absent, +Ledger v3 defaults to three replicas. Zero and negative values are rejected. ## Settings (v2.4+) diff --git a/docs/09-Configuration reference/01-Settings.md b/docs/09-Configuration reference/01-Settings.md index af7c5276a..8a3ec2044 100644 --- a/docs/09-Configuration reference/01-Settings.md +++ b/docs/09-Configuration reference/01-Settings.md @@ -33,7 +33,6 @@ While we have some basic types (string, number, bool ...), we also have some com | ledger.worker.async-block-hasher | Map | max-block-size=1000, schedule="0 * * * * *" | Configure async block hasher for the Ledger worker (v2.3+). Fields: `max-block-size`, `schedule` | | ledger.worker.bucket-cleanup | Map | retention-period=720h, schedule="0 0 * * *" | Configure bucket cleanup for the Ledger worker (v2.4+). Fields: `retention-period`, `schedule` | | ledger.worker.pipelines | Map | pull-interval=5s, push-retry-period=10s, sync-period=1m, logs-page-size=100 | Configure pipelines for the Ledger worker (v2.3+). Fields: `pull-interval`, `push-retry-period`, `sync-period`, `logs-page-size` | -| module.ledger.v3.replicas | Int | 3 | Number of Ledger v3 replicas. Must be a positive odd number. | | payments.encryption-key | string | | Payments data encryption key | | payments.clear-temporal | bool | true | Whether to clean up Temporal workflows when the Payments module is deleted (default: true). Set to `false` to skip the cleanup, e.g. when migrating a stack across clusters. | | payments.worker.temporal-max-concurrent-workflow-task-pollers | Int | | Payments worker max concurrent workflow task pollers configuration | diff --git a/docs/09-Configuration reference/settings.catalog.json b/docs/09-Configuration reference/settings.catalog.json index b7a93c556..78f63fdf5 100644 --- a/docs/09-Configuration reference/settings.catalog.json +++ b/docs/09-Configuration reference/settings.catalog.json @@ -191,6 +191,14 @@ "internal/resources/applications/application.go:357" ] }, + { + "key": "deployments.ledger.replicas", + "valueType": "int32", + "default": "3", + "sources": [ + "internal/resources/ledgers/v3.go:192" + ] + }, { "key": "gateway.caddyfile.grace-period", "valueType": "string", @@ -496,14 +504,6 @@ "internal/resources/applications/application.go:392" ] }, - { - "key": "module.ledger.v3.replicas", - "valueType": "int32", - "default": "3", - "sources": [ - "internal/resources/ledgers/v3.go:182" - ] - }, { "key": "modules.\u003cmodule-name\u003e.database.connection-pool", "valueType": "object", diff --git a/internal/resources/ledgers/v3.go b/internal/resources/ledgers/v3.go index 152d36e45..2b3ea66b9 100644 --- a/internal/resources/ledgers/v3.go +++ b/internal/resources/ledgers/v3.go @@ -173,18 +173,35 @@ func v3ClusterExists(ctx core.Context, stack *v1beta1.Stack) (bool, error) { return err == nil, err } +func normalizeLedgerV3Replicas(configured int32) (int32, bool, error) { + if configured < 1 { + return 0, false, fmt.Errorf("deployments.ledger.replicas must be positive, got %d", configured) + } + if configured%2 == 0 { + return configured + 1, true, nil + } + return configured, false, nil +} + func createOrUpdateV3Cluster(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, version string) (*unstructured.Unstructured, error) { image, err := registries.GetFormanceImage(ctx, stack, "ledger", version) if err != nil { return nil, err } - replicas, err := settings.GetInt32OrDefault(ctx, stack.Name, 3, "module", "ledger", "v3", "replicas") + configuredReplicas, err := settings.GetInt32OrDefault(ctx, stack.Name, 3, "deployments", "ledger", "replicas") + if err != nil { + return nil, err + } + replicas, normalized, err := normalizeLedgerV3Replicas(configuredReplicas) if err != nil { return nil, err } - if replicas < 1 || replicas%2 == 0 { - return nil, fmt.Errorf("module.ledger.v3.replicas must be a positive odd number, got %d", replicas) + if normalized { + log.FromContext(ctx).Info("Normalized Ledger v3 replicas to an odd number", + "setting", "deployments.ledger.replicas", + "configuredReplicas", configuredReplicas, + "replicas", replicas) } extraEnv, err := buildV3ExtraEnv(ctx, stack, ledger) diff --git a/internal/resources/ledgers/v3_test.go b/internal/resources/ledgers/v3_test.go index f2d2f40bb..f284f00be 100644 --- a/internal/resources/ledgers/v3_test.go +++ b/internal/resources/ledgers/v3_test.go @@ -27,3 +27,45 @@ func TestIsLedgerV3(t *testing.T) { }) } } + +func TestNormalizeLedgerV3Replicas(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + configured int32 + want int32 + wantNormalized bool + wantError bool + }{ + {name: "single replica", configured: 1, want: 1}, + {name: "odd replica count", configured: 3, want: 3}, + {name: "two replicas", configured: 2, want: 3, wantNormalized: true}, + {name: "four replicas", configured: 4, want: 5, wantNormalized: true}, + {name: "zero replicas", configured: 0, wantError: true}, + {name: "negative replicas", configured: -1, wantError: true}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + got, normalized, err := normalizeLedgerV3Replicas(test.configured) + if test.wantError { + if err == nil { + t.Fatalf("normalizeLedgerV3Replicas(%d) expected an error", test.configured) + } + return + } + if err != nil { + t.Fatalf("normalizeLedgerV3Replicas(%d) returned error: %v", test.configured, err) + } + if got != test.want { + t.Fatalf("normalizeLedgerV3Replicas(%d) = %d, want %d", test.configured, got, test.want) + } + if normalized != test.wantNormalized { + t.Fatalf("normalizeLedgerV3Replicas(%d) normalized = %t, want %t", test.configured, normalized, test.wantNormalized) + } + }) + } +} diff --git a/internal/tests/ledger_v3_controller_test.go b/internal/tests/ledger_v3_controller_test.go index 16fd84456..27187d5ce 100644 --- a/internal/tests/ledger_v3_controller_test.go +++ b/internal/tests/ledger_v3_controller_test.go @@ -90,6 +90,23 @@ var _ = Describe("Ledger v3 controller", func() { }).Should(BeTrue()) }) + It("reuses and normalizes the historical Ledger replica setting", func() { + replicaSettings := settings.New(uuid.NewString(), "deployments.ledger.replicas", "4", stack.Name) + Expect(Create(replicaSettings)).To(Succeed()) + DeferCleanup(func() { + Expect(client.IgnoreNotFound(Delete(replicaSettings))).To(Succeed()) + }) + + cluster := newLedgerV3Cluster() + Eventually(func(g Gomega) int64 { + g.Expect(Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name}, cluster)).To(Succeed()) + replicas, found, err := unstructured.NestedInt64(cluster.Object, "spec", "replicas") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(found).To(BeTrue()) + return replicas + }).Should(Equal(int64(5))) + }) + It("mirrors Cluster readiness on the Formance Ledger", func() { cluster := newLedgerV3Cluster() Eventually(func() error { From 5740dcb5ce343824f29a0924c8e65ac11f061157 Mon Sep 17 00:00:00 2001 From: Maxence Maireaux Date: Tue, 14 Jul 2026 19:25:32 +0200 Subject: [PATCH 08/30] feat(ledger): configure v3 auth and monitoring --- internal/resources/auths/env.go | 64 ++++-- internal/resources/ledgers/init.go | 1 + internal/resources/ledgers/v3.go | 102 ++++++++-- internal/resources/settings/opentelemetry.go | 106 ++++++++++ internal/tests/ledger_v3_controller_test.go | 194 +++++++++++++++++++ 5 files changed, 435 insertions(+), 32 deletions(-) diff --git a/internal/resources/auths/env.go b/internal/resources/auths/env.go index c916694f7..c4bff5dd7 100644 --- a/internal/resources/auths/env.go +++ b/internal/resources/auths/env.go @@ -2,6 +2,7 @@ package auths import ( "strconv" + "strings" v1 "k8s.io/api/core/v1" @@ -10,15 +11,21 @@ import ( "github.com/formancehq/operator/v3/internal/resources/settings" ) -func ProtectedEnvVars(ctx Context, stack *v1beta1.Stack, moduleName string, auth *v1beta1.AuthConfig) ([]v1.EnvVar, error) { - ret := make([]v1.EnvVar, 0) +type ProtectedAuthConfiguration struct { + Issuer string + Issuers []string + ReadKeySetMaxRetries int + CheckScopes bool + Service string +} +func GetProtectedConfiguration(ctx Context, stack *v1beta1.Stack, moduleName string, auth *v1beta1.AuthConfig) (*ProtectedAuthConfiguration, error) { hasAuth, err := HasDependency(ctx, stack.Name, &v1beta1.Auth{}) if err != nil { return nil, err } if !hasAuth { - return ret, nil + return nil, nil } url, err := getUrl(ctx, stack.Name) @@ -26,37 +33,54 @@ func ProtectedEnvVars(ctx Context, stack *v1beta1.Stack, moduleName string, auth return nil, err } - ret = append(ret, - Env("AUTH_ENABLED", "true"), - Env("AUTH_ISSUER", url), - ) - - issuers, err := settings.GetStringOrEmpty(ctx, stack.Name, "auth", "issuers") + issuers, err := settings.GetTrimmedStringSlice(ctx, stack.Name, "auth", "issuers") if err != nil { return nil, err } - if issuers != "" { - ret = append(ret, Env("AUTH_ISSUERS", issuers)) - } + configuration := &ProtectedAuthConfiguration{ + Issuer: url, + Issuers: issuers, + Service: moduleName, + } if auth != nil { - if auth.ReadKeySetMaxRetries != 0 { - ret = append(ret, - Env("AUTH_READ_KEY_SET_MAX_RETRIES", strconv.Itoa(auth.ReadKeySetMaxRetries)), - ) - } + configuration.ReadKeySetMaxRetries = auth.ReadKeySetMaxRetries } // Check if scope verification is enabled via Settings or module spec - checkScopes, err := shouldCheckScopes(ctx, stack.Name, moduleName, auth) + configuration.CheckScopes, err = shouldCheckScopes(ctx, stack.Name, moduleName, auth) + if err != nil { + return nil, err + } + return configuration, nil +} + +func ProtectedEnvVars(ctx Context, stack *v1beta1.Stack, moduleName string, auth *v1beta1.AuthConfig) ([]v1.EnvVar, error) { + configuration, err := GetProtectedConfiguration(ctx, stack, moduleName, auth) if err != nil { return nil, err } + if configuration == nil { + return nil, nil + } + + ret := []v1.EnvVar{ + Env("AUTH_ENABLED", "true"), + Env("AUTH_ISSUER", configuration.Issuer), + } + if len(configuration.Issuers) > 0 { + ret = append(ret, Env("AUTH_ISSUERS", strings.Join(configuration.Issuers, ","))) + } + if configuration.ReadKeySetMaxRetries != 0 { + ret = append(ret, + Env("AUTH_READ_KEY_SET_MAX_RETRIES", strconv.Itoa(configuration.ReadKeySetMaxRetries)), + ) + } - if checkScopes { + if configuration.CheckScopes { ret = append(ret, Env("AUTH_CHECK_SCOPES", "true"), - Env("AUTH_SERVICE", moduleName), + Env("AUTH_SERVICE", configuration.Service), ) } diff --git a/internal/resources/ledgers/init.go b/internal/resources/ledgers/init.go index c8169d100..bffb7619e 100644 --- a/internal/resources/ledgers/init.go +++ b/internal/resources/ledgers/init.go @@ -140,6 +140,7 @@ func init() { WithOwn[*v1beta1.Ledger](&v1beta1.BenthosStream{}), withLedgerV3ClusterWatch(), WithWatchSettings[*v1beta1.Ledger](), + WithWatchDependency[*v1beta1.Ledger](&v1beta1.Auth{}), WithWatchDependency[*v1beta1.Ledger](&v1beta1.Search{}), brokertopics.Watch[*v1beta1.Ledger]("ledger"), databases.Watch[*v1beta1.Ledger](), diff --git a/internal/resources/ledgers/v3.go b/internal/resources/ledgers/v3.go index 82060c8ca..5c6d959cf 100644 --- a/internal/resources/ledgers/v3.go +++ b/internal/resources/ledgers/v3.go @@ -2,6 +2,8 @@ package ledgers import ( "fmt" + "slices" + "strconv" "strings" "golang.org/x/mod/semver" @@ -213,7 +215,15 @@ func createOrUpdateV3Cluster(ctx core.Context, stack *v1beta1.Stack, ledger *v1b "replicas", replicas) } - extraEnv, err := buildV3ExtraEnv(ctx, stack, ledger) + monitoringConfiguration, err := settings.GetOpenTelemetryConfiguration(ctx, stack.Name, fmt.Sprintf("ledger-%s", stack.Name)) + if err != nil { + return nil, err + } + authConfiguration, err := auths.GetProtectedConfiguration(ctx, stack, "ledger", ledger.Spec.Auth) + if err != nil { + return nil, err + } + extraEnv, err := buildV3ExtraEnv(stack, ledger) if err != nil { return nil, err } @@ -274,6 +284,22 @@ func createOrUpdateV3Cluster(ctx core.Context, stack *v1beta1.Stack, ledger *v1b unstructured.RemoveNestedField(cluster.Object, "spec", "extraEnv") } + if monitoringConfiguration != nil { + if err := unstructured.SetNestedMap(cluster.Object, ledgerV3MonitoringSpec(monitoringConfiguration), "spec", "monitoring"); err != nil { + return err + } + } else { + unstructured.RemoveNestedField(cluster.Object, "spec", "monitoring") + } + + if authConfiguration != nil { + if err := unstructured.SetNestedMap(cluster.Object, ledgerV3AuthSpec(authConfiguration), "spec", "auth"); err != nil { + return err + } + } else { + unstructured.RemoveNestedField(cluster.Object, "spec", "auth") + } + if serviceAccountName != "" { if err := unstructured.SetNestedField(cluster.Object, false, "spec", "serviceAccount", "create"); err != nil { return err @@ -297,17 +323,8 @@ func imageRepository(image *registries.ImageConfiguration) string { return strings.TrimSuffix(image.Registry, "/") + "/" + image.Image } -func buildV3ExtraEnv(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger) ([]any, error) { - env, err := settings.GetOTELEnvVars(ctx, stack.Name, "ledger", " ") - if err != nil { - return nil, err - } - authEnv, err := auths.ProtectedEnvVars(ctx, stack, "ledger", ledger.Spec.Auth) - if err != nil { - return nil, err - } - env = append(env, authEnv...) - env = append(env, core.GetDevEnvVars(stack, ledger)...) +func buildV3ExtraEnv(stack *v1beta1.Stack, ledger *v1beta1.Ledger) ([]any, error) { + env := core.GetDevEnvVars(stack, ledger) ret := make([]any, 0, len(env)) for i := range env { @@ -320,6 +337,67 @@ func buildV3ExtraEnv(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Led return ret, nil } +func ledgerV3AuthSpec(configuration *auths.ProtectedAuthConfiguration) map[string]any { + spec := map[string]any{ + "enabled": true, + "issuer": configuration.Issuer, + } + if len(configuration.Issuers) > 0 { + issuers := make([]any, 0, len(configuration.Issuers)) + for _, issuer := range configuration.Issuers { + issuers = append(issuers, issuer) + } + spec["issuers"] = issuers + } + if configuration.ReadKeySetMaxRetries != 0 { + spec["readKeySetMaxRetries"] = int64(configuration.ReadKeySetMaxRetries) + } + if configuration.CheckScopes { + spec["checkScopes"] = true + spec["service"] = configuration.Service + } + return spec +} + +func ledgerV3MonitoringSpec(configuration *settings.OpenTelemetryConfiguration) map[string]any { + spec := map[string]any{ + "serviceName": configuration.ServiceName, + } + if len(configuration.Attributes) > 0 { + attributes := make([]string, 0, len(configuration.Attributes)) + for key, value := range configuration.Attributes { + attributes = append(attributes, fmt.Sprintf("%s=%s", key, value)) + } + slices.Sort(attributes) + spec["attributes"] = strings.Join(attributes, ",") + } + if configuration.Traces != nil { + traces := ledgerV3MonitoringSignalSpec(configuration.Traces) + traces["batch"] = "true" + spec["traces"] = traces + } + if configuration.Metrics != nil { + metrics := ledgerV3MonitoringSignalSpec(configuration.Metrics) + metrics["runtime"] = true + spec["metrics"] = metrics + } + if configuration.Logs != nil { + spec["logs"] = ledgerV3MonitoringSignalSpec(configuration.Logs) + } + return spec +} + +func ledgerV3MonitoringSignalSpec(configuration *settings.OpenTelemetrySignalConfiguration) map[string]any { + return map[string]any{ + "enabled": true, + "exporter": "otlp", + "endpoint": configuration.Endpoint, + "port": configuration.Port, + "insecure": strconv.FormatBool(configuration.Insecure), + "mode": configuration.Mode, + } +} + func isV3ClusterReady(cluster *unstructured.Unstructured) (bool, string, error) { phase, _, err := unstructured.NestedString(cluster.Object, "status", "phase") if err != nil { diff --git a/internal/resources/settings/opentelemetry.go b/internal/resources/settings/opentelemetry.go index 68986209e..6d5a5b1c1 100644 --- a/internal/resources/settings/opentelemetry.go +++ b/internal/resources/settings/opentelemetry.go @@ -17,11 +17,117 @@ type MonitoringType string const ( MonitoringTypeTraces MonitoringType = "TRACES" MonitoringTypeMetrics MonitoringType = "METRICS" + MonitoringTypeLogs MonitoringType = "LOGS" collectorServiceName = "otel-collector" collectorServicePort = 4318 ) +type OpenTelemetrySignalConfiguration struct { + Endpoint string + Port string + Insecure bool + Mode string +} + +type OpenTelemetryConfiguration struct { + ServiceName string + Attributes map[string]string + Traces *OpenTelemetrySignalConfiguration + Metrics *OpenTelemetrySignalConfiguration + Logs *OpenTelemetrySignalConfiguration +} + +func GetOpenTelemetryConfiguration(ctx core.Context, stack, serviceName string) (*OpenTelemetryConfiguration, error) { + info, err := getCollectorInfo(ctx, stack) + if err != nil { + return nil, err + } + + configuration := &OpenTelemetryConfiguration{ + ServiceName: serviceName, + Attributes: map[string]string{ + "pod-name": "$(POD_NAME)", + "stack": stack, + }, + } + if info != nil { + endpoint, port, _ := strings.Cut(info.endpoint, ":") + newSignal := func() *OpenTelemetrySignalConfiguration { + return &OpenTelemetrySignalConfiguration{ + Endpoint: endpoint, + Port: port, + Insecure: true, + Mode: "http", + } + } + if info.hasTraces { + configuration.Traces = newSignal() + } + if info.hasMetrics { + configuration.Metrics = newSignal() + } + for _, signal := range []string{"traces", "metrics"} { + attributes, err := GetMap(ctx, stack, "opentelemetry", signal, "resource-attributes") + if err != nil { + return nil, err + } + for key, value := range attributes { + configuration.Attributes[key] = value + } + } + configuration.Logs, err = getConfiguredOpenTelemetrySignal(ctx, stack, "logs", configuration.Attributes) + if err != nil { + return nil, err + } + if configuration.Traces == nil && configuration.Metrics == nil && configuration.Logs == nil { + return nil, nil + } + return configuration, nil + } + + for _, signal := range []struct { + name string + target **OpenTelemetrySignalConfiguration + }{ + {name: "traces", target: &configuration.Traces}, + {name: "metrics", target: &configuration.Metrics}, + {name: "logs", target: &configuration.Logs}, + } { + *signal.target, err = getConfiguredOpenTelemetrySignal(ctx, stack, signal.name, configuration.Attributes) + if err != nil { + return nil, err + } + } + + if configuration.Traces == nil && configuration.Metrics == nil && configuration.Logs == nil { + return nil, nil + } + return configuration, nil +} + +func getConfiguredOpenTelemetrySignal(ctx core.Context, stack, signal string, attributesTarget map[string]string) (*OpenTelemetrySignalConfiguration, error) { + dsn, err := GetURL(ctx, stack, "opentelemetry", signal, "dsn") + if err != nil || dsn == nil { + return nil, err + } + + attributes, err := GetMap(ctx, stack, "opentelemetry", signal, "resource-attributes") + if err != nil { + return nil, err + } + for key, value := range attributes { + attributesTarget[key] = value + } + + return &OpenTelemetrySignalConfiguration{ + Endpoint: dsn.Hostname(), + Port: dsn.Port(), + Insecure: IsTrue(dsn.Query().Get("insecure")), + Mode: dsn.Scheme, + }, nil +} + func GetOTELEnvVars(ctx core.Context, stack, serviceName string, sliceStringSeparator string) ([]corev1.EnvVar, error) { info, err := getCollectorInfo(ctx, stack) if err != nil { diff --git a/internal/tests/ledger_v3_controller_test.go b/internal/tests/ledger_v3_controller_test.go index 1f9e37a36..3991b5ac4 100644 --- a/internal/tests/ledger_v3_controller_test.go +++ b/internal/tests/ledger_v3_controller_test.go @@ -5,7 +5,9 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" @@ -77,6 +79,13 @@ var _ = Describe("Ledger v3 controller", func() { Expect(found).To(BeTrue()) Expect(tag).To(Equal("v3.0.0-alpha.1")) + _, found, err = unstructured.NestedMap(cluster.Object, "spec", "auth") + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeFalse()) + _, found, err = unstructured.NestedMap(cluster.Object, "spec", "monitoring") + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeFalse()) + deployment := &appsv1.Deployment{} Consistently(func() bool { err := Get(core.GetNamespacedResourceName(stack.Name, "ledger"), deployment) @@ -115,6 +124,191 @@ var _ = Describe("Ledger v3 controller", func() { }).Should(Equal(int64(5))) }) + It("configures authentication and monitoring from the existing stack settings", func() { + auth := &v1beta1.Auth{ + ObjectMeta: RandObjectMeta(), + Spec: v1beta1.AuthSpec{ + StackDependency: v1beta1.StackDependency{Stack: stack.Name}, + }, + } + stackSettings := []*v1beta1.Settings{ + settings.New(uuid.NewString(), "auth.issuers", "https://issuer-one.example, https://issuer-two.example", stack.Name), + settings.New(uuid.NewString(), "opentelemetry.traces.dsn", "grpc://otel-traces.monitoring.svc.cluster.local:4317?insecure=true", stack.Name), + settings.New(uuid.NewString(), "opentelemetry.metrics.dsn", "http://otel-metrics.monitoring.svc.cluster.local:4318?insecure=false", stack.Name), + settings.New(uuid.NewString(), "opentelemetry.logs.dsn", "grpc://otel-logs.monitoring.svc.cluster.local:4317?insecure=true", stack.Name), + settings.New(uuid.NewString(), "opentelemetry.traces.resource-attributes", "service.namespace=formance,team=ledger", stack.Name), + } + Expect(Create(auth)).To(Succeed()) + for _, setting := range stackSettings { + Expect(Create(setting)).To(Succeed()) + } + DeferCleanup(func() { + for _, setting := range stackSettings { + Expect(client.IgnoreNotFound(Delete(setting))).To(Succeed()) + } + Expect(client.IgnoreNotFound(Delete(auth))).To(Succeed()) + }) + + Expect(LoadResource("", ledger.Name, ledger)).To(Succeed()) + patch := client.MergeFrom(ledger.DeepCopy()) + ledger.Spec.Auth = &v1beta1.AuthConfig{ + CheckScopes: true, + ReadKeySetMaxRetries: 7, + } + Expect(Patch(ledger, patch)).To(Succeed()) + + cluster := newLedgerV3Cluster() + Eventually(func(g Gomega) map[string]any { + g.Expect(Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name}, cluster)).To(Succeed()) + authSpec, found, err := unstructured.NestedMap(cluster.Object, "spec", "auth") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(found).To(BeTrue()) + return authSpec + }).Should(SatisfyAll( + HaveKeyWithValue("enabled", true), + HaveKeyWithValue("issuer", "http://auth:8080"), + HaveKeyWithValue("issuers", []any{"https://issuer-one.example", "https://issuer-two.example"}), + HaveKeyWithValue("checkScopes", true), + HaveKeyWithValue("service", "ledger"), + HaveKeyWithValue("readKeySetMaxRetries", int64(7)), + )) + + monitoring, found, err := unstructured.NestedMap(cluster.Object, "spec", "monitoring") + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(monitoring).To(SatisfyAll( + HaveKeyWithValue("serviceName", "ledger-"+stack.Name), + HaveKeyWithValue("attributes", "pod-name=$(POD_NAME),service.namespace=formance,stack="+stack.Name+",team=ledger"), + HaveKeyWithValue("traces", map[string]any{ + "enabled": true, + "exporter": "otlp", + "endpoint": "otel-traces.monitoring.svc.cluster.local", + "port": "4317", + "insecure": "true", + "mode": "grpc", + "batch": "true", + }), + HaveKeyWithValue("metrics", map[string]any{ + "enabled": true, + "exporter": "otlp", + "endpoint": "otel-metrics.monitoring.svc.cluster.local", + "port": "4318", + "insecure": "false", + "mode": "http", + "runtime": true, + }), + HaveKeyWithValue("logs", map[string]any{ + "enabled": true, + "exporter": "otlp", + "endpoint": "otel-logs.monitoring.svc.cluster.local", + "port": "4317", + "insecure": "true", + "mode": "grpc", + }), + Not(HaveKey("pyroscope")), + )) + }) + + It("reacts to the Auth dependency lifecycle", func() { + cluster := newLedgerV3Cluster() + Eventually(func() error { + return Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name}, cluster) + }).Should(Succeed()) + + auth := &v1beta1.Auth{ + ObjectMeta: RandObjectMeta(), + Spec: v1beta1.AuthSpec{ + StackDependency: v1beta1.StackDependency{Stack: stack.Name}, + }, + } + Expect(Create(auth)).To(Succeed()) + DeferCleanup(func() { + Expect(client.IgnoreNotFound(Delete(auth))).To(Succeed()) + }) + + Eventually(func(g Gomega) bool { + g.Expect(Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name}, cluster)).To(Succeed()) + enabled, found, err := unstructured.NestedBool(cluster.Object, "spec", "auth", "enabled") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(found).To(BeTrue()) + return enabled + }).Should(BeTrue()) + + Expect(Delete(auth)).To(Succeed()) + Eventually(func(g Gomega) bool { + g.Expect(Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name}, cluster)).To(Succeed()) + _, found, err := unstructured.NestedMap(cluster.Object, "spec", "auth") + g.Expect(err).NotTo(HaveOccurred()) + return found + }).Should(BeFalse()) + }) + + It("combines a managed collector with logs configured in Settings", func() { + cluster := newLedgerV3Cluster() + Eventually(func() error { + return Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name}, cluster) + }).Should(Succeed()) + + collector := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: stack.Name, + Name: "otel-collector", + Labels: map[string]string{ + core.CollectorManagedByLabel: core.CollectorManagedByValue, + }, + Annotations: map[string]string{ + core.SignalTracesAnnotation: "true", + core.SignalMetricsAnnotation: "true", + }, + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{{Port: 4318}}, + }, + } + logsSetting := settings.New(uuid.NewString(), "opentelemetry.logs.dsn", "grpc://otel-logs.monitoring.svc.cluster.local:4317?insecure=true", stack.Name) + Expect(Create(collector)).To(Succeed()) + Expect(Create(logsSetting)).To(Succeed()) + DeferCleanup(func() { + Expect(client.IgnoreNotFound(Delete(logsSetting))).To(Succeed()) + Expect(client.IgnoreNotFound(Delete(collector))).To(Succeed()) + }) + + Eventually(func(g Gomega) map[string]any { + g.Expect(Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name}, cluster)).To(Succeed()) + monitoring, found, err := unstructured.NestedMap(cluster.Object, "spec", "monitoring") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(found).To(BeTrue()) + return monitoring + }).Should(SatisfyAll( + HaveKeyWithValue("traces", map[string]any{ + "enabled": true, + "exporter": "otlp", + "endpoint": "otel-collector." + stack.Name, + "port": "4318", + "insecure": "true", + "mode": "http", + "batch": "true", + }), + HaveKeyWithValue("metrics", map[string]any{ + "enabled": true, + "exporter": "otlp", + "endpoint": "otel-collector." + stack.Name, + "port": "4318", + "insecure": "true", + "mode": "http", + "runtime": true, + }), + HaveKeyWithValue("logs", map[string]any{ + "enabled": true, + "exporter": "otlp", + "endpoint": "otel-logs.monitoring.svc.cluster.local", + "port": "4317", + "insecure": "true", + "mode": "grpc", + }), + )) + }) + It("mirrors Cluster readiness on the Formance Ledger", func() { cluster := newLedgerV3Cluster() Eventually(func() error { From 959063affc67f8f3702aa4849af7081e7c9b82c4 Mon Sep 17 00:00:00 2001 From: Maxence Maireaux Date: Tue, 14 Jul 2026 19:41:03 +0200 Subject: [PATCH 09/30] fix(ledger): preserve monitoring service name --- internal/resources/ledgers/v3.go | 2 +- internal/tests/ledger_v3_controller_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/resources/ledgers/v3.go b/internal/resources/ledgers/v3.go index 5c6d959cf..360214c69 100644 --- a/internal/resources/ledgers/v3.go +++ b/internal/resources/ledgers/v3.go @@ -215,7 +215,7 @@ func createOrUpdateV3Cluster(ctx core.Context, stack *v1beta1.Stack, ledger *v1b "replicas", replicas) } - monitoringConfiguration, err := settings.GetOpenTelemetryConfiguration(ctx, stack.Name, fmt.Sprintf("ledger-%s", stack.Name)) + monitoringConfiguration, err := settings.GetOpenTelemetryConfiguration(ctx, stack.Name, "ledger") if err != nil { return nil, err } diff --git a/internal/tests/ledger_v3_controller_test.go b/internal/tests/ledger_v3_controller_test.go index 3991b5ac4..a425ed958 100644 --- a/internal/tests/ledger_v3_controller_test.go +++ b/internal/tests/ledger_v3_controller_test.go @@ -177,7 +177,7 @@ var _ = Describe("Ledger v3 controller", func() { Expect(err).NotTo(HaveOccurred()) Expect(found).To(BeTrue()) Expect(monitoring).To(SatisfyAll( - HaveKeyWithValue("serviceName", "ledger-"+stack.Name), + HaveKeyWithValue("serviceName", "ledger"), HaveKeyWithValue("attributes", "pod-name=$(POD_NAME),service.namespace=formance,stack="+stack.Name+",team=ledger"), HaveKeyWithValue("traces", map[string]any{ "enabled": true, From 122316d7c3426ed891a116fe251a042f9a7b2c70 Mon Sep 17 00:00:00 2001 From: Maxence Maireaux Date: Tue, 14 Jul 2026 20:18:14 +0200 Subject: [PATCH 10/30] fix(ledger): clear legacy readiness conditions for v3 --- internal/resources/ledgers/v3.go | 16 +++++++++++ internal/tests/ledger_v3_controller_test.go | 31 +++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/internal/resources/ledgers/v3.go b/internal/resources/ledgers/v3.go index 360214c69..ac08d40a7 100644 --- a/internal/resources/ledgers/v3.go +++ b/internal/resources/ledgers/v3.go @@ -96,6 +96,8 @@ func reconcileV3(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, return core.NewPendingError().WithMessage("Ledger v3 operator unavailable: Cluster CRD is not installed") } + clearLegacyLedgerConditions(ledger) + cluster := newV3Cluster() clusterKey := types.NamespacedName{Namespace: stack.Name, Name: stack.Name} err := ctx.GetClient().Get(ctx, clusterKey, cluster) @@ -141,6 +143,20 @@ func reconcileV3(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, return nil } +func clearLegacyLedgerConditions(ledger *v1beta1.Ledger) { + conditions := ledger.GetConditions() + for _, conditionType := range []string{ + "DatabaseReady", + "DeploymentReady", + "PodDisruptionBudget", + "PodDisruptionBudgetConfigured", + } { + for conditions.Get(conditionType) != nil { + conditions.Delete(v1beta1.ConditionTypeMatch(conditionType)) + } + } +} + func setLedgerV3Condition(ledger *v1beta1.Ledger, status metav1.ConditionStatus, reason, message string) { ledger.GetConditions().AppendOrReplace(v1beta1.Condition{ Type: ledgerV3ClusterReadyCondition, diff --git a/internal/tests/ledger_v3_controller_test.go b/internal/tests/ledger_v3_controller_test.go index a425ed958..db9379ea9 100644 --- a/internal/tests/ledger_v3_controller_test.go +++ b/internal/tests/ledger_v3_controller_test.go @@ -331,6 +331,37 @@ var _ = Describe("Ledger v3 controller", func() { }).Should(BeTrue()) }) + It("removes stale v2 readiness conditions", func() { + legacyTransitionTime := metav1.Now() + Eventually(func() error { + if err := LoadResource("", ledger.Name, ledger); err != nil { + return err + } + patch := client.MergeFrom(ledger.DeepCopy()) + ledger.Status.Conditions = v1beta1.Conditions{ + {Type: "DatabaseReady", Status: metav1.ConditionTrue, LastTransitionTime: legacyTransitionTime}, + {Type: "DeploymentReady", Status: metav1.ConditionFalse, Reason: "Ledger", LastTransitionTime: legacyTransitionTime}, + {Type: "DeploymentReady", Status: metav1.ConditionTrue, Reason: "LedgerWorker", LastTransitionTime: legacyTransitionTime}, + {Type: "PodDisruptionBudget", Status: metav1.ConditionTrue, Reason: "Ledger", LastTransitionTime: legacyTransitionTime}, + } + return TestContext().GetClient().Status().Patch(TestContext(), ledger, patch) + }).Should(Succeed()) + + Expect(LoadResource("", ledger.Name, ledger)).To(Succeed()) + patch := client.MergeFrom(ledger.DeepCopy()) + ledger.Spec.Debug = true + Expect(Patch(ledger, patch)).To(Succeed()) + + Eventually(func(g Gomega) []v1beta1.Condition { + g.Expect(LoadResource("", ledger.Name, ledger)).To(Succeed()) + return ledger.Status.Conditions + }).Should(And( + Not(ContainElement(HaveField("Type", "DatabaseReady"))), + Not(ContainElement(HaveField("Type", "DeploymentReady"))), + Not(ContainElement(HaveField("Type", "PodDisruptionBudget"))), + )) + }) + It("removes the Cluster when the stack is disabled", func() { cluster := newLedgerV3Cluster() Eventually(func() error { From 7261997885bc6d536b7b4b19f7dbcc382620c22c Mon Sep 17 00:00:00 2001 From: Maxence Maireaux Date: Tue, 14 Jul 2026 20:42:31 +0200 Subject: [PATCH 11/30] chore: ignore local temporary files --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 9218cdfb0..d5a8889cd 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,6 @@ charts # E2E test artifacts tests/e2e/artifacts/ + +# Local scratch files +tmp/ From 062bea6a77e917a152fc8da74b67292f611aa25d Mon Sep 17 00:00:00 2001 From: Maxence Maireaux Date: Tue, 14 Jul 2026 21:13:19 +0200 Subject: [PATCH 12/30] feat(ledger): configure v3 runtime resources and TLS --- config/rbac/role.yaml | 1 + ..._v1_clusterrole_formance-manager-role.yaml | 1 + internal/resources/ledgers/v3.go | 91 +++++++-- internal/resources/ledgers/v3_tls.go | 180 ++++++++++++++++++ .../crds/cert-manager.io_certificates.yaml | 34 ++++ .../tests/crds/cert-manager.io_issuers.yaml | 34 ++++ internal/tests/ledger_v3_controller_test.go | 180 ++++++++++++++++++ 7 files changed, 501 insertions(+), 20 deletions(-) create mode 100644 internal/resources/ledgers/v3_tls.go create mode 100644 internal/tests/crds/cert-manager.io_certificates.yaml create mode 100644 internal/tests/crds/cert-manager.io_issuers.yaml diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index c893be809..ab8a14222 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -59,6 +59,7 @@ rules: - cert-manager.io resources: - certificates + - issuers verbs: - create - delete diff --git a/helm/operator/templates/gen/rbac.authorization.k8s.io_v1_clusterrole_formance-manager-role.yaml b/helm/operator/templates/gen/rbac.authorization.k8s.io_v1_clusterrole_formance-manager-role.yaml index 530fed6d9..f1494fd7b 100644 --- a/helm/operator/templates/gen/rbac.authorization.k8s.io_v1_clusterrole_formance-manager-role.yaml +++ b/helm/operator/templates/gen/rbac.authorization.k8s.io_v1_clusterrole_formance-manager-role.yaml @@ -58,6 +58,7 @@ rules: - cert-manager.io resources: - certificates + - issuers verbs: - create - delete diff --git a/internal/resources/ledgers/v3.go b/internal/resources/ledgers/v3.go index ac08d40a7..f4867051e 100644 --- a/internal/resources/ledgers/v3.go +++ b/internal/resources/ledgers/v3.go @@ -8,6 +8,7 @@ import ( "golang.org/x/mod/semver" appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -41,7 +42,8 @@ var ( Version: "v1alpha1", Kind: "Cluster", } - ledgerV3ClusterAvailable bool + ledgerV3ClusterAvailable bool + ledgerV3CertManagerAvailable bool ) //+kubebuilder:rbac:groups=ledger.formance.com,resources=clusters,verbs=get;list;watch;create;update;patch;delete @@ -68,28 +70,41 @@ func withLedgerV3ClusterWatch() core.ReconcilerOption[*v1beta1.Ledger] { return err } - for _, crd := range crds.Items { - if crd.Spec.Group != ledgerV3ClusterGVK.Group || crd.Spec.Names.Kind != ledgerV3ClusterGVK.Kind { - continue - } - for _, version := range crd.Spec.Versions { - if version.Name == ledgerV3ClusterGVK.Version && version.Served { - ledgerV3ClusterAvailable = true - cluster := newV3Cluster() - options.Owns[cluster] = nil - b.Owns(cluster) - log.FromContext(ctx).Info("Ledger v3 Cluster CRD is available", "gvk", ledgerV3ClusterGVK) - return nil - } - } - } - - log.FromContext(ctx).Info("Ledger v3 Cluster CRD is not available", "gvk", ledgerV3ClusterGVK) + ledgerV3ClusterAvailable = watchLedgerV3Resource(ctx, b, options, crds, ledgerV3ClusterGVK) + issuerAvailable := watchLedgerV3Resource(ctx, b, options, crds, ledgerV3IssuerGVK) + certificateAvailable := watchLedgerV3Resource(ctx, b, options, crds, ledgerV3CertificateGVK) + ledgerV3CertManagerAvailable = issuerAvailable && certificateAvailable return nil }) } } +func watchLedgerV3Resource( + ctx core.Context, + b *builder.Builder, + options *core.ReconcilerOptions[*v1beta1.Ledger], + crds *apiextensionsv1.CustomResourceDefinitionList, + gvk schema.GroupVersionKind, +) bool { + for _, crd := range crds.Items { + if crd.Spec.Group != gvk.Group || crd.Spec.Names.Kind != gvk.Kind { + continue + } + for _, version := range crd.Spec.Versions { + if version.Name == gvk.Version && version.Served { + resource := newLedgerV3Resource(gvk) + options.Owns[resource] = nil + b.Owns(resource) + log.FromContext(ctx).Info("Ledger v3 dependency CRD is available", "gvk", gvk) + return true + } + } + } + + log.FromContext(ctx).Info("Ledger v3 dependency CRD is not available", "gvk", gvk) + return false +} + func reconcileV3(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, version string) error { if !ledgerV3ClusterAvailable { setLedgerV3Condition(ledger, metav1.ConditionFalse, "OperatorUnavailable", "Ledger v3 Cluster CRD is not installed") @@ -124,11 +139,21 @@ func reconcileV3(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, return err } - cluster, err = createOrUpdateV3Cluster(ctx, stack, ledger, version) + tlsReady, tlsMessage, err := createOrUpdateV3TLSResources(ctx, stack, ledger) + if err != nil { + setLedgerV3Condition(ledger, metav1.ConditionFalse, "TLSReconcileFailed", err.Error()) + return err + } + + cluster, err = createOrUpdateV3Cluster(ctx, stack, ledger, version, tlsReady) if err != nil { setLedgerV3Condition(ledger, metav1.ConditionFalse, "ReconcileFailed", err.Error()) return err } + if !tlsReady { + setLedgerV3Condition(ledger, metav1.ConditionFalse, "TLSCertificatePending", tlsMessage) + return core.NewPendingError().WithMessage("Ledger v3 TLS is not ready: %s", tlsMessage) + } ready, message, err := isV3ClusterReady(cluster) if err != nil { @@ -210,7 +235,7 @@ func normalizeLedgerV3Replicas(configured int32) (int32, bool, error) { return configured, false, nil } -func createOrUpdateV3Cluster(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, version string) (*unstructured.Unstructured, error) { +func createOrUpdateV3Cluster(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, version string, tlsReady bool) (*unstructured.Unstructured, error) { image, err := registries.GetFormanceImage(ctx, stack, "ledger", version) if err != nil { return nil, err @@ -230,6 +255,11 @@ func createOrUpdateV3Cluster(ctx core.Context, stack *v1beta1.Stack, ledger *v1b "configuredReplicas", configuredReplicas, "replicas", replicas) } + resourceRequirements, err := settings.GetResourceRequirements(ctx, stack.Name, + "deployments", "ledger", "containers", "ledger", "resource-requirements") + if err != nil { + return nil, err + } monitoringConfiguration, err := settings.GetOpenTelemetryConfiguration(ctx, stack.Name, "ledger") if err != nil { @@ -279,6 +309,23 @@ func createOrUpdateV3Cluster(ctx core.Context, stack *v1beta1.Stack, ledger *v1b if err := unstructured.SetNestedField(cluster.Object, stack.Spec.Debug || ledger.Spec.Debug, "spec", "debug"); err != nil { return err } + if tlsReady { + if err := unstructured.SetNestedMap(cluster.Object, ledgerV3TLSSpec(stack.Name), "spec", "tls"); err != nil { + return err + } + } + + if hasResourceRequirements(resourceRequirements) { + resources, err := runtime.DefaultUnstructuredConverter.ToUnstructured(resourceRequirements) + if err != nil { + return err + } + if err := unstructured.SetNestedMap(cluster.Object, resources, "spec", "resources"); err != nil { + return err + } + } else { + unstructured.RemoveNestedField(cluster.Object, "spec", "resources") + } if len(image.PullSecrets) > 0 { pullSecrets := make([]any, 0, len(image.PullSecrets)) @@ -332,6 +379,10 @@ func createOrUpdateV3Cluster(ctx core.Context, stack *v1beta1.Stack, ledger *v1b return cluster, err } +func hasResourceRequirements(resources *corev1.ResourceRequirements) bool { + return resources != nil && (len(resources.Limits) > 0 || len(resources.Requests) > 0 || len(resources.Claims) > 0) +} + func imageRepository(image *registries.ImageConfiguration) string { if image.Registry == "" { return image.Image diff --git a/internal/resources/ledgers/v3_tls.go b/internal/resources/ledgers/v3_tls.go new file mode 100644 index 000000000..56a1dedb6 --- /dev/null +++ b/internal/resources/ledgers/v3_tls.go @@ -0,0 +1,180 @@ +package ledgers + +import ( + "fmt" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + "github.com/formancehq/operator/v3/api/formance.com/v1beta1" + "github.com/formancehq/operator/v3/internal/core" +) + +const ( + ledgerV3TLSCertificateDuration = "87600h" + ledgerV3TLSCertificateRenewBefore = "720h" + ledgerV3TLSCASecretKey = "ca.crt" +) + +var ( + ledgerV3IssuerGVK = schema.GroupVersionKind{ + Group: "cert-manager.io", + Version: "v1", + Kind: "Issuer", + } + ledgerV3CertificateGVK = schema.GroupVersionKind{ + Group: "cert-manager.io", + Version: "v1", + Kind: "Certificate", + } +) + +//+kubebuilder:rbac:groups=cert-manager.io,resources=issuers;certificates,verbs=get;list;watch;create;update;patch;delete + +func newLedgerV3Resource(gvk schema.GroupVersionKind) *unstructured.Unstructured { + resource := &unstructured.Unstructured{} + resource.SetGroupVersionKind(gvk) + return resource +} + +func createOrUpdateV3TLSResources(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger) (bool, string, error) { + if !ledgerV3CertManagerAvailable { + return false, "cert-manager Issuer and Certificate CRDs are not installed", nil + } + + issuer := newLedgerV3Resource(ledgerV3IssuerGVK) + issuer.SetNamespace(stack.Name) + issuer.SetName(ledgerV3IssuerName(stack.Name)) + if _, err := controllerutil.CreateOrUpdate(ctx, ctx.GetClient(), issuer, func() error { + setLedgerV3TLSResourceMetadata(issuer, stack.Name) + if err := controllerutil.SetControllerReference(ledger, issuer, ctx.GetScheme()); err != nil { + return err + } + return unstructured.SetNestedMap(issuer.Object, map[string]any{}, "spec", "selfSigned") + }); err != nil { + return false, "", err + } + + certificate := newLedgerV3Resource(ledgerV3CertificateGVK) + certificate.SetNamespace(stack.Name) + certificate.SetName(ledgerV3TLSName(stack.Name)) + if _, err := controllerutil.CreateOrUpdate(ctx, ctx.GetClient(), certificate, func() error { + setLedgerV3TLSResourceMetadata(certificate, stack.Name) + if err := controllerutil.SetControllerReference(ledger, certificate, ctx.GetScheme()); err != nil { + return err + } + certificate.Object["spec"] = ledgerV3CertificateSpec(stack.Name) + return nil + }); err != nil { + return false, "", err + } + + ready, message, err := ledgerV3CertificateReady(certificate) + if err != nil || !ready { + return ready, message, err + } + + secret := &corev1.Secret{} + err = ctx.GetClient().Get(ctx, types.NamespacedName{Namespace: stack.Name, Name: ledgerV3TLSName(stack.Name)}, secret) + if apierrors.IsNotFound(err) { + return false, fmt.Sprintf("TLS Secret %s/%s does not exist", stack.Name, ledgerV3TLSName(stack.Name)), nil + } + if err != nil { + return false, "", err + } + for _, key := range []string{corev1.TLSCertKey, corev1.TLSPrivateKeyKey, ledgerV3TLSCASecretKey} { + if len(secret.Data[key]) == 0 { + return false, fmt.Sprintf("TLS Secret %s/%s is missing %q", stack.Name, secret.Name, key), nil + } + } + + return true, "Certificate and TLS Secret are ready", nil +} + +func setLedgerV3TLSResourceMetadata(resource *unstructured.Unstructured, stackName string) { + labels := resource.GetLabels() + if labels == nil { + labels = map[string]string{} + } + labels[v1beta1.StackLabel] = stackName + resource.SetLabels(labels) +} + +func ledgerV3IssuerName(stackName string) string { + return stackName + "-selfsigned" +} + +func ledgerV3TLSName(stackName string) string { + return stackName + "-tls" +} + +func ledgerV3CertificateSpec(stackName string) map[string]any { + serviceName := "ledger-" + stackName + return map[string]any{ + "secretName": ledgerV3TLSName(stackName), + "duration": ledgerV3TLSCertificateDuration, + "renewBefore": ledgerV3TLSCertificateRenewBefore, + "issuerRef": map[string]any{ + "name": ledgerV3IssuerName(stackName), + "kind": "Issuer", + }, + "commonName": serviceName + "." + stackName + ".svc.cluster.local", + "dnsNames": ledgerV3TLSDNSNames(stackName), + } +} + +func ledgerV3TLSDNSNames(stackName string) []any { + serviceName := "ledger-" + stackName + headlessServiceName := serviceName + "-headless" + dnsNames := make([]any, 0, 16) + for _, name := range []string{serviceName, serviceName + "-grpc", headlessServiceName, "*." + headlessServiceName} { + dnsNames = append(dnsNames, ledgerV3ServiceDNSNames(name, stackName)...) + } + return dnsNames +} + +func ledgerV3ServiceDNSNames(serviceName, namespace string) []any { + return []any{ + serviceName, + serviceName + "." + namespace, + serviceName + "." + namespace + ".svc", + serviceName + "." + namespace + ".svc.cluster.local", + } +} + +func ledgerV3CertificateReady(certificate *unstructured.Unstructured) (bool, string, error) { + conditions, found, err := unstructured.NestedSlice(certificate.Object, "status", "conditions") + if err != nil { + return false, "", err + } + if !found { + return false, fmt.Sprintf("Certificate %s/%s has no Ready condition", certificate.GetNamespace(), certificate.GetName()), nil + } + for _, rawCondition := range conditions { + condition, ok := rawCondition.(map[string]any) + if !ok || condition["type"] != "Ready" { + continue + } + if condition["status"] == "True" { + return true, "Certificate is ready", nil + } + message, _ := condition["message"].(string) + if message == "" { + message = fmt.Sprintf("Certificate Ready condition is %v", condition["status"]) + } + return false, message, nil + } + return false, fmt.Sprintf("Certificate %s/%s has no Ready condition", certificate.GetNamespace(), certificate.GetName()), nil +} + +func ledgerV3TLSSpec(stackName string) map[string]any { + return map[string]any{ + "enabled": true, + "secretName": ledgerV3TLSName(stackName), + "caSecretKey": ledgerV3TLSCASecretKey, + } +} diff --git a/internal/tests/crds/cert-manager.io_certificates.yaml b/internal/tests/crds/cert-manager.io_certificates.yaml new file mode 100644 index 000000000..1c17eed9a --- /dev/null +++ b/internal/tests/crds/cert-manager.io_certificates.yaml @@ -0,0 +1,34 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: certificates.cert-manager.io +spec: + group: cert-manager.io + names: + kind: Certificate + listKind: CertificateList + plural: certificates + singular: certificate + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + status: + type: object + x-kubernetes-preserve-unknown-fields: true + subresources: + status: {} diff --git a/internal/tests/crds/cert-manager.io_issuers.yaml b/internal/tests/crds/cert-manager.io_issuers.yaml new file mode 100644 index 000000000..bbdb52b67 --- /dev/null +++ b/internal/tests/crds/cert-manager.io_issuers.yaml @@ -0,0 +1,34 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: issuers.cert-manager.io +spec: + group: cert-manager.io + names: + kind: Issuer + listKind: IssuerList + plural: issuers + singular: issuer + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + status: + type: object + x-kubernetes-preserve-unknown-fields: true + subresources: + status: {} diff --git a/internal/tests/ledger_v3_controller_test.go b/internal/tests/ledger_v3_controller_test.go index db9379ea9..b3c338d5f 100644 --- a/internal/tests/ledger_v3_controller_test.go +++ b/internal/tests/ledger_v3_controller_test.go @@ -25,12 +25,36 @@ var ledgerV3ClusterGVK = schema.GroupVersionKind{ Kind: "Cluster", } +var ledgerV3IssuerGVK = schema.GroupVersionKind{ + Group: "cert-manager.io", + Version: "v1", + Kind: "Issuer", +} + +var ledgerV3CertificateGVK = schema.GroupVersionKind{ + Group: "cert-manager.io", + Version: "v1", + Kind: "Certificate", +} + func newLedgerV3Cluster() *unstructured.Unstructured { cluster := &unstructured.Unstructured{} cluster.SetGroupVersionKind(ledgerV3ClusterGVK) return cluster } +func newLedgerV3Issuer() *unstructured.Unstructured { + issuer := &unstructured.Unstructured{} + issuer.SetGroupVersionKind(ledgerV3IssuerGVK) + return issuer +} + +func newLedgerV3Certificate() *unstructured.Unstructured { + certificate := &unstructured.Unstructured{} + certificate.SetGroupVersionKind(ledgerV3CertificateGVK) + return certificate +} + var _ = Describe("Ledger v3 controller", func() { var ( stack *v1beta1.Stack @@ -124,6 +148,137 @@ var _ = Describe("Ledger v3 controller", func() { }).Should(Equal(int64(5))) }) + It("reuses the historical Ledger container resource settings", func() { + resourceSettings := []*v1beta1.Settings{ + settings.New(uuid.NewString(), "deployments.ledger.containers.ledger.resource-requirements.requests", "cpu=50m,memory=6Gi", stack.Name), + settings.New(uuid.NewString(), "deployments.ledger.containers.ledger.resource-requirements.limits", "cpu=2,memory=6Gi", stack.Name), + } + for _, setting := range resourceSettings { + Expect(Create(setting)).To(Succeed()) + } + DeferCleanup(func() { + for _, setting := range resourceSettings { + Expect(client.IgnoreNotFound(Delete(setting))).To(Succeed()) + } + }) + + cluster := newLedgerV3Cluster() + Eventually(func(g Gomega) map[string]any { + g.Expect(Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name}, cluster)).To(Succeed()) + resources, found, err := unstructured.NestedMap(cluster.Object, "spec", "resources") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(found).To(BeTrue()) + return resources + }).Should(Equal(map[string]any{ + "requests": map[string]any{ + "cpu": "50m", + "memory": "6Gi", + }, + "limits": map[string]any{ + "cpu": "2", + "memory": "6Gi", + }, + })) + + for _, setting := range resourceSettings { + Expect(Delete(setting)).To(Succeed()) + } + Eventually(func(g Gomega) bool { + g.Expect(Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name}, cluster)).To(Succeed()) + _, found, err := unstructured.NestedMap(cluster.Object, "spec", "resources") + g.Expect(err).NotTo(HaveOccurred()) + return found + }).Should(BeFalse()) + }) + + It("enables TLS only after the managed certificate is ready", func() { + issuer := newLedgerV3Issuer() + certificate := newLedgerV3Certificate() + cluster := newLedgerV3Cluster() + tlsName := stack.Name + "-tls" + + Eventually(func(g Gomega) { + g.Expect(Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name + "-selfsigned"}, issuer)).To(Succeed()) + g.Expect(issuer).To(BeControlledBy(ledger)) + selfSigned, found, err := unstructured.NestedMap(issuer.Object, "spec", "selfSigned") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(found).To(BeTrue()) + g.Expect(selfSigned).To(BeEmpty()) + + g.Expect(Get(types.NamespacedName{Namespace: stack.Name, Name: tlsName}, certificate)).To(Succeed()) + g.Expect(certificate).To(BeControlledBy(ledger)) + }).Should(Succeed()) + + commonName, found, err := unstructured.NestedString(certificate.Object, "spec", "commonName") + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(commonName).To(Equal("ledger-" + stack.Name + "." + stack.Name + ".svc.cluster.local")) + + dnsNames, found, err := unstructured.NestedStringSlice(certificate.Object, "spec", "dnsNames") + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(dnsNames).To(ConsistOf( + "ledger-"+stack.Name, + "ledger-"+stack.Name+"."+stack.Name, + "ledger-"+stack.Name+"."+stack.Name+".svc", + "ledger-"+stack.Name+"."+stack.Name+".svc.cluster.local", + "ledger-"+stack.Name+"-grpc", + "ledger-"+stack.Name+"-grpc."+stack.Name, + "ledger-"+stack.Name+"-grpc."+stack.Name+".svc", + "ledger-"+stack.Name+"-grpc."+stack.Name+".svc.cluster.local", + "ledger-"+stack.Name+"-headless", + "ledger-"+stack.Name+"-headless."+stack.Name, + "ledger-"+stack.Name+"-headless."+stack.Name+".svc", + "ledger-"+stack.Name+"-headless."+stack.Name+".svc.cluster.local", + "*.ledger-"+stack.Name+"-headless", + "*.ledger-"+stack.Name+"-headless."+stack.Name, + "*.ledger-"+stack.Name+"-headless."+stack.Name+".svc", + "*.ledger-"+stack.Name+"-headless."+stack.Name+".svc.cluster.local", + )) + + Eventually(func(g Gomega) bool { + g.Expect(Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name}, cluster)).To(Succeed()) + _, found, err := unstructured.NestedMap(cluster.Object, "spec", "tls") + g.Expect(err).NotTo(HaveOccurred()) + return found + }).Should(BeFalse()) + + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: stack.Name, Name: tlsName}, + Data: map[string][]byte{ + "tls.crt": []byte("certificate"), + "tls.key": []byte("private key"), + "ca.crt": []byte("certificate authority"), + }, + } + Expect(Create(secret)).To(Succeed()) + DeferCleanup(func() { + Expect(client.IgnoreNotFound(Delete(secret))).To(Succeed()) + }) + + Eventually(func(g Gomega) error { + g.Expect(Get(types.NamespacedName{Namespace: stack.Name, Name: tlsName}, certificate)).To(Succeed()) + certificate.Object["status"] = map[string]any{ + "conditions": []any{ + map[string]any{"type": "Ready", "status": "True", "reason": "Ready"}, + }, + } + return TestContext().GetClient().Status().Update(TestContext(), certificate) + }).Should(Succeed()) + + Eventually(func(g Gomega) map[string]any { + g.Expect(Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name}, cluster)).To(Succeed()) + tls, found, err := unstructured.NestedMap(cluster.Object, "spec", "tls") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(found).To(BeTrue()) + return tls + }).Should(Equal(map[string]any{ + "enabled": true, + "secretName": tlsName, + "caSecretKey": "ca.crt", + })) + }) + It("configures authentication and monitoring from the existing stack settings", func() { auth := &v1beta1.Auth{ ObjectMeta: RandObjectMeta(), @@ -310,6 +465,31 @@ var _ = Describe("Ledger v3 controller", func() { }) It("mirrors Cluster readiness on the Formance Ledger", func() { + certificate := newLedgerV3Certificate() + tlsName := stack.Name + "-tls" + Eventually(func(g Gomega) error { + g.Expect(Get(types.NamespacedName{Namespace: stack.Name, Name: tlsName}, certificate)).To(Succeed()) + return nil + }).Should(Succeed()) + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: stack.Name, Name: tlsName}, + Data: map[string][]byte{ + "tls.crt": []byte("certificate"), + "tls.key": []byte("private key"), + "ca.crt": []byte("certificate authority"), + }, + } + Expect(Create(secret)).To(Succeed()) + DeferCleanup(func() { + Expect(client.IgnoreNotFound(Delete(secret))).To(Succeed()) + }) + certificate.Object["status"] = map[string]any{ + "conditions": []any{ + map[string]any{"type": "Ready", "status": "True", "reason": "Ready"}, + }, + } + Expect(TestContext().GetClient().Status().Update(TestContext(), certificate)).To(Succeed()) + cluster := newLedgerV3Cluster() Eventually(func() error { return Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name}, cluster) From 2d6e7decfc783800501b4d19d97acfc38a9d3c5a Mon Sep 17 00:00:00 2001 From: Maxence Maireaux Date: Tue, 14 Jul 2026 22:38:32 +0200 Subject: [PATCH 13/30] feat(ledger): add v3 preview mode --- .../v1beta1/gatewaybackend_types.go | 48 ++++ .../v1beta1/gatewaygrpcapi_types.go | 3 + .../v1beta1/gatewayhttpapi_types.go | 3 + .../v1beta1/zz_generated.deepcopy.go | 45 ++++ .../bases/formance.com_gatewaygrpcapis.yaml | 37 +++ .../bases/formance.com_gatewayhttpapis.yaml | 42 ++++ docs/04-Modules/03-Ledger.md | 35 +++ .../02-Custom Resource Definitions.md | 55 +++++ .../settings.catalog.json | 59 ++++- ...finition_gatewaygrpcapis.formance.com.yaml | 37 +++ ...finition_gatewayhttpapis.formance.com.yaml | 42 ++++ internal/resources/gatewaygrpcapis/create.go | 6 + internal/resources/gatewaygrpcapis/init.go | 4 + internal/resources/gatewayhttpapis/create.go | 7 + internal/resources/gateways/Caddyfile.gotpl | 30 ++- internal/resources/gateways/caddyfile.go | 26 +- internal/resources/gateways/deployment.go | 72 +++++- internal/resources/gateways/init.go | 12 +- internal/resources/ledgers/init.go | 48 +++- internal/resources/ledgers/v3.go | 72 +++--- internal/resources/ledgers/v3_preview.go | 162 ++++++++++++ internal/resources/ledgers/v3_tls.go | 19 +- internal/tests/gateway_controller_test.go | 97 ++++++++ internal/tests/ledger_v3_controller_test.go | 233 +++++++++++++++++- 24 files changed, 1126 insertions(+), 68 deletions(-) create mode 100644 api/formance.com/v1beta1/gatewaybackend_types.go create mode 100644 internal/resources/ledgers/v3_preview.go diff --git a/api/formance.com/v1beta1/gatewaybackend_types.go b/api/formance.com/v1beta1/gatewaybackend_types.go new file mode 100644 index 000000000..ac56fb124 --- /dev/null +++ b/api/formance.com/v1beta1/gatewaybackend_types.go @@ -0,0 +1,48 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +const GatewayBackendTLSSecretLabel = "formance.com/gateway-backend-tls" + +// GatewayBackendTLS configures TLS when Gateway connects to a backend. +type GatewayBackendTLS struct { + // SecretName contains the CA used to verify the backend certificate. + // +kubebuilder:validation:MinLength=1 + SecretName string `json:"secretName"` + // CASecretKey is the key containing the CA certificate. + // +optional + // +kubebuilder:default:="ca.crt" + CASecretKey string `json:"caSecretKey,omitempty"` + // ServerName is used for backend certificate verification. + // +kubebuilder:validation:MinLength=1 + ServerName string `json:"serverName"` +} + +// GatewayBackendRef selects the Kubernetes Service used by a Gateway route. +// When omitted, Gateway keeps using the module's historical Service. +type GatewayBackendRef struct { + // Name is the backend Service name in the Stack namespace. + // +kubebuilder:validation:MinLength=1 + Name string `json:"name"` + // Port is the backend Service port. + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=65535 + Port int32 `json:"port"` + // TLS enables a verified TLS connection to the backend. + // +optional + TLS *GatewayBackendTLS `json:"tls,omitempty"` +} diff --git a/api/formance.com/v1beta1/gatewaygrpcapi_types.go b/api/formance.com/v1beta1/gatewaygrpcapi_types.go index 760ba134f..9b9d7a4a8 100644 --- a/api/formance.com/v1beta1/gatewaygrpcapi_types.go +++ b/api/formance.com/v1beta1/gatewaygrpcapi_types.go @@ -31,6 +31,9 @@ type GatewayGRPCAPISpec struct { //+optional //+kubebuilder:default:=8081 Port int32 `json:"port,omitempty"` + // BackendRef overrides the historical -grpc Service. + // +optional + BackendRef *GatewayBackendRef `json:"backendRef,omitempty"` } type GatewayGRPCAPIStatus struct { diff --git a/api/formance.com/v1beta1/gatewayhttpapi_types.go b/api/formance.com/v1beta1/gatewayhttpapi_types.go index 5a199b5ad..35200248c 100644 --- a/api/formance.com/v1beta1/gatewayhttpapi_types.go +++ b/api/formance.com/v1beta1/gatewayhttpapi_types.go @@ -27,6 +27,9 @@ type GatewayHTTPAPIRule struct { //+optional //+kubebuilder:default:=false Secured bool `json:"secured"` + // BackendRef overrides the historical module Service for this rule. + // +optional + BackendRef *GatewayBackendRef `json:"backendRef,omitempty"` } type GatewayHTTPAPISpec struct { diff --git a/api/formance.com/v1beta1/zz_generated.deepcopy.go b/api/formance.com/v1beta1/zz_generated.deepcopy.go index 79a5e6ba5..a1d01e41e 100644 --- a/api/formance.com/v1beta1/zz_generated.deepcopy.go +++ b/api/formance.com/v1beta1/zz_generated.deepcopy.go @@ -965,6 +965,41 @@ func (in *Gateway) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GatewayBackendRef) DeepCopyInto(out *GatewayBackendRef) { + *out = *in + if in.TLS != nil { + in, out := &in.TLS, &out.TLS + *out = new(GatewayBackendTLS) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GatewayBackendRef. +func (in *GatewayBackendRef) DeepCopy() *GatewayBackendRef { + if in == nil { + return nil + } + out := new(GatewayBackendRef) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GatewayBackendTLS) DeepCopyInto(out *GatewayBackendTLS) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GatewayBackendTLS. +func (in *GatewayBackendTLS) DeepCopy() *GatewayBackendTLS { + if in == nil { + return nil + } + out := new(GatewayBackendTLS) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GatewayGRPCAPI) DeepCopyInto(out *GatewayGRPCAPI) { *out = *in @@ -1033,6 +1068,11 @@ func (in *GatewayGRPCAPISpec) DeepCopyInto(out *GatewayGRPCAPISpec) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.BackendRef != nil { + in, out := &in.BackendRef, &out.BackendRef + *out = new(GatewayBackendRef) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GatewayGRPCAPISpec. @@ -1128,6 +1168,11 @@ func (in *GatewayHTTPAPIRule) DeepCopyInto(out *GatewayHTTPAPIRule) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.BackendRef != nil { + in, out := &in.BackendRef, &out.BackendRef + *out = new(GatewayBackendRef) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GatewayHTTPAPIRule. diff --git a/config/crd/bases/formance.com_gatewaygrpcapis.yaml b/config/crd/bases/formance.com_gatewaygrpcapis.yaml index 0145e7e0a..bd28b5384 100644 --- a/config/crd/bases/formance.com_gatewaygrpcapis.yaml +++ b/config/crd/bases/formance.com_gatewaygrpcapis.yaml @@ -47,6 +47,43 @@ spec: type: object spec: properties: + backendRef: + description: BackendRef overrides the historical -grpc Service. + properties: + name: + description: Name is the backend Service name in the Stack namespace. + minLength: 1 + type: string + port: + description: Port is the backend Service port. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + tls: + description: TLS enables a verified TLS connection to the backend. + properties: + caSecretKey: + default: ca.crt + description: CASecretKey is the key containing the CA certificate. + type: string + secretName: + description: SecretName contains the CA used to verify the + backend certificate. + minLength: 1 + type: string + serverName: + description: ServerName is used for backend certificate verification. + minLength: 1 + type: string + required: + - secretName + - serverName + type: object + required: + - name + - port + type: object grpcServices: description: |- GRPCServices is the list of fully-qualified gRPC service names diff --git a/config/crd/bases/formance.com_gatewayhttpapis.yaml b/config/crd/bases/formance.com_gatewayhttpapis.yaml index a29950d0a..e6a2d49c1 100644 --- a/config/crd/bases/formance.com_gatewayhttpapis.yaml +++ b/config/crd/bases/formance.com_gatewayhttpapis.yaml @@ -57,6 +57,48 @@ spec: description: Rules items: properties: + backendRef: + description: BackendRef overrides the historical module Service + for this rule. + properties: + name: + description: Name is the backend Service name in the Stack + namespace. + minLength: 1 + type: string + port: + description: Port is the backend Service port. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + tls: + description: TLS enables a verified TLS connection to the + backend. + properties: + caSecretKey: + default: ca.crt + description: CASecretKey is the key containing the CA + certificate. + type: string + secretName: + description: SecretName contains the CA used to verify + the backend certificate. + minLength: 1 + type: string + serverName: + description: ServerName is used for backend certificate + verification. + minLength: 1 + type: string + required: + - secretName + - serverName + type: object + required: + - name + - port + type: object methods: items: type: string diff --git a/docs/04-Modules/03-Ledger.md b/docs/04-Modules/03-Ledger.md index 7b4a2cf28..1cbb6e249 100644 --- a/docs/04-Modules/03-Ledger.md +++ b/docs/04-Modules/03-Ledger.md @@ -44,6 +44,41 @@ Ledger reports that an explicit migration is required and does not create the v3 `Cluster`. The reverse transition is guarded in the same way: legacy resources are not created while a v3 `Cluster` still exists. +### Ledger v3 preview alongside Ledger v2 + +For migration preparation, a stack that still runs Ledger v2 can start a +separate Ledger v3 cluster without changing its module version: + +```yaml +apiVersion: formance.com/v1beta1 +kind: Settings +metadata: + name: ledger-v3-preview +spec: + stacks: ["formance-dev"] + key: ledger.v3.preview-version + value: "v3.0.0-alpha.11" +``` + +The value must be a Ledger version strictly newer than `v3.0.0-alpha`. In this +mode, the Operator keeps all v2 Deployments and the v2 Database running, and +creates an isolated v3 `Cluster`. Gateway exposes the preview through: + +- HTTP under `/api/ledger/v3`, while `/api/ledger/v2` and the other historical + Ledger routes continue to target v2; +- gRPC service `ledger.BucketService`, which targets the TLS-enabled v3 + cluster. + +The Operator creates and mounts a cert-manager-managed CA for the Gateway to +verify the v3 gRPC backend. The gRPC route is published only after both the +`Certificate` and its TLS `Secret` are ready. + +This preview mode does not start or supervise data mirroring. Mirroring and +client validation remain explicit migration steps. Changing the Ledger module +version to v3 is still blocked while legacy resources exist, even when the +preview cluster is already running. Removing the Setting deletes only preview +resources and routes; Ledger v2 remains active. + The v3 cluster size defaults to three replicas and can be configured per stack: ```yaml diff --git a/docs/09-Configuration reference/02-Custom Resource Definitions.md b/docs/09-Configuration reference/02-Custom Resource Definitions.md index 245670d6b..35bd4a8d8 100644 --- a/docs/09-Configuration reference/02-Custom Resource Definitions.md +++ b/docs/09-Configuration reference/02-Custom Resource Definitions.md @@ -2300,6 +2300,60 @@ GatewayGRPCAPI is the Schema for the GRPCAPIs API | `name` _string_ | Name indicates the module name (e.g. "ledger") | | | | `grpcServices` _string array_ | GRPCServices is the list of fully-qualified gRPC service names
exposed by this module (e.g. "formance.ledger.v1.LedgerService") | | | | `port` _integer_ | Port is the gRPC port on the backend service | 8081 | | +| `backendRef` _[GatewayBackendRef](#gatewaybackendref)_ | BackendRef overrides the historical -grpc Service. | | | + +###### GatewayBackendRef + + + +GatewayBackendRef selects the Kubernetes Service used by a Gateway route. +When omitted, Gateway keeps using the module's historical Service. + + + + + + + + + + + + + + + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `name` _string_ | Name is the backend Service name in the Stack namespace. | | MinLength: 1
| +| `port` _integer_ | Port is the backend Service port. | | Maximum: 65535
Minimum: 1
| +| `tls` _[GatewayBackendTLS](#gatewaybackendtls)_ | TLS enables a verified TLS connection to the backend. | | | + +###### GatewayBackendTLS + + + +GatewayBackendTLS configures TLS when Gateway connects to a backend. + + + + + + + + + + + + + + + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `secretName` _string_ | SecretName contains the CA used to verify the backend certificate. | | MinLength: 1
| +| `caSecretKey` _string_ | CASecretKey is the key containing the CA certificate. | ca.crt | | +| `serverName` _string_ | ServerName is used for backend certificate verification. | | MinLength: 1
| @@ -2414,6 +2468,7 @@ GatewayHTTPAPI is the Schema for the HTTPAPIs API | `path` _string_ | | | | | `methods` _string array_ | | | | | `secured` _boolean_ | | false | | +| `backendRef` _[GatewayBackendRef](#gatewaybackendref)_ | BackendRef overrides the historical module Service for this rule. | | | diff --git a/docs/09-Configuration reference/settings.catalog.json b/docs/09-Configuration reference/settings.catalog.json index 9a2a01439..9373f207e 100644 --- a/docs/09-Configuration reference/settings.catalog.json +++ b/docs/09-Configuration reference/settings.catalog.json @@ -5,15 +5,21 @@ "key": "auth.\u003cmodule-name\u003e.check-scopes", "valueType": "bool", "sources": [ - "internal/resources/auths/env.go:79" + "internal/resources/auths/env.go:103" ] }, { "key": "auth.issuers", "valueType": "string", "sources": [ - "internal/resources/auths/deployment.go:69", - "internal/resources/auths/env.go:34" + "internal/resources/auths/deployment.go:69" + ] + }, + { + "key": "auth.issuers", + "valueType": "string[]", + "sources": [ + "internal/resources/auths/env.go:36" ] }, { @@ -191,12 +197,33 @@ "internal/resources/applications/application.go:357" ] }, + { + "key": "deployments.ledger.containers.ledger.resource-requirements.claims", + "valueType": "string[]", + "sources": [ + "internal/resources/ledgers/v3.go:254" + ] + }, + { + "key": "deployments.ledger.containers.ledger.resource-requirements.limits", + "valueType": "map[string]string", + "sources": [ + "internal/resources/ledgers/v3.go:254" + ] + }, + { + "key": "deployments.ledger.containers.ledger.resource-requirements.requests", + "valueType": "map[string]string", + "sources": [ + "internal/resources/ledgers/v3.go:254" + ] + }, { "key": "deployments.ledger.replicas", "valueType": "int32", "default": "3", "sources": [ - "internal/resources/ledgers/v3.go:201" + "internal/resources/ledgers/v3.go:240" ] }, { @@ -435,6 +462,13 @@ "internal/resources/ledgers/deployments.go:247" ] }, + { + "key": "ledger.v3.preview-version", + "valueType": "string", + "sources": [ + "internal/resources/ledgers/v3_preview.go:36" + ] + }, { "key": "ledger.worker.async-block-hasher", "valueType": "object", @@ -555,21 +589,30 @@ "key": "opentelemetry.\u003cmonitoring-type\u003e.dsn", "valueType": "uri", "sources": [ - "internal/resources/settings/opentelemetry.go:156" + "internal/resources/settings/opentelemetry.go:262" ] }, { "key": "opentelemetry.\u003cmonitoring-type\u003e.resource-attributes", "valueType": "map[string]string", "sources": [ - "internal/resources/settings/opentelemetry.go:200" + "internal/resources/settings/opentelemetry.go:306" + ] + }, + { + "key": "opentelemetry.\u003csignal\u003e.dsn", + "valueType": "uri", + "sources": [ + "internal/resources/settings/opentelemetry.go:110" ] }, { "key": "opentelemetry.\u003csignal\u003e.resource-attributes", "valueType": "map[string]string", "sources": [ - "internal/resources/settings/opentelemetry.go:81" + "internal/resources/settings/opentelemetry.go:115", + "internal/resources/settings/opentelemetry.go:187", + "internal/resources/settings/opentelemetry.go:71" ] }, { @@ -584,7 +627,7 @@ "valueType": "uri", "sources": [ "internal/resources/otelexporterendpoints/init.go:682", - "internal/resources/settings/opentelemetry.go:146" + "internal/resources/settings/opentelemetry.go:252" ] }, { diff --git a/helm/crds/templates/crds/apiextensions.k8s.io_v1_customresourcedefinition_gatewaygrpcapis.formance.com.yaml b/helm/crds/templates/crds/apiextensions.k8s.io_v1_customresourcedefinition_gatewaygrpcapis.formance.com.yaml index 43e2e0be0..dceaebccf 100644 --- a/helm/crds/templates/crds/apiextensions.k8s.io_v1_customresourcedefinition_gatewaygrpcapis.formance.com.yaml +++ b/helm/crds/templates/crds/apiextensions.k8s.io_v1_customresourcedefinition_gatewaygrpcapis.formance.com.yaml @@ -50,6 +50,43 @@ spec: type: object spec: properties: + backendRef: + description: BackendRef overrides the historical -grpc Service. + properties: + name: + description: Name is the backend Service name in the Stack namespace. + minLength: 1 + type: string + port: + description: Port is the backend Service port. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + tls: + description: TLS enables a verified TLS connection to the backend. + properties: + caSecretKey: + default: ca.crt + description: CASecretKey is the key containing the CA certificate. + type: string + secretName: + description: SecretName contains the CA used to verify the + backend certificate. + minLength: 1 + type: string + serverName: + description: ServerName is used for backend certificate verification. + minLength: 1 + type: string + required: + - secretName + - serverName + type: object + required: + - name + - port + type: object grpcServices: description: |- GRPCServices is the list of fully-qualified gRPC service names diff --git a/helm/crds/templates/crds/apiextensions.k8s.io_v1_customresourcedefinition_gatewayhttpapis.formance.com.yaml b/helm/crds/templates/crds/apiextensions.k8s.io_v1_customresourcedefinition_gatewayhttpapis.formance.com.yaml index e896b2afa..062619ccf 100644 --- a/helm/crds/templates/crds/apiextensions.k8s.io_v1_customresourcedefinition_gatewayhttpapis.formance.com.yaml +++ b/helm/crds/templates/crds/apiextensions.k8s.io_v1_customresourcedefinition_gatewayhttpapis.formance.com.yaml @@ -60,6 +60,48 @@ spec: description: Rules items: properties: + backendRef: + description: BackendRef overrides the historical module Service + for this rule. + properties: + name: + description: Name is the backend Service name in the Stack + namespace. + minLength: 1 + type: string + port: + description: Port is the backend Service port. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + tls: + description: TLS enables a verified TLS connection to the + backend. + properties: + caSecretKey: + default: ca.crt + description: CASecretKey is the key containing the CA + certificate. + type: string + secretName: + description: SecretName contains the CA used to verify + the backend certificate. + minLength: 1 + type: string + serverName: + description: ServerName is used for backend certificate + verification. + minLength: 1 + type: string + required: + - secretName + - serverName + type: object + required: + - name + - port + type: object methods: items: type: string diff --git a/internal/resources/gatewaygrpcapis/create.go b/internal/resources/gatewaygrpcapis/create.go index 0d2b2bde0..8acb8dd6c 100644 --- a/internal/resources/gatewaygrpcapis/create.go +++ b/internal/resources/gatewaygrpcapis/create.go @@ -43,3 +43,9 @@ func WithPort(port int32) func(grpcapi *v1beta1.GatewayGRPCAPI) { grpcapi.Spec.Port = port } } + +func WithBackendRef(backendRef v1beta1.GatewayBackendRef) func(grpcapi *v1beta1.GatewayGRPCAPI) { + return func(grpcapi *v1beta1.GatewayGRPCAPI) { + grpcapi.Spec.BackendRef = &backendRef + } +} diff --git a/internal/resources/gatewaygrpcapis/init.go b/internal/resources/gatewaygrpcapis/init.go index 1f577a0a1..464874da6 100644 --- a/internal/resources/gatewaygrpcapis/init.go +++ b/internal/resources/gatewaygrpcapis/init.go @@ -29,6 +29,10 @@ import ( //+kubebuilder:rbac:groups=formance.com,resources=gatewaygrpcapis/finalizers,verbs=update func Reconcile(ctx Context, _ *v1beta1.Stack, grpcAPI *v1beta1.GatewayGRPCAPI) error { + if grpcAPI.Spec.BackendRef != nil { + return DeleteIfExists[*corev1.Service](ctx, GetNamespacedResourceName(grpcAPI.Spec.Stack, grpcAPI.Spec.Name+"-grpc")) + } + _, err := services.Create(ctx, grpcAPI, grpcAPI.Spec.Name+"-grpc", services.WithConfig(services.PortConfig{ ServiceName: grpcAPI.Spec.Name, diff --git a/internal/resources/gatewayhttpapis/create.go b/internal/resources/gatewayhttpapis/create.go index e843b8235..d65971797 100644 --- a/internal/resources/gatewayhttpapis/create.go +++ b/internal/resources/gatewayhttpapis/create.go @@ -63,3 +63,10 @@ func RuleUnsecured() v1beta1.GatewayHTTPAPIRule { Secured: true, } } + +func RuleSecuredWithBackend(path string, backendRef v1beta1.GatewayBackendRef) v1beta1.GatewayHTTPAPIRule { + return v1beta1.GatewayHTTPAPIRule{ + Path: path, + BackendRef: &backendRef, + } +} diff --git a/internal/resources/gateways/Caddyfile.gotpl b/internal/resources/gateways/Caddyfile.gotpl index b074fd2b6..769fd75fa 100644 --- a/internal/resources/gateways/Caddyfile.gotpl +++ b/internal/resources/gateways/Caddyfile.gotpl @@ -110,14 +110,27 @@ {{- range $i, $service := .Services }} {{- range $j, $rule := $service.Rules }} + {{- $backendName := $service.Name }} + {{- $backendPort := 8080 }} + {{- if $rule.BackendRef }} + {{- $backendName = $rule.BackendRef.Name }} + {{- $backendPort = $rule.BackendRef.Port }} + {{- end }} handle /api/{{ $service.Name }}{{ $rule.Path }}* { {{- if $rule.Methods }} method {{ join $rule.Methods " " }} {{- end }} uri strip_prefix /api/{{ $service.Name }} import cors - reverse_proxy {{ $service.Name }}:8080 { + reverse_proxy {{ if and $rule.BackendRef $rule.BackendRef.TLS }}https://{{ end }}{{ $backendName }}:{{ $backendPort }} { header_up Host {upstream_hostport} + {{- if and $rule.BackendRef $rule.BackendRef.TLS }} + transport http { + tls + tls_trust_pool file /etc/gateway/tls/{{ $rule.BackendRef.TLS.SecretName }}/{{ $rule.BackendRef.TLS.CASecretKey }} + tls_server_name {{ $rule.BackendRef.TLS.ServerName }} + } + {{- end }} } } {{- end }} @@ -148,10 +161,23 @@ {{- range $i, $service := .GRPCServices }} {{- range $j, $svcName := $service.GRPCServices }} + {{- $backendName := printf "%s-grpc" $service.Name }} + {{- $backendPort := $service.Port }} + {{- if $service.BackendRef }} + {{- $backendName = $service.BackendRef.Name }} + {{- $backendPort = $service.BackendRef.Port }} + {{- end }} handle /{{ $svcName }}/* { - reverse_proxy {{ $service.Name }}-grpc:{{ $service.Port }} { + reverse_proxy {{ if and $service.BackendRef $service.BackendRef.TLS }}https://{{ end }}{{ $backendName }}:{{ $backendPort }} { transport http { + {{- if and $service.BackendRef $service.BackendRef.TLS }} + tls + tls_trust_pool file /etc/gateway/tls/{{ $service.BackendRef.TLS.SecretName }}/{{ $service.BackendRef.TLS.CASecretKey }} + tls_server_name {{ $service.BackendRef.TLS.ServerName }} + versions 2 + {{- else }} versions h2c 2 + {{- end }} } } } diff --git a/internal/resources/gateways/caddyfile.go b/internal/resources/gateways/caddyfile.go index 3ff8652ad..5758f8814 100644 --- a/internal/resources/gateways/caddyfile.go +++ b/internal/resources/gateways/caddyfile.go @@ -1,6 +1,7 @@ package gateways import ( + "sort" "strings" "time" @@ -17,12 +18,23 @@ func CreateCaddyfile(ctx core.Context, stack *v1beta1.Stack, gateway *v1beta1.Gateway, httpAPIs []*v1beta1.GatewayHTTPAPI, grpcAPIs []*v1beta1.GatewayGRPCAPI, broker *v1beta1.Broker, options ...CaddyOptions) (string, error) { + services := collectionutils.Map(httpAPIs, func(from *v1beta1.GatewayHTTPAPI) v1beta1.GatewayHTTPAPISpec { + spec := *from.Spec.DeepCopy() + for i := range spec.Rules { + normalizeBackendTLS(spec.Rules[i].BackendRef) + } + sort.SliceStable(spec.Rules, func(i, j int) bool { + return len(spec.Rules[i].Path) > len(spec.Rules[j].Path) + }) + return spec + }) + data := map[string]any{ - "Services": collectionutils.Map(httpAPIs, func(from *v1beta1.GatewayHTTPAPI) v1beta1.GatewayHTTPAPISpec { - return from.Spec - }), + "Services": services, "GRPCServices": collectionutils.Map(grpcAPIs, func(from *v1beta1.GatewayGRPCAPI) v1beta1.GatewayGRPCAPISpec { - return from.Spec + spec := *from.Spec.DeepCopy() + normalizeBackendTLS(spec.BackendRef) + return spec }), "Platform": ctx.GetPlatform(), "Debug": stack.Spec.Debug, @@ -46,6 +58,12 @@ func CreateCaddyfile(ctx core.Context, stack *v1beta1.Stack, return caddy.ComputeCaddyfile(ctx, stack, Caddyfile, data) } +func normalizeBackendTLS(backendRef *v1beta1.GatewayBackendRef) { + if backendRef != nil && backendRef.TLS != nil && backendRef.TLS.CASecretKey == "" { + backendRef.TLS.CASecretKey = "ca.crt" + } +} + func withTrustedProxies(options []string) func(data map[string]any) error { return func(data map[string]any) error { data["TrustedProxies"] = strings.Join(options, " ") diff --git a/internal/resources/gateways/deployment.go b/internal/resources/gateways/deployment.go index 0a2ba0a51..14ebcca28 100644 --- a/internal/resources/gateways/deployment.go +++ b/internal/resources/gateways/deployment.go @@ -1,8 +1,13 @@ package gateways import ( + "crypto/sha256" + "encoding/json" + "fmt" + "sort" "strings" + appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" "github.com/formancehq/operator/v3/api/formance.com/v1beta1" @@ -18,6 +23,8 @@ func createDeployment( stack *v1beta1.Stack, gateway *v1beta1.Gateway, caddyfileConfigMap *v1.ConfigMap, + httpAPIs []*v1beta1.GatewayHTTPAPI, + grpcAPIs []*v1beta1.GatewayGRPCAPI, broker *v1beta1.Broker, version string, ) error { @@ -57,11 +64,13 @@ func createDeployment( if err != nil { return err } - caddyTpl, err := caddy.DeploymentTemplate(ctx, stack, gateway, caddyfileConfigMap, imageConfiguration, env) if err != nil { return err } + if err := configureBackendTLSVolumes(ctx, stack.Name, caddyTpl, httpAPIs, grpcAPIs); err != nil { + return err + } if broker != nil { var topicPrefix string @@ -84,3 +93,64 @@ func createDeployment( IsEE(). Install(ctx) } + +func configureBackendTLSVolumes( + ctx core.Context, + namespace string, + deployment *appsv1.Deployment, + httpAPIs []*v1beta1.GatewayHTTPAPI, + grpcAPIs []*v1beta1.GatewayGRPCAPI, +) error { + secretNames := map[string]struct{}{} + for _, httpAPI := range httpAPIs { + for _, rule := range httpAPI.Spec.Rules { + if rule.BackendRef != nil && rule.BackendRef.TLS != nil { + secretNames[rule.BackendRef.TLS.SecretName] = struct{}{} + } + } + } + for _, grpcAPI := range grpcAPIs { + if grpcAPI.Spec.BackendRef != nil && grpcAPI.Spec.BackendRef.TLS != nil { + secretNames[grpcAPI.Spec.BackendRef.TLS.SecretName] = struct{}{} + } + } + + sortedSecretNames := make([]string, 0, len(secretNames)) + for secretName := range secretNames { + sortedSecretNames = append(sortedSecretNames, secretName) + } + sort.Strings(sortedSecretNames) + secretsDigest := sha256.New() + for _, secretName := range sortedSecretNames { + secret := &v1.Secret{} + if err := ctx.GetClient().Get(ctx, core.GetNamespacedResourceName(namespace, secretName), secret); err != nil { + return fmt.Errorf("getting backend TLS Secret %s/%s: %w", namespace, secretName, err) + } + secretData, err := json.Marshal(secret.Data) + if err != nil { + return fmt.Errorf("hashing backend TLS Secret %s/%s: %w", namespace, secretName, err) + } + _, _ = secretsDigest.Write([]byte(secretName)) + _, _ = secretsDigest.Write(secretData) + + digest := sha256.Sum256([]byte(secretName)) + volumeName := fmt.Sprintf("backend-tls-%x", digest[:6]) + deployment.Spec.Template.Spec.Volumes = append(deployment.Spec.Template.Spec.Volumes, v1.Volume{ + Name: volumeName, + VolumeSource: v1.VolumeSource{Secret: &v1.SecretVolumeSource{ + SecretName: secretName, + }}, + }) + deployment.Spec.Template.Spec.Containers[0].VolumeMounts = append( + deployment.Spec.Template.Spec.Containers[0].VolumeMounts, + core.NewVolumeMount(volumeName, "/etc/gateway/tls/"+secretName, true), + ) + } + if len(sortedSecretNames) > 0 { + if deployment.Spec.Template.Annotations == nil { + deployment.Spec.Template.Annotations = map[string]string{} + } + deployment.Spec.Template.Annotations["formance.com/backend-tls-secrets-hash"] = fmt.Sprintf("%x", secretsDigest.Sum(nil)) + } + return nil +} diff --git a/internal/resources/gateways/init.go b/internal/resources/gateways/init.go index d0cd250c2..9ccdc53af 100644 --- a/internal/resources/gateways/init.go +++ b/internal/resources/gateways/init.go @@ -27,7 +27,9 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" externaldnsv1alpha1 "sigs.k8s.io/external-dns/apis/v1alpha1" . "github.com/formancehq/go-libs/v5/pkg/types/collections" @@ -97,7 +99,7 @@ func Reconcile(ctx Context, stack *v1beta1.Stack, gateway *v1beta1.Gateway, vers return err } - if err := createDeployment(ctx, stack, gateway, configMap, broker, version); err != nil { + if err := createDeployment(ctx, stack, gateway, configMap, httpAPIs, grpcAPIs, broker, version); err != nil { return err } @@ -176,6 +178,14 @@ func init() { WithWatchDependency[*v1beta1.Gateway](&v1beta1.GatewayHTTPAPI{}), WithWatchDependency[*v1beta1.Gateway](&v1beta1.GatewayGRPCAPI{}), WithWatchDependency[*v1beta1.Gateway](&v1beta1.Auth{}), + WithWatch[*v1beta1.Gateway](func(ctx Context, secret *corev1.Secret) []reconcile.Request { + if secret.Labels[v1beta1.GatewayBackendTLSSecretLabel] != "true" { + return nil + } + return BuildReconcileRequests(ctx, ctx.GetClient(), ctx.GetScheme(), &v1beta1.Gateway{}, client.MatchingFields{ + "stack": secret.Namespace, + }) + }), brokertopics.Watch[*v1beta1.Gateway]("gateway"), ), ) diff --git a/internal/resources/ledgers/init.go b/internal/resources/ledgers/init.go index bffb7619e..c17118ffe 100644 --- a/internal/resources/ledgers/init.go +++ b/internal/resources/ledgers/init.go @@ -43,25 +43,56 @@ func Reconcile(ctx Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, versio if isLedgerV3(version) { return reconcileV3(ctx, stack, ledger, version) } + previewVersion, err := ledgerV3PreviewVersion(ctx, stack) + if err != nil { + return err + } if ledgerV3ClusterAvailable { - exists, err := v3ClusterExists(ctx, stack) + cluster, exists, err := getV3Cluster(ctx, stack) if err != nil { return err } - if exists { + if exists && !isLedgerV3Preview(cluster) { setLedgerV3Condition(ledger, metav1.ConditionFalse, "MigrationRequired", "A Ledger v3 Cluster exists; an explicit v3 to v2 migration is required") return NewPendingError().WithMessage("migration required before switching Ledger from v3 to v2") } + if previewVersion == "" { + if err := deleteLedgerV3Preview(ctx, stack); err != nil { + return err + } + } } ledger.GetConditions().Delete(v1beta1.ConditionTypeMatch(ledgerV3ClusterReadyCondition)) - return reconcileLegacy(ctx, stack, ledger, version) + ledger.GetConditions().Delete(v1beta1.ConditionTypeMatch(ledgerV3PreviewReadyCondition)) + if err := reconcileLegacy(ctx, stack, ledger, version, previewVersion); err != nil { + return err + } + if previewVersion != "" { + return reconcileV3Preview(ctx, stack, ledger, previewVersion) + } + return nil } -func reconcileLegacy(ctx Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, version string) error { - if err := DeleteIfExists[*v1beta1.GatewayGRPCAPI](ctx, GetResourceName(GetObjectName(stack.Name, "ledger"))); err != nil { - return err +func reconcileLegacy(ctx Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, version, previewVersion string) error { + if previewVersion == "" { + if err := DeleteIfExists[*v1beta1.GatewayGRPCAPI](ctx, GetResourceName(GetObjectName(stack.Name, "ledger"))); err != nil { + return err + } + } + + httpRules := []v1beta1.GatewayHTTPAPIRule{gatewayhttpapis.RuleSecured()} + if previewVersion != "" { + httpRules = append([]v1beta1.GatewayHTTPAPIRule{ + gatewayhttpapis.RuleSecuredWithBackend("/v3", ledgerV3HTTPBackendRef(stack.Name)), + }, httpRules...) + } + + if previewVersion != "" && !ledgerV3ClusterAvailable { + if err := DeleteIfExists[*v1beta1.GatewayGRPCAPI](ctx, GetResourceName(GetObjectName(stack.Name, "ledger"))); err != nil { + return err + } } database, err := databases.Create(ctx, stack, ledger) @@ -74,7 +105,10 @@ func reconcileLegacy(ctx Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, return err } - if err := gatewayhttpapis.Create(ctx, ledger, gatewayhttpapis.WithHealthCheckEndpoint("_healthcheck")); err != nil { + if err := gatewayhttpapis.Create(ctx, ledger, + gatewayhttpapis.WithHealthCheckEndpoint("_healthcheck"), + gatewayhttpapis.WithRules(httpRules...), + ); err != nil { return err } diff --git a/internal/resources/ledgers/v3.go b/internal/resources/ledgers/v3.go index f4867051e..70cc6a190 100644 --- a/internal/resources/ledgers/v3.go +++ b/internal/resources/ledgers/v3.go @@ -32,7 +32,10 @@ import ( const ( ledgerV3Threshold = "v3.0.0-alpha" ledgerV3ClusterReadyCondition = "LedgerV3ClusterReady" + ledgerV3PreviewReadyCondition = "LedgerV3PreviewReady" + ledgerV3PreviewLabel = "formance.com/ledger-v3-preview" ledgerV3GRPCPort = int32(8888) + ledgerV3HTTPPort = int32(9000) ledgerV3PublicGRPCService = "ledger.BucketService" ) @@ -113,39 +116,22 @@ func reconcileV3(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, clearLegacyLedgerConditions(ledger) - cluster := newV3Cluster() - clusterKey := types.NamespacedName{Namespace: stack.Name, Name: stack.Name} - err := ctx.GetClient().Get(ctx, clusterKey, cluster) - if apierrors.IsNotFound(err) { - hasLegacyResources, err := legacyLedgerResourcesExist(ctx, stack) - if err != nil { - return err - } - if hasLegacyResources { - setLedgerV3Condition(ledger, metav1.ConditionFalse, "MigrationRequired", "Legacy Ledger resources exist; an explicit v2 to v3 migration is required") - return core.NewPendingError().WithMessage("migration required before switching Ledger from v2 to v3") - } - } else if err != nil { - return err - } - - if err := gatewayhttpapis.Create(ctx, ledger, gatewayhttpapis.WithHealthCheckEndpoint("livez")); err != nil { + hasLegacyResources, err := legacyLedgerResourcesExist(ctx, stack) + if err != nil { return err } - if err := gatewaygrpcapis.Create(ctx, ledger, - gatewaygrpcapis.WithGRPCServices(ledgerV3PublicGRPCService), - gatewaygrpcapis.WithPort(ledgerV3GRPCPort), - ); err != nil { - return err + if hasLegacyResources { + setLedgerV3Condition(ledger, metav1.ConditionFalse, "MigrationRequired", "Legacy Ledger resources exist; an explicit v2 to v3 migration is required") + return core.NewPendingError().WithMessage("migration required before switching Ledger from v2 to v3") } - tlsReady, tlsMessage, err := createOrUpdateV3TLSResources(ctx, stack, ledger) + tlsReady, tlsMessage, err := createOrUpdateV3TLSResources(ctx, stack, ledger, false) if err != nil { setLedgerV3Condition(ledger, metav1.ConditionFalse, "TLSReconcileFailed", err.Error()) return err } - cluster, err = createOrUpdateV3Cluster(ctx, stack, ledger, version, tlsReady) + cluster, err := createOrUpdateV3Cluster(ctx, stack, ledger, version, false) if err != nil { setLedgerV3Condition(ledger, metav1.ConditionFalse, "ReconcileFailed", err.Error()) return err @@ -154,6 +140,16 @@ func reconcileV3(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, setLedgerV3Condition(ledger, metav1.ConditionFalse, "TLSCertificatePending", tlsMessage) return core.NewPendingError().WithMessage("Ledger v3 TLS is not ready: %s", tlsMessage) } + if err := gatewayhttpapis.Create(ctx, ledger, gatewayhttpapis.WithHealthCheckEndpoint("livez")); err != nil { + return err + } + if err := gatewaygrpcapis.Create(ctx, ledger, + gatewaygrpcapis.WithGRPCServices(ledgerV3PublicGRPCService), + gatewaygrpcapis.WithPort(ledgerV3GRPCPort), + gatewaygrpcapis.WithBackendRef(ledgerV3GRPCBackendRef(stack.Name)), + ); err != nil { + return err + } ready, message, err := isV3ClusterReady(cluster) if err != nil { @@ -216,13 +212,13 @@ func legacyLedgerResourcesExist(ctx core.Context, stack *v1beta1.Stack) (bool, e return false, nil } -func v3ClusterExists(ctx core.Context, stack *v1beta1.Stack) (bool, error) { +func getV3Cluster(ctx core.Context, stack *v1beta1.Stack) (*unstructured.Unstructured, bool, error) { cluster := newV3Cluster() err := ctx.GetClient().Get(ctx, types.NamespacedName{Namespace: stack.Name, Name: stack.Name}, cluster) if apierrors.IsNotFound(err) { - return false, nil + return cluster, false, nil } - return err == nil, err + return cluster, err == nil, err } func normalizeLedgerV3Replicas(configured int32) (int32, bool, error) { @@ -235,7 +231,7 @@ func normalizeLedgerV3Replicas(configured int32) (int32, bool, error) { return configured, false, nil } -func createOrUpdateV3Cluster(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, version string, tlsReady bool) (*unstructured.Unstructured, error) { +func createOrUpdateV3Cluster(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, version string, preview bool) (*unstructured.Unstructured, error) { image, err := registries.GetFormanceImage(ctx, stack, "ledger", version) if err != nil { return nil, err @@ -288,6 +284,11 @@ func createOrUpdateV3Cluster(ctx core.Context, stack *v1beta1.Stack, ledger *v1b labels = map[string]string{} } labels[v1beta1.StackLabel] = stack.Name + if preview { + labels[ledgerV3PreviewLabel] = "true" + } else { + delete(labels, ledgerV3PreviewLabel) + } cluster.SetLabels(labels) if err := controllerutil.SetControllerReference(ledger, cluster, ctx.GetScheme()); err != nil { @@ -309,10 +310,21 @@ func createOrUpdateV3Cluster(ctx core.Context, stack *v1beta1.Stack, ledger *v1b if err := unstructured.SetNestedField(cluster.Object, stack.Spec.Debug || ledger.Spec.Debug, "spec", "debug"); err != nil { return err } - if tlsReady { - if err := unstructured.SetNestedMap(cluster.Object, ledgerV3TLSSpec(stack.Name), "spec", "tls"); err != nil { + // Configure TLS from the first Cluster revision. Pods can wait for the + // cert-manager Secret, but must never bootstrap a plaintext Raft cluster + // that is switched to TLS by a later StatefulSet revision. + if err := unstructured.SetNestedMap(cluster.Object, ledgerV3TLSSpec(stack.Name), "spec", "tls"); err != nil { + return err + } + if preview { + if err := unstructured.SetNestedStringMap(cluster.Object, map[string]string{ + "app.kubernetes.io/name": "ledger-v3-preview", + ledgerV3PreviewLabel: "true", + }, "spec", "additionalLabels"); err != nil { return err } + } else { + unstructured.RemoveNestedField(cluster.Object, "spec", "additionalLabels") } if hasResourceRequirements(resourceRequirements) { diff --git a/internal/resources/ledgers/v3_preview.go b/internal/resources/ledgers/v3_preview.go new file mode 100644 index 000000000..d88c63da3 --- /dev/null +++ b/internal/resources/ledgers/v3_preview.go @@ -0,0 +1,162 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ledgers + +import ( + "fmt" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/formancehq/operator/v3/api/formance.com/v1beta1" + "github.com/formancehq/operator/v3/internal/core" + "github.com/formancehq/operator/v3/internal/resources/gatewaygrpcapis" + "github.com/formancehq/operator/v3/internal/resources/settings" +) + +func ledgerV3PreviewVersion(ctx core.Context, stack *v1beta1.Stack) (string, error) { + version, err := settings.GetStringOrEmpty(ctx, stack.Name, "ledger", "v3", "preview-version") + if err != nil { + return "", err + } + if version != "" && !isLedgerV3(version) { + return "", fmt.Errorf("ledger.v3.preview-version must be greater than %s, got %q", ledgerV3Threshold, version) + } + return version, nil +} + +func reconcileV3Preview(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, version string) error { + if !ledgerV3ClusterAvailable { + setLedgerV3PreviewCondition(ledger, metav1.ConditionFalse, "OperatorUnavailable", "Ledger v3 Cluster CRD is not installed") + return core.NewPendingError().WithMessage("Ledger v3 preview unavailable: Cluster CRD is not installed") + } + + tlsReady, tlsMessage, err := createOrUpdateV3TLSResources(ctx, stack, ledger, true) + if err != nil { + setLedgerV3PreviewCondition(ledger, metav1.ConditionFalse, "TLSReconcileFailed", err.Error()) + return err + } + cluster, err := createOrUpdateV3Cluster(ctx, stack, ledger, version, true) + if err != nil { + setLedgerV3PreviewCondition(ledger, metav1.ConditionFalse, "ReconcileFailed", err.Error()) + return err + } + if !tlsReady { + if err := core.DeleteIfExists[*v1beta1.GatewayGRPCAPI](ctx, core.GetResourceName(core.GetObjectName(stack.Name, "ledger"))); err != nil { + return err + } + setLedgerV3PreviewCondition(ledger, metav1.ConditionFalse, "TLSCertificatePending", tlsMessage) + return core.NewPendingError().WithMessage("Ledger v3 preview TLS is not ready: %s", tlsMessage) + } + + if err := gatewaygrpcapis.Create(ctx, ledger, + gatewaygrpcapis.WithGRPCServices(ledgerV3PublicGRPCService), + gatewaygrpcapis.WithPort(ledgerV3GRPCPort), + gatewaygrpcapis.WithBackendRef(ledgerV3GRPCBackendRef(stack.Name)), + ); err != nil { + setLedgerV3PreviewCondition(ledger, metav1.ConditionFalse, "GatewayReconcileFailed", err.Error()) + return err + } + + ready, message, err := isV3ClusterReady(cluster) + if err != nil { + return err + } + if !ready { + setLedgerV3PreviewCondition(ledger, metav1.ConditionFalse, "Pending", message) + return core.NewPendingError().WithMessage("Ledger v3 preview Cluster is not ready: %s", message) + } + + setLedgerV3PreviewCondition(ledger, metav1.ConditionTrue, "Running", message) + return nil +} + +func ledgerV3HTTPBackendRef(stackName string) v1beta1.GatewayBackendRef { + return v1beta1.GatewayBackendRef{ + Name: "ledger-" + stackName, + Port: ledgerV3HTTPPort, + } +} + +func ledgerV3GRPCBackendRef(stackName string) v1beta1.GatewayBackendRef { + return v1beta1.GatewayBackendRef{ + Name: "ledger-" + stackName, + Port: ledgerV3GRPCPort, + TLS: &v1beta1.GatewayBackendTLS{ + SecretName: ledgerV3TLSName(stackName), + CASecretKey: ledgerV3TLSCASecretKey, + ServerName: "ledger-" + stackName + "." + stackName + ".svc.cluster.local", + }, + } +} + +func setLedgerV3PreviewCondition(ledger *v1beta1.Ledger, status metav1.ConditionStatus, reason, message string) { + ledger.GetConditions().AppendOrReplace(v1beta1.Condition{ + Type: ledgerV3PreviewReadyCondition, + Status: status, + ObservedGeneration: ledger.GetGeneration(), + LastTransitionTime: metav1.Now(), + Reason: reason, + Message: message, + }, v1beta1.ConditionTypeMatch(ledgerV3PreviewReadyCondition)) +} + +func isLedgerV3Preview(cluster *unstructured.Unstructured) bool { + return cluster.GetLabels()[ledgerV3PreviewLabel] == "true" +} + +func deleteLedgerV3Preview(ctx core.Context, stack *v1beta1.Stack) error { + cluster, exists, err := getV3Cluster(ctx, stack) + if err != nil { + return err + } + if exists && isLedgerV3Preview(cluster) { + if err := client.IgnoreNotFound(ctx.GetClient().Delete(ctx, cluster)); err != nil { + return err + } + } + + certificate := newLedgerV3Resource(ledgerV3CertificateGVK) + err = ctx.GetClient().Get(ctx, types.NamespacedName{Namespace: stack.Name, Name: ledgerV3TLSName(stack.Name)}, certificate) + if err == nil && certificate.GetLabels()[ledgerV3PreviewLabel] == "true" { + if err := ctx.GetClient().Delete(ctx, certificate); err != nil && !apierrors.IsNotFound(err) { + return err + } + secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: stack.Name, Name: ledgerV3TLSName(stack.Name)}} + if err := ctx.GetClient().Delete(ctx, secret); err != nil && !apierrors.IsNotFound(err) { + return err + } + } else if err != nil && !apierrors.IsNotFound(err) { + return err + } + + issuer := newLedgerV3Resource(ledgerV3IssuerGVK) + err = ctx.GetClient().Get(ctx, types.NamespacedName{Namespace: stack.Name, Name: ledgerV3IssuerName(stack.Name)}, issuer) + if err == nil && issuer.GetLabels()[ledgerV3PreviewLabel] == "true" { + if err := ctx.GetClient().Delete(ctx, issuer); err != nil && !apierrors.IsNotFound(err) { + return err + } + } else if err != nil && !apierrors.IsNotFound(err) { + return err + } + + return nil +} diff --git a/internal/resources/ledgers/v3_tls.go b/internal/resources/ledgers/v3_tls.go index 56a1dedb6..d3fc055d0 100644 --- a/internal/resources/ledgers/v3_tls.go +++ b/internal/resources/ledgers/v3_tls.go @@ -41,7 +41,7 @@ func newLedgerV3Resource(gvk schema.GroupVersionKind) *unstructured.Unstructured return resource } -func createOrUpdateV3TLSResources(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger) (bool, string, error) { +func createOrUpdateV3TLSResources(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, preview bool) (bool, string, error) { if !ledgerV3CertManagerAvailable { return false, "cert-manager Issuer and Certificate CRDs are not installed", nil } @@ -50,7 +50,7 @@ func createOrUpdateV3TLSResources(ctx core.Context, stack *v1beta1.Stack, ledger issuer.SetNamespace(stack.Name) issuer.SetName(ledgerV3IssuerName(stack.Name)) if _, err := controllerutil.CreateOrUpdate(ctx, ctx.GetClient(), issuer, func() error { - setLedgerV3TLSResourceMetadata(issuer, stack.Name) + setLedgerV3TLSResourceMetadata(issuer, stack.Name, preview) if err := controllerutil.SetControllerReference(ledger, issuer, ctx.GetScheme()); err != nil { return err } @@ -63,7 +63,7 @@ func createOrUpdateV3TLSResources(ctx core.Context, stack *v1beta1.Stack, ledger certificate.SetNamespace(stack.Name) certificate.SetName(ledgerV3TLSName(stack.Name)) if _, err := controllerutil.CreateOrUpdate(ctx, ctx.GetClient(), certificate, func() error { - setLedgerV3TLSResourceMetadata(certificate, stack.Name) + setLedgerV3TLSResourceMetadata(certificate, stack.Name, preview) if err := controllerutil.SetControllerReference(ledger, certificate, ctx.GetScheme()); err != nil { return err } @@ -95,12 +95,17 @@ func createOrUpdateV3TLSResources(ctx core.Context, stack *v1beta1.Stack, ledger return true, "Certificate and TLS Secret are ready", nil } -func setLedgerV3TLSResourceMetadata(resource *unstructured.Unstructured, stackName string) { +func setLedgerV3TLSResourceMetadata(resource *unstructured.Unstructured, stackName string, preview bool) { labels := resource.GetLabels() if labels == nil { labels = map[string]string{} } labels[v1beta1.StackLabel] = stackName + if preview { + labels[ledgerV3PreviewLabel] = "true" + } else { + delete(labels, ledgerV3PreviewLabel) + } resource.SetLabels(labels) } @@ -118,6 +123,12 @@ func ledgerV3CertificateSpec(stackName string) map[string]any { "secretName": ledgerV3TLSName(stackName), "duration": ledgerV3TLSCertificateDuration, "renewBefore": ledgerV3TLSCertificateRenewBefore, + "isCA": true, + "secretTemplate": map[string]any{ + "labels": map[string]any{ + v1beta1.GatewayBackendTLSSecretLabel: "true", + }, + }, "issuerRef": map[string]any{ "name": ledgerV3IssuerName(stackName), "kind": "Issuer", diff --git a/internal/tests/gateway_controller_test.go b/internal/tests/gateway_controller_test.go index dcae3aac5..d18aca371 100644 --- a/internal/tests/gateway_controller_test.go +++ b/internal/tests/gateway_controller_test.go @@ -321,6 +321,103 @@ var _ = Describe("GatewayController", func() { }).Should(MatchGoldenFile("gateway-controller", "configmap-with-ledger-and-grpc.yaml")) }) }) + Context("Then adding explicit preview backends", func() { + var ( + grpcAPI *v1beta1.GatewayGRPCAPI + tlsSecret *corev1.Secret + ) + BeforeEach(func() { + httpAPI.Spec.Rules = []v1beta1.GatewayHTTPAPIRule{ + { + Path: "/v3", + BackendRef: &v1beta1.GatewayBackendRef{ + Name: "ledger-preview", + Port: 9000, + }, + }, + gatewayhttpapis.RuleSecured(), + } + grpcAPI = &v1beta1.GatewayGRPCAPI{ + ObjectMeta: RandObjectMeta(), + Spec: v1beta1.GatewayGRPCAPISpec{ + StackDependency: v1beta1.StackDependency{Stack: stack.Name}, + Name: "ledger", + GRPCServices: []string{"ledger.BucketService"}, + BackendRef: &v1beta1.GatewayBackendRef{ + Name: "ledger-preview", + Port: 8888, + TLS: &v1beta1.GatewayBackendTLS{ + SecretName: "ledger-preview-tls", + CASecretKey: "ca.crt", + ServerName: "ledger-preview.stack.svc.cluster.local", + }, + }, + }, + } + tlsSecret = &corev1.Secret{ + ObjectMeta: RandObjectMeta(), + Data: map[string][]byte{"ca.crt": []byte("initial CA")}, + } + tlsSecret.Name = "ledger-preview-tls" + tlsSecret.Namespace = stack.Name + tlsSecret.Labels = map[string]string{v1beta1.GatewayBackendTLSSecretLabel: "true"} + }) + JustBeforeEach(func() { + Expect(Create(tlsSecret)).To(Succeed()) + Expect(Create(grpcAPI)).To(Succeed()) + }) + AfterEach(func() { + Expect(client.IgnoreNotFound(Delete(grpcAPI))).To(Succeed()) + Expect(client.IgnoreNotFound(Delete(tlsSecret))).To(Succeed()) + }) + It("Should route HTTP and TLS gRPC to their explicit backends", func() { + Eventually(func(g Gomega) string { + cm := &corev1.ConfigMap{} + g.Expect(LoadResource(stack.Name, "gateway", cm)).To(Succeed()) + return cm.Data["Caddyfile"] + }).Should(SatisfyAll( + ContainSubstring("handle /api/ledger/v3*"), + ContainSubstring("reverse_proxy ledger-preview:9000"), + ContainSubstring("reverse_proxy https://ledger-preview:8888"), + ContainSubstring("tls_trust_pool file /etc/gateway/tls/ledger-preview-tls/ca.crt"), + ContainSubstring("tls_server_name ledger-preview.stack.svc.cluster.local"), + )) + + Eventually(func(g Gomega) *appsv1.Deployment { + deployment := &appsv1.Deployment{} + g.Expect(LoadResource(stack.Name, "gateway", deployment)).To(Succeed()) + return deployment + }).Should(SatisfyAll( + HaveField("Spec.Template.Spec.Volumes", ContainElement(HaveField("VolumeSource.Secret.SecretName", "ledger-preview-tls"))), + HaveField("Spec.Template.Spec.Containers", ContainElement(HaveField("VolumeMounts", ContainElement(SatisfyAll( + HaveField("MountPath", "/etc/gateway/tls/ledger-preview-tls"), + HaveField("ReadOnly", true), + ))))), + )) + + Consistently(func() error { + return LoadResource(stack.Name, "ledger-grpc", &corev1.Service{}) + }, time.Second).Should(BeNotFound()) + }) + It("Should roll out the Gateway when the backend CA changes", func() { + deployment := &appsv1.Deployment{} + Eventually(func(g Gomega) string { + g.Expect(LoadResource(stack.Name, "gateway", deployment)).To(Succeed()) + return deployment.Spec.Template.Annotations["formance.com/backend-tls-secrets-hash"] + }).ShouldNot(BeEmpty()) + initialHash := deployment.Spec.Template.Annotations["formance.com/backend-tls-secrets-hash"] + + Expect(LoadResource(stack.Name, tlsSecret.Name, tlsSecret)).To(Succeed()) + patch := client.MergeFrom(tlsSecret.DeepCopy()) + tlsSecret.Data["ca.crt"] = []byte("renewed CA") + Expect(Patch(tlsSecret, patch)).To(Succeed()) + + Eventually(func(g Gomega) string { + g.Expect(LoadResource(stack.Name, "gateway", deployment)).To(Succeed()) + return deployment.Spec.Template.Annotations["formance.com/backend-tls-secrets-hash"] + }).ShouldNot(Equal(initialHash)) + }) + }) Context("With a consumer on gateway", func() { var ( brokerNatsDSNSettings *v1beta1.Settings diff --git a/internal/tests/ledger_v3_controller_test.go b/internal/tests/ledger_v3_controller_test.go index b3c338d5f..b1a76f364 100644 --- a/internal/tests/ledger_v3_controller_test.go +++ b/internal/tests/ledger_v3_controller_test.go @@ -123,12 +123,10 @@ var _ = Describe("Ledger v3 controller", func() { }).Should(BeTrue()) grpcAPI := &v1beta1.GatewayGRPCAPI{} - Eventually(func(g Gomega) { - g.Expect(Get(core.GetResourceName(core.GetObjectName(stack.Name, "ledger")), grpcAPI)).To(Succeed()) - g.Expect(grpcAPI.Spec.Name).To(Equal("ledger")) - g.Expect(grpcAPI.Spec.Port).To(Equal(int32(8888))) - g.Expect(grpcAPI.Spec.GRPCServices).To(Equal([]string{"ledger.BucketService"})) - }).Should(Succeed()) + Consistently(func() bool { + err := Get(core.GetResourceName(core.GetObjectName(stack.Name, "ledger")), grpcAPI) + return apierrors.IsNotFound(err) + }).Should(BeTrue()) }) It("reuses and normalizes the historical Ledger replica setting", func() { @@ -191,7 +189,7 @@ var _ = Describe("Ledger v3 controller", func() { }).Should(BeFalse()) }) - It("enables TLS only after the managed certificate is ready", func() { + It("configures TLS before the managed certificate is ready", func() { issuer := newLedgerV3Issuer() certificate := newLedgerV3Certificate() cluster := newLedgerV3Cluster() @@ -213,6 +211,14 @@ var _ = Describe("Ledger v3 controller", func() { Expect(err).NotTo(HaveOccurred()) Expect(found).To(BeTrue()) Expect(commonName).To(Equal("ledger-" + stack.Name + "." + stack.Name + ".svc.cluster.local")) + isCA, found, err := unstructured.NestedBool(certificate.Object, "spec", "isCA") + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(isCA).To(BeTrue()) + secretLabels, found, err := unstructured.NestedStringMap(certificate.Object, "spec", "secretTemplate", "labels") + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(secretLabels).To(HaveKeyWithValue(v1beta1.GatewayBackendTLSSecretLabel, "true")) dnsNames, found, err := unstructured.NestedStringSlice(certificate.Object, "spec", "dnsNames") Expect(err).NotTo(HaveOccurred()) @@ -236,12 +242,17 @@ var _ = Describe("Ledger v3 controller", func() { "*.ledger-"+stack.Name+"-headless."+stack.Name+".svc.cluster.local", )) - Eventually(func(g Gomega) bool { + Eventually(func(g Gomega) map[string]any { g.Expect(Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name}, cluster)).To(Succeed()) - _, found, err := unstructured.NestedMap(cluster.Object, "spec", "tls") + tls, found, err := unstructured.NestedMap(cluster.Object, "spec", "tls") g.Expect(err).NotTo(HaveOccurred()) - return found - }).Should(BeFalse()) + g.Expect(found).To(BeTrue()) + return tls + }).Should(Equal(map[string]any{ + "enabled": true, + "secretName": tlsName, + "caSecretKey": "ca.crt", + })) secret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{Namespace: stack.Name, Name: tlsName}, @@ -277,6 +288,27 @@ var _ = Describe("Ledger v3 controller", func() { "secretName": tlsName, "caSecretKey": "ca.crt", })) + + httpAPI := &v1beta1.GatewayHTTPAPI{} + Eventually(func(g Gomega) { + g.Expect(Get(core.GetResourceName(core.GetObjectName(stack.Name, "ledger")), httpAPI)).To(Succeed()) + }).Should(Succeed()) + + grpcAPI := &v1beta1.GatewayGRPCAPI{} + Eventually(func(g Gomega) *v1beta1.GatewayBackendRef { + g.Expect(Get(core.GetResourceName(core.GetObjectName(stack.Name, "ledger")), grpcAPI)).To(Succeed()) + g.Expect(grpcAPI.Spec.Name).To(Equal("ledger")) + g.Expect(grpcAPI.Spec.GRPCServices).To(ConsistOf("ledger.BucketService")) + return grpcAPI.Spec.BackendRef + }).Should(Equal(&v1beta1.GatewayBackendRef{ + Name: "ledger-" + stack.Name, + Port: 8888, + TLS: &v1beta1.GatewayBackendTLS{ + SecretName: tlsName, + CASecretKey: "ca.crt", + ServerName: "ledger-" + stack.Name + "." + stack.Name + ".svc.cluster.local", + }, + })) }) It("configures authentication and monitoring from the existing stack settings", func() { @@ -634,5 +666,184 @@ var _ = Describe("Ledger v3 controller", func() { return ledger.Status.Info }).Should(ContainSubstring("migration required")) }) + + It("rejects a preview version at or below the v3 threshold", func() { + previewSettings := settings.New(uuid.NewString(), "ledger.v3.preview-version", "v3.0.0-alpha", stack.Name) + Expect(Create(previewSettings)).To(Succeed()) + DeferCleanup(func() { + Expect(client.IgnoreNotFound(Delete(previewSettings))).To(Succeed()) + }) + + cluster := newLedgerV3Cluster() + Consistently(func() bool { + err := Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name}, cluster) + return apierrors.IsNotFound(err) + }).Should(BeTrue()) + Eventually(func(g Gomega) string { + g.Expect(LoadResource("", ledger.Name, ledger)).To(Succeed()) + return ledger.Status.Info + }).Should(ContainSubstring("ledger.v3.preview-version must be greater than v3.0.0-alpha")) + }) + + Context("with a Ledger v3 preview version", func() { + var previewSettings *v1beta1.Settings + + BeforeEach(func() { + previewSettings = settings.New(uuid.NewString(), "ledger.v3.preview-version", "v3.0.0-alpha.11", stack.Name) + Expect(Create(previewSettings)).To(Succeed()) + }) + + AfterEach(func() { + Expect(client.IgnoreNotFound(Delete(previewSettings))).To(Succeed()) + }) + + It("runs v2 and a separately routed v3 Cluster at the same time", func() { + EventualDeployment := func() error { + return Get(core.GetNamespacedResourceName(stack.Name, "ledger"), &appsv1.Deployment{}) + } + Eventually(EventualDeployment).Should(Succeed()) + + cluster := newLedgerV3Cluster() + Eventually(func(g Gomega) map[string]any { + g.Expect(Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name}, cluster)).To(Succeed()) + tag, found, err := unstructured.NestedString(cluster.Object, "spec", "image", "tag") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(found).To(BeTrue()) + g.Expect(tag).To(Equal("v3.0.0-alpha.11")) + labels, found, err := unstructured.NestedStringMap(cluster.Object, "spec", "additionalLabels") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(found).To(BeTrue()) + return map[string]any{"metadata": cluster.GetLabels(), "additional": labels} + }).Should(Equal(map[string]any{ + "metadata": map[string]string{ + v1beta1.StackLabel: stack.Name, + "formance.com/ledger-v3-preview": "true", + }, + "additional": map[string]string{ + "app.kubernetes.io/name": "ledger-v3-preview", + "formance.com/ledger-v3-preview": "true", + }, + })) + + Eventually(EventualDeployment).Should(Succeed()) + Eventually(func() error { + return Get(core.GetResourceName(core.GetObjectName(stack.Name, "ledger")), &v1beta1.Database{}) + }).Should(Succeed()) + + httpAPI := &v1beta1.GatewayHTTPAPI{} + Eventually(func(g Gomega) []v1beta1.GatewayHTTPAPIRule { + g.Expect(Get(core.GetResourceName(core.GetObjectName(stack.Name, "ledger")), httpAPI)).To(Succeed()) + return httpAPI.Spec.Rules + }).Should(ContainElement(SatisfyAll( + HaveField("Path", "/v3"), + HaveField("BackendRef.Name", "ledger-"+stack.Name), + HaveField("BackendRef.Port", int32(9000)), + ))) + + certificate := newLedgerV3Certificate() + tlsName := stack.Name + "-tls" + Eventually(func(g Gomega) error { + g.Expect(Get(types.NamespacedName{Namespace: stack.Name, Name: tlsName}, certificate)).To(Succeed()) + return nil + }).Should(Succeed()) + + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: stack.Name, Name: tlsName}, + Data: map[string][]byte{ + "tls.crt": []byte("certificate"), + "tls.key": []byte("private key"), + "ca.crt": []byte("certificate authority"), + }, + } + Expect(Create(secret)).To(Succeed()) + DeferCleanup(func() { + Expect(client.IgnoreNotFound(Delete(secret))).To(Succeed()) + }) + + certificate.Object["status"] = map[string]any{ + "conditions": []any{ + map[string]any{"type": "Ready", "status": "True", "reason": "Ready"}, + }, + } + Expect(TestContext().GetClient().Status().Update(TestContext(), certificate)).To(Succeed()) + + grpcAPI := &v1beta1.GatewayGRPCAPI{} + Eventually(func(g Gomega) *v1beta1.GatewayBackendRef { + g.Expect(Get(core.GetResourceName(core.GetObjectName(stack.Name, "ledger")), grpcAPI)).To(Succeed()) + g.Expect(grpcAPI.Spec.GRPCServices).To(ConsistOf("ledger.BucketService")) + return grpcAPI.Spec.BackendRef + }).Should(Equal(&v1beta1.GatewayBackendRef{ + Name: "ledger-" + stack.Name, + Port: 8888, + TLS: &v1beta1.GatewayBackendTLS{ + SecretName: stack.Name + "-tls", + CASecretKey: "ca.crt", + ServerName: "ledger-" + stack.Name + "." + stack.Name + ".svc.cluster.local", + }, + })) + }) + + It("does not allow the preview Cluster to bypass the migration guard", func() { + cluster := newLedgerV3Cluster() + Eventually(func() error { + return Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name}, cluster) + }).Should(Succeed()) + + Expect(LoadResource("", ledger.Name, ledger)).To(Succeed()) + patch := client.MergeFrom(ledger.DeepCopy()) + ledger.Spec.Version = "v3.0.0-alpha.11" + Expect(Patch(ledger, patch)).To(Succeed()) + + Consistently(func() error { + return Get(core.GetNamespacedResourceName(stack.Name, "ledger"), &appsv1.Deployment{}) + }).Should(Succeed()) + Eventually(func(g Gomega) string { + g.Expect(LoadResource("", ledger.Name, ledger)).To(Succeed()) + return ledger.Status.Info + }).Should(ContainSubstring("migration required")) + }) + + It("removes only the preview resources when the setting is deleted", func() { + cluster := newLedgerV3Cluster() + Eventually(func() error { + return Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name}, cluster) + }).Should(Succeed()) + + certificate := newLedgerV3Certificate() + issuer := newLedgerV3Issuer() + Eventually(func(g Gomega) { + g.Expect(Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name + "-tls"}, certificate)).To(Succeed()) + g.Expect(Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name + "-selfsigned"}, issuer)).To(Succeed()) + }).Should(Succeed()) + + Expect(Delete(previewSettings)).To(Succeed()) + + Eventually(func() error { + return Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name}, cluster) + }).Should(BeNotFound()) + Eventually(func() error { + return Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name + "-tls"}, certificate) + }).Should(BeNotFound()) + Eventually(func() error { + return Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name + "-selfsigned"}, issuer) + }).Should(BeNotFound()) + + Eventually(func() error { + return Get(core.GetNamespacedResourceName(stack.Name, "ledger"), &appsv1.Deployment{}) + }).Should(Succeed()) + Eventually(func() error { + return Get(core.GetResourceName(core.GetObjectName(stack.Name, "ledger")), &v1beta1.Database{}) + }).Should(Succeed()) + + httpAPI := &v1beta1.GatewayHTTPAPI{} + Eventually(func(g Gomega) []v1beta1.GatewayHTTPAPIRule { + g.Expect(Get(core.GetResourceName(core.GetObjectName(stack.Name, "ledger")), httpAPI)).To(Succeed()) + return httpAPI.Spec.Rules + }).ShouldNot(ContainElement(HaveField("Path", "/v3"))) + Eventually(func() error { + return Get(core.GetResourceName(core.GetObjectName(stack.Name, "ledger")), &v1beta1.GatewayGRPCAPI{}) + }).Should(BeNotFound()) + }) + }) }) }) From 3daf0dfca60973062b1f85e891007146c4e13972 Mon Sep 17 00:00:00 2001 From: Maxence Maireaux Date: Wed, 15 Jul 2026 11:19:29 +0200 Subject: [PATCH 14/30] feat(ledger): secure v3 cluster networking --- api/formance.com/v1beta1/ledger_types.go | 2 + .../09-Configuration reference/01-Settings.md | 14 +- .../settings.catalog.json | 2 +- internal/resources/ledgers/v3.go | 36 +++++- internal/resources/ledgers/v3_preview.go | 4 +- internal/resources/ledgers/v3_tls.go | 26 ++-- internal/resources/stacks/networkpolicies.go | 120 +++++++++++++++++- internal/tests/gateway_controller_test.go | 3 + internal/tests/ledger_v3_controller_test.go | 38 ++++++ .../tests/networkpolicy_controller_test.go | 52 +++++++- .../asserts/networkpolicies.yaml | 70 ++++++++++ 11 files changed, 343 insertions(+), 24 deletions(-) diff --git a/api/formance.com/v1beta1/ledger_types.go b/api/formance.com/v1beta1/ledger_types.go index 3062ccc47..1d972a0c6 100644 --- a/api/formance.com/v1beta1/ledger_types.go +++ b/api/formance.com/v1beta1/ledger_types.go @@ -20,6 +20,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +const LedgerV3Label = "formance.com/ledger-v3" + type LedgerSpec struct { ModuleProperties `json:",inline"` StackDependency `json:",inline"` diff --git a/docs/09-Configuration reference/01-Settings.md b/docs/09-Configuration reference/01-Settings.md index 8a3ec2044..af3ae6d37 100644 --- a/docs/09-Configuration reference/01-Settings.md +++ b/docs/09-Configuration reference/01-Settings.md @@ -84,7 +84,7 @@ While we have some basic types (string, number, bool ...), we also have some com | gateway.dns.public.record-type | string | CNAME | DNS record type (e.g., CNAME, A, AAAA) | | gateway.dns.public.provider-specific | Map | alias=true,aws/target-hosted-zone=same-zone | Provider-specific DNS settings for public endpoints | | gateway.dns.public.annotations | Map | | Annotations to add to the public DNSEndpoint resource | -| networkpolicies.enabled | bool | true | Enable network micro-segmentation within a Stack namespace. When enabled, only the Gateway can reach other services | +| networkpolicies.enabled | bool | true | Enable network micro-segmentation within a Stack namespace, including Ledger v3 Raft and migration traffic isolation | ### Postgres URI format @@ -439,8 +439,13 @@ The operator can create Kubernetes NetworkPolicies to enforce network micro-segm - **All ingress traffic is denied by default** to all pods in the namespace - **The Gateway is accessible by everyone** (it is the entry point) - **All other services** (Ledger, Payments, Auth, etc.) **are only accessible from the Gateway** +- **Ledger v3 replicas can communicate with each other** on the Raft and service gRPC ports +- **Ledger v3 can read Ledger v2 only in its own Stack namespace** -Egress traffic is not restricted — pods can still reach DNS, databases, brokers, and external services. +Egress traffic is not restricted. This is required for public Auth issuers and +external monitoring endpoints. The Ledger v3 mirror must use the internal +`http://ledger:8080` Service; ingress isolation on the destination ensures that +only a Ledger v3 pod from the same Stack namespace can reach it directly. NetworkPolicies are owned by the Stack and are automatically garbage-collected when the Stack is deleted. @@ -460,13 +465,16 @@ spec: #### Created NetworkPolicies -When enabled, 3 NetworkPolicies are created in the Stack namespace: +When enabled, 6 NetworkPolicies are created in the Stack namespace: | Name | Effect | |------|--------| | `default-deny-ingress` | Denies all ingress traffic to all pods | | `allow-gateway-ingress` | Allows all ingress traffic to pods labeled `app.kubernetes.io/name: gateway` | | `allow-from-gateway` | Allows ingress traffic from gateway pods to all other pods | +| `allow-ledger-v3-cluster` | Allows Raft and service gRPC traffic between direct Ledger v3 replicas on ports 7777 and 8888 | +| `allow-ledger-v3-preview-cluster` | Allows the same cluster traffic for preview Ledger v3 replicas while preserving their historical pod selector | +| `allow-ledger-v2-from-v3` | Allows Ledger v3 to reach Ledger v2 on port 8080 within the same Stack namespace | Since Kubernetes NetworkPolicies are additive, the Gateway receives both `deny-all` and `allow-all`, making it fully accessible. Other services receive `deny-all` and `allow-from-gateway`, restricting access to Gateway only. diff --git a/docs/09-Configuration reference/settings.catalog.json b/docs/09-Configuration reference/settings.catalog.json index 9373f207e..09f69d2a6 100644 --- a/docs/09-Configuration reference/settings.catalog.json +++ b/docs/09-Configuration reference/settings.catalog.json @@ -582,7 +582,7 @@ "key": "networkpolicies.enabled", "valueType": "bool", "sources": [ - "internal/resources/stacks/networkpolicies.go:14" + "internal/resources/stacks/networkpolicies.go:16" ] }, { diff --git a/internal/resources/ledgers/v3.go b/internal/resources/ledgers/v3.go index 70cc6a190..c4f535c10 100644 --- a/internal/resources/ledgers/v3.go +++ b/internal/resources/ledgers/v3.go @@ -125,13 +125,13 @@ func reconcileV3(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, return core.NewPendingError().WithMessage("migration required before switching Ledger from v2 to v3") } - tlsReady, tlsMessage, err := createOrUpdateV3TLSResources(ctx, stack, ledger, false) + tlsReady, tlsMessage, tlsCAHash, err := createOrUpdateV3TLSResources(ctx, stack, ledger, false) if err != nil { setLedgerV3Condition(ledger, metav1.ConditionFalse, "TLSReconcileFailed", err.Error()) return err } - cluster, err := createOrUpdateV3Cluster(ctx, stack, ledger, version, false) + cluster, err := createOrUpdateV3Cluster(ctx, stack, ledger, version, false, tlsCAHash) if err != nil { setLedgerV3Condition(ledger, metav1.ConditionFalse, "ReconcileFailed", err.Error()) return err @@ -231,7 +231,7 @@ func normalizeLedgerV3Replicas(configured int32) (int32, bool, error) { return configured, false, nil } -func createOrUpdateV3Cluster(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, version string, preview bool) (*unstructured.Unstructured, error) { +func createOrUpdateV3Cluster(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, version string, preview bool, tlsCAHash string) (*unstructured.Unstructured, error) { image, err := registries.GetFormanceImage(ctx, stack, "ledger", version) if err != nil { return nil, err @@ -284,6 +284,7 @@ func createOrUpdateV3Cluster(ctx core.Context, stack *v1beta1.Stack, ledger *v1b labels = map[string]string{} } labels[v1beta1.StackLabel] = stack.Name + labels[v1beta1.LedgerV3Label] = "true" if preview { labels[ledgerV3PreviewLabel] = "true" } else { @@ -316,11 +317,32 @@ func createOrUpdateV3Cluster(ctx core.Context, stack *v1beta1.Stack, ledger *v1b if err := unstructured.SetNestedMap(cluster.Object, ledgerV3TLSSpec(stack.Name), "spec", "tls"); err != nil { return err } + podAnnotations, _, err := unstructured.NestedStringMap(cluster.Object, "spec", "podAnnotations") + if err != nil { + return err + } + if podAnnotations == nil { + podAnnotations = map[string]string{} + } + if tlsCAHash != "" { + podAnnotations[ledgerV3TLSCAHashAnnotation] = tlsCAHash + } else { + delete(podAnnotations, ledgerV3TLSCAHashAnnotation) + } + if len(podAnnotations) > 0 { + if err := unstructured.SetNestedStringMap(cluster.Object, podAnnotations, "spec", "podAnnotations"); err != nil { + return err + } + } else { + unstructured.RemoveNestedField(cluster.Object, "spec", "podAnnotations") + } + additionalLabels := map[string]string{} if preview { - if err := unstructured.SetNestedStringMap(cluster.Object, map[string]string{ - "app.kubernetes.io/name": "ledger-v3-preview", - ledgerV3PreviewLabel: "true", - }, "spec", "additionalLabels"); err != nil { + additionalLabels["app.kubernetes.io/name"] = "ledger-v3-preview" + additionalLabels[ledgerV3PreviewLabel] = "true" + } + if len(additionalLabels) > 0 { + if err := unstructured.SetNestedStringMap(cluster.Object, additionalLabels, "spec", "additionalLabels"); err != nil { return err } } else { diff --git a/internal/resources/ledgers/v3_preview.go b/internal/resources/ledgers/v3_preview.go index d88c63da3..0a86d8a4f 100644 --- a/internal/resources/ledgers/v3_preview.go +++ b/internal/resources/ledgers/v3_preview.go @@ -49,12 +49,12 @@ func reconcileV3Preview(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1. return core.NewPendingError().WithMessage("Ledger v3 preview unavailable: Cluster CRD is not installed") } - tlsReady, tlsMessage, err := createOrUpdateV3TLSResources(ctx, stack, ledger, true) + tlsReady, tlsMessage, tlsCAHash, err := createOrUpdateV3TLSResources(ctx, stack, ledger, true) if err != nil { setLedgerV3PreviewCondition(ledger, metav1.ConditionFalse, "TLSReconcileFailed", err.Error()) return err } - cluster, err := createOrUpdateV3Cluster(ctx, stack, ledger, version, true) + cluster, err := createOrUpdateV3Cluster(ctx, stack, ledger, version, true, tlsCAHash) if err != nil { setLedgerV3PreviewCondition(ledger, metav1.ConditionFalse, "ReconcileFailed", err.Error()) return err diff --git a/internal/resources/ledgers/v3_tls.go b/internal/resources/ledgers/v3_tls.go index d3fc055d0..bdb294904 100644 --- a/internal/resources/ledgers/v3_tls.go +++ b/internal/resources/ledgers/v3_tls.go @@ -1,6 +1,8 @@ package ledgers import ( + "crypto/sha256" + "encoding/hex" "fmt" corev1 "k8s.io/api/core/v1" @@ -18,6 +20,7 @@ const ( ledgerV3TLSCertificateDuration = "87600h" ledgerV3TLSCertificateRenewBefore = "720h" ledgerV3TLSCASecretKey = "ca.crt" + ledgerV3TLSCAHashAnnotation = "formance.com/ledger-v3-ca-sha256" ) var ( @@ -41,9 +44,9 @@ func newLedgerV3Resource(gvk schema.GroupVersionKind) *unstructured.Unstructured return resource } -func createOrUpdateV3TLSResources(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, preview bool) (bool, string, error) { +func createOrUpdateV3TLSResources(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, preview bool) (bool, string, string, error) { if !ledgerV3CertManagerAvailable { - return false, "cert-manager Issuer and Certificate CRDs are not installed", nil + return false, "cert-manager Issuer and Certificate CRDs are not installed", "", nil } issuer := newLedgerV3Resource(ledgerV3IssuerGVK) @@ -56,7 +59,7 @@ func createOrUpdateV3TLSResources(ctx core.Context, stack *v1beta1.Stack, ledger } return unstructured.SetNestedMap(issuer.Object, map[string]any{}, "spec", "selfSigned") }); err != nil { - return false, "", err + return false, "", "", err } certificate := newLedgerV3Resource(ledgerV3CertificateGVK) @@ -70,29 +73,34 @@ func createOrUpdateV3TLSResources(ctx core.Context, stack *v1beta1.Stack, ledger certificate.Object["spec"] = ledgerV3CertificateSpec(stack.Name) return nil }); err != nil { - return false, "", err + return false, "", "", err } ready, message, err := ledgerV3CertificateReady(certificate) if err != nil || !ready { - return ready, message, err + return ready, message, "", err } secret := &corev1.Secret{} err = ctx.GetClient().Get(ctx, types.NamespacedName{Namespace: stack.Name, Name: ledgerV3TLSName(stack.Name)}, secret) if apierrors.IsNotFound(err) { - return false, fmt.Sprintf("TLS Secret %s/%s does not exist", stack.Name, ledgerV3TLSName(stack.Name)), nil + return false, fmt.Sprintf("TLS Secret %s/%s does not exist", stack.Name, ledgerV3TLSName(stack.Name)), "", nil } if err != nil { - return false, "", err + return false, "", "", err } for _, key := range []string{corev1.TLSCertKey, corev1.TLSPrivateKeyKey, ledgerV3TLSCASecretKey} { if len(secret.Data[key]) == 0 { - return false, fmt.Sprintf("TLS Secret %s/%s is missing %q", stack.Name, secret.Name, key), nil + return false, fmt.Sprintf("TLS Secret %s/%s is missing %q", stack.Name, secret.Name, key), "", nil } } - return true, "Certificate and TLS Secret are ready", nil + return true, "Certificate and TLS Secret are ready", ledgerV3TLSCAHash(secret.Data[ledgerV3TLSCASecretKey]), nil +} + +func ledgerV3TLSCAHash(ca []byte) string { + sum := sha256.Sum256(ca) + return hex.EncodeToString(sum[:]) } func setLedgerV3TLSResourceMetadata(resource *unstructured.Unstructured, stackName string, preview bool) { diff --git a/internal/resources/stacks/networkpolicies.go b/internal/resources/stacks/networkpolicies.go index 679f9e0a0..ad2fb3397 100644 --- a/internal/resources/stacks/networkpolicies.go +++ b/internal/resources/stacks/networkpolicies.go @@ -1,9 +1,11 @@ package stacks import ( + corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" "github.com/formancehq/operator/v3/api/formance.com/v1beta1" . "github.com/formancehq/operator/v3/internal/core" @@ -96,11 +98,104 @@ func createNetworkPolicies(ctx Context, stack *v1beta1.Stack) error { return err } + // 4. allow-ledger-v3-cluster: allow Raft and service gRPC between direct + // Ledger v3 replicas. These labels are part of the Ledger Operator's stable + // selector and also work for clusters created before this policy. + if _, _, err := CreateOrUpdate[*networkingv1.NetworkPolicy](ctx, + types.NamespacedName{ + Namespace: stack.Name, + Name: "allow-ledger-v3-cluster", + }, + func(np *networkingv1.NetworkPolicy) error { + selector := directLedgerV3Selector(stack.Name) + np.Spec = networkingv1.NetworkPolicySpec{ + PodSelector: selector, + PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeIngress}, + Ingress: []networkingv1.NetworkPolicyIngressRule{ + { + From: []networkingv1.NetworkPolicyPeer{ + {PodSelector: &selector}, + }, + Ports: networkPolicyTCPPorts(7777, 8888), + }, + }, + } + return nil + }, + WithController[*networkingv1.NetworkPolicy](ctx.GetScheme(), stack), + ); err != nil { + return err + } + + // 5. allow-ledger-v3-preview-cluster: use the historical preview label so + // existing clusters do not require immutable selector changes. + if _, _, err := CreateOrUpdate[*networkingv1.NetworkPolicy](ctx, + types.NamespacedName{ + Namespace: stack.Name, + Name: "allow-ledger-v3-preview-cluster", + }, + func(np *networkingv1.NetworkPolicy) error { + selector := previewLedgerV3Selector() + np.Spec = networkingv1.NetworkPolicySpec{ + PodSelector: selector, + PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeIngress}, + Ingress: []networkingv1.NetworkPolicyIngressRule{{ + From: []networkingv1.NetworkPolicyPeer{{PodSelector: &selector}}, + Ports: networkPolicyTCPPorts(7777, 8888), + }}, + } + return nil + }, + WithController[*networkingv1.NetworkPolicy](ctx.GetScheme(), stack), + ); err != nil { + return err + } + + // 6. allow-ledger-v2-from-v3: let the preview cluster read only the legacy + // Ledger pods in the same Stack namespace. A PodSelector without a + // NamespaceSelector never matches pods from another namespace. + if _, _, err := CreateOrUpdate[*networkingv1.NetworkPolicy](ctx, + types.NamespacedName{ + Namespace: stack.Name, + Name: "allow-ledger-v2-from-v3", + }, + func(np *networkingv1.NetworkPolicy) error { + directSelector := directLedgerV3Selector(stack.Name) + previewSelector := previewLedgerV3Selector() + np.Spec = networkingv1.NetworkPolicySpec{ + PodSelector: metav1.LabelSelector{MatchLabels: map[string]string{ + "app.kubernetes.io/name": "ledger", + }}, + PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeIngress}, + Ingress: []networkingv1.NetworkPolicyIngressRule{ + { + From: []networkingv1.NetworkPolicyPeer{ + {PodSelector: &directSelector}, + {PodSelector: &previewSelector}, + }, + Ports: networkPolicyTCPPorts(8080), + }, + }, + } + return nil + }, + WithController[*networkingv1.NetworkPolicy](ctx.GetScheme(), stack), + ); err != nil { + return err + } + return nil } func deleteNetworkPolicies(ctx Context, stack *v1beta1.Stack) error { - for _, name := range []string{"default-deny-ingress", "allow-gateway-ingress", "allow-from-gateway"} { + for _, name := range []string{ + "default-deny-ingress", + "allow-gateway-ingress", + "allow-from-gateway", + "allow-ledger-v3-cluster", + "allow-ledger-v3-preview-cluster", + "allow-ledger-v2-from-v3", + } { if err := DeleteIfExists[*networkingv1.NetworkPolicy](ctx, types.NamespacedName{ Namespace: stack.Name, Name: name, @@ -110,3 +205,26 @@ func deleteNetworkPolicies(ctx Context, stack *v1beta1.Stack) error { } return nil } + +func directLedgerV3Selector(stackName string) metav1.LabelSelector { + return metav1.LabelSelector{MatchLabels: map[string]string{ + "app.kubernetes.io/instance": stackName, + "app.kubernetes.io/name": "ledger", + }} +} + +func previewLedgerV3Selector() metav1.LabelSelector { + return metav1.LabelSelector{MatchLabels: map[string]string{ + "formance.com/ledger-v3-preview": "true", + }} +} + +func networkPolicyTCPPorts(ports ...int) []networkingv1.NetworkPolicyPort { + protocol := corev1.ProtocolTCP + ret := make([]networkingv1.NetworkPolicyPort, 0, len(ports)) + for _, port := range ports { + value := intstr.FromInt(port) + ret = append(ret, networkingv1.NetworkPolicyPort{Protocol: &protocol, Port: &value}) + } + return ret +} diff --git a/internal/tests/gateway_controller_test.go b/internal/tests/gateway_controller_test.go index d18aca371..7aacda66a 100644 --- a/internal/tests/gateway_controller_test.go +++ b/internal/tests/gateway_controller_test.go @@ -363,6 +363,9 @@ var _ = Describe("GatewayController", func() { tlsSecret.Labels = map[string]string{v1beta1.GatewayBackendTLSSecretLabel: "true"} }) JustBeforeEach(func() { + Eventually(func() error { + return LoadResource("", stack.Name, &corev1.Namespace{}) + }).Should(Succeed()) Expect(Create(tlsSecret)).To(Succeed()) Expect(Create(grpcAPI)).To(Succeed()) }) diff --git a/internal/tests/ledger_v3_controller_test.go b/internal/tests/ledger_v3_controller_test.go index b1a76f364..c60bd6aaa 100644 --- a/internal/tests/ledger_v3_controller_test.go +++ b/internal/tests/ledger_v3_controller_test.go @@ -1,6 +1,9 @@ package tests_test import ( + "crypto/sha256" + "fmt" + "github.com/google/uuid" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -55,6 +58,10 @@ func newLedgerV3Certificate() *unstructured.Unstructured { return certificate } +func sha256Hex(data []byte) string { + return fmt.Sprintf("%x", sha256.Sum256(data)) +} + var _ = Describe("Ledger v3 controller", func() { var ( stack *v1beta1.Stack @@ -103,6 +110,12 @@ var _ = Describe("Ledger v3 controller", func() { Expect(found).To(BeTrue()) Expect(tag).To(Equal("v3.0.0-alpha.1")) + additionalLabels, found, err := unstructured.NestedStringMap(cluster.Object, "spec", "additionalLabels") + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeFalse()) + Expect(additionalLabels).To(BeNil()) + Expect(cluster.GetLabels()).To(HaveKeyWithValue(v1beta1.LedgerV3Label, "true")) + _, found, err = unstructured.NestedMap(cluster.Object, "spec", "auth") Expect(err).NotTo(HaveOccurred()) Expect(found).To(BeFalse()) @@ -289,6 +302,30 @@ var _ = Describe("Ledger v3 controller", func() { "caSecretKey": "ca.crt", })) + Eventually(func(g Gomega) string { + g.Expect(Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name}, cluster)).To(Succeed()) + annotations, found, err := unstructured.NestedStringMap(cluster.Object, "spec", "podAnnotations") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(found).To(BeTrue()) + return annotations["formance.com/ledger-v3-ca-sha256"] + }).Should(Equal(sha256Hex(secret.Data["ca.crt"]))) + + // A cert-manager CA rotation must change the pod template so that Ledger + // reloads the client trust pool used for follower-to-leader forwarding. + rotatedCA := []byte("rotated certificate authority") + secret.Data["ca.crt"] = rotatedCA + Expect(Update(secret)).To(Succeed()) + certificate.SetAnnotations(map[string]string{"tests.formance.com/rotation": uuid.NewString()}) + Expect(Update(certificate)).To(Succeed()) + + Eventually(func(g Gomega) string { + g.Expect(Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name}, cluster)).To(Succeed()) + annotations, found, err := unstructured.NestedStringMap(cluster.Object, "spec", "podAnnotations") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(found).To(BeTrue()) + return annotations["formance.com/ledger-v3-ca-sha256"] + }).Should(Equal(sha256Hex(rotatedCA))) + httpAPI := &v1beta1.GatewayHTTPAPI{} Eventually(func(g Gomega) { g.Expect(Get(core.GetResourceName(core.GetObjectName(stack.Name, "ledger")), httpAPI)).To(Succeed()) @@ -717,6 +754,7 @@ var _ = Describe("Ledger v3 controller", func() { }).Should(Equal(map[string]any{ "metadata": map[string]string{ v1beta1.StackLabel: stack.Name, + v1beta1.LedgerV3Label: "true", "formance.com/ledger-v3-preview": "true", }, "additional": map[string]string{ diff --git a/internal/tests/networkpolicy_controller_test.go b/internal/tests/networkpolicy_controller_test.go index b3c3247be..a4597a9f3 100644 --- a/internal/tests/networkpolicy_controller_test.go +++ b/internal/tests/networkpolicy_controller_test.go @@ -50,7 +50,7 @@ var _ = Describe("NetworkPolicyController", func() { Expect(Delete(enabledSetting)).To(Succeed()) }) - It("Should create 3 NetworkPolicies controlled by the stack", func() { + It("Should create NetworkPolicies controlled by the stack", func() { // Check default-deny-ingress Eventually(func(g Gomega) { np := &networkingv1.NetworkPolicy{} @@ -80,6 +80,47 @@ var _ = Describe("NetworkPolicyController", func() { g.Expect(np.Spec.Ingress[0].From).To(HaveLen(1)) g.Expect(np.Spec.Ingress[0].From[0].PodSelector.MatchLabels).To(HaveKeyWithValue("app.kubernetes.io/name", "gateway")) }).Should(Succeed()) + + // Direct Ledger v3 replicas can communicate over Raft and service gRPC. + Eventually(func(g Gomega) { + np := &networkingv1.NetworkPolicy{} + g.Expect(LoadResource(stack.Name, "allow-ledger-v3-cluster", np)).To(Succeed()) + g.Expect(np).To(BeControlledBy(stack)) + g.Expect(np.Spec.PodSelector.MatchLabels).To(Equal(map[string]string{ + "app.kubernetes.io/instance": stack.Name, + "app.kubernetes.io/name": "ledger", + })) + g.Expect(np.Spec.Ingress).To(HaveLen(1)) + g.Expect(np.Spec.Ingress[0].From).To(HaveLen(1)) + g.Expect(np.Spec.Ingress[0].From[0].NamespaceSelector).To(BeNil()) + g.Expect(np.Spec.Ingress[0].From[0].PodSelector.MatchLabels).To(Equal(np.Spec.PodSelector.MatchLabels)) + g.Expect(np.Spec.Ingress[0].Ports).To(HaveLen(2)) + }).Should(Succeed()) + + // Existing preview clusters keep their historical immutable selector. + Eventually(func(g Gomega) { + np := &networkingv1.NetworkPolicy{} + g.Expect(LoadResource(stack.Name, "allow-ledger-v3-preview-cluster", np)).To(Succeed()) + g.Expect(np).To(BeControlledBy(stack)) + g.Expect(np.Spec.PodSelector.MatchLabels).To(HaveKeyWithValue("formance.com/ledger-v3-preview", "true")) + g.Expect(np.Spec.Ingress[0].From[0].PodSelector.MatchLabels).To(Equal(np.Spec.PodSelector.MatchLabels)) + }).Should(Succeed()) + + // Ledger v3 can reach only the legacy Ledger from the same namespace. + Eventually(func(g Gomega) { + np := &networkingv1.NetworkPolicy{} + g.Expect(LoadResource(stack.Name, "allow-ledger-v2-from-v3", np)).To(Succeed()) + g.Expect(np).To(BeControlledBy(stack)) + g.Expect(np.Spec.PodSelector.MatchLabels).To(HaveKeyWithValue("app.kubernetes.io/name", "ledger")) + g.Expect(np.Spec.Ingress).To(HaveLen(1)) + g.Expect(np.Spec.Ingress[0].From).To(HaveLen(2)) + g.Expect(np.Spec.Ingress[0].From[0].NamespaceSelector).To(BeNil()) + g.Expect(np.Spec.Ingress[0].From[0].PodSelector.MatchLabels).To(HaveKeyWithValue("app.kubernetes.io/instance", stack.Name)) + g.Expect(np.Spec.Ingress[0].From[1].PodSelector.MatchLabels).To(HaveKeyWithValue("formance.com/ledger-v3-preview", "true")) + g.Expect(np.Spec.Ingress[0].Ports).To(HaveLen(1)) + g.Expect(np.Spec.Ingress[0].Ports[0].Port.IntValue()).To(Equal(8080)) + }).Should(Succeed()) + }) Context("Then disabling networkpolicies", func() { @@ -104,6 +145,15 @@ var _ = Describe("NetworkPolicyController", func() { Eventually(func() error { return LoadResource(stack.Name, "allow-from-gateway", &networkingv1.NetworkPolicy{}) }).Should(BeNotFound()) + Eventually(func() error { + return LoadResource(stack.Name, "allow-ledger-v3-cluster", &networkingv1.NetworkPolicy{}) + }).Should(BeNotFound()) + Eventually(func() error { + return LoadResource(stack.Name, "allow-ledger-v3-preview-cluster", &networkingv1.NetworkPolicy{}) + }).Should(BeNotFound()) + Eventually(func() error { + return LoadResource(stack.Name, "allow-ledger-v2-from-v3", &networkingv1.NetworkPolicy{}) + }).Should(BeNotFound()) }) }) }) diff --git a/tests/e2e/chainsaw/02-stack-lifecycle/asserts/networkpolicies.yaml b/tests/e2e/chainsaw/02-stack-lifecycle/asserts/networkpolicies.yaml index 27de21aaa..57c399a00 100644 --- a/tests/e2e/chainsaw/02-stack-lifecycle/asserts/networkpolicies.yaml +++ b/tests/e2e/chainsaw/02-stack-lifecycle/asserts/networkpolicies.yaml @@ -48,3 +48,73 @@ spec: app.kubernetes.io/name: gateway policyTypes: - Ingress +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-ledger-v3-cluster + namespace: chainsaw-networkpolicies +spec: + podSelector: + matchLabels: + app.kubernetes.io/instance: chainsaw-networkpolicies + app.kubernetes.io/name: ledger + ingress: + - from: + - podSelector: + matchLabels: + app.kubernetes.io/instance: chainsaw-networkpolicies + app.kubernetes.io/name: ledger + ports: + - protocol: TCP + port: 7777 + - protocol: TCP + port: 8888 + policyTypes: + - Ingress +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-ledger-v3-preview-cluster + namespace: chainsaw-networkpolicies +spec: + podSelector: + matchLabels: + formance.com/ledger-v3-preview: "true" + ingress: + - from: + - podSelector: + matchLabels: + formance.com/ledger-v3-preview: "true" + ports: + - protocol: TCP + port: 7777 + - protocol: TCP + port: 8888 + policyTypes: + - Ingress +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-ledger-v2-from-v3 + namespace: chainsaw-networkpolicies +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: ledger + ingress: + - from: + - podSelector: + matchLabels: + app.kubernetes.io/instance: chainsaw-networkpolicies + app.kubernetes.io/name: ledger + - podSelector: + matchLabels: + formance.com/ledger-v3-preview: "true" + ports: + - protocol: TCP + port: 8080 + policyTypes: + - Ingress From 1b0cb6848336e9c3b3338465457e72418054a454 Mon Sep 17 00:00:00 2001 From: Maxence Maireaux Date: Wed, 15 Jul 2026 12:38:38 +0200 Subject: [PATCH 15/30] feat(ledger): add stack-targeted v3 configuration --- PROJECT | 6 + .../v1beta1/ledgerconfiguration_types.go | 70 + .../v1beta1/zz_generated.deepcopy.go | 79 + .../formance.com_ledgerconfigurations.yaml | 4045 ++++++++++++++++ config/crd/kustomization.yaml | 1 + .../rbac/ledgerconfiguration_editor_role.yaml | 25 + .../rbac/ledgerconfiguration_viewer_role.yaml | 21 + config/rbac/role.yaml | 8 + ...mance.com_v1beta1_ledgerconfiguration.yaml | 14 + config/samples/kustomization.yaml | 1 + .../02-Custom Resource Definitions.md | 60 + .../settings.catalog.json | 8 +- go.mod | 5 +- go.sum | 2 + ...ion_ledgerconfigurations.formance.com.yaml | 4048 +++++++++++++++++ ..._v1_clusterrole_formance-manager-role.yaml | 8 + internal/core/setup.go | 14 + internal/resources/ledgers/init.go | 1 + internal/resources/ledgers/v3.go | 287 +- internal/resources/ledgers/v3_spec.go | 185 + internal/resources/ledgers/v3_spec_test.go | 151 + internal/resources/ledgers/v3_tls.go | 8 - .../crds/ledger.formance.com_clusters.yaml | 7 + internal/tests/ledger_v3_controller_test.go | 205 + tools/kubectl-stacks/go.mod | 7 +- tools/kubectl-stacks/go.sum | 6 +- 26 files changed, 9062 insertions(+), 210 deletions(-) create mode 100644 api/formance.com/v1beta1/ledgerconfiguration_types.go create mode 100644 config/crd/bases/formance.com_ledgerconfigurations.yaml create mode 100644 config/rbac/ledgerconfiguration_editor_role.yaml create mode 100644 config/rbac/ledgerconfiguration_viewer_role.yaml create mode 100644 config/samples/formance.com_v1beta1_ledgerconfiguration.yaml create mode 100644 helm/crds/templates/crds/apiextensions.k8s.io_v1_customresourcedefinition_ledgerconfigurations.formance.com.yaml create mode 100644 internal/resources/ledgers/v3_spec.go create mode 100644 internal/resources/ledgers/v3_spec_test.go diff --git a/PROJECT b/PROJECT index 714cff25c..95a5376df 100644 --- a/PROJECT +++ b/PROJECT @@ -173,4 +173,10 @@ resources: kind: Broker path: github.com/formancehq/operator/api/formance.com/v1beta1 version: v1beta1 +- api: + crdVersion: v1 + group: formance.com + kind: LedgerConfiguration + path: github.com/formancehq/operator/api/formance.com/v1beta1 + version: v1beta1 version: "3" diff --git a/api/formance.com/v1beta1/ledgerconfiguration_types.go b/api/formance.com/v1beta1/ledgerconfiguration_types.go new file mode 100644 index 000000000..80ca33841 --- /dev/null +++ b/api/formance.com/v1beta1/ledgerconfiguration_types.go @@ -0,0 +1,70 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + ledgerv1alpha1 "github.com/formancehq/ledger/misc/operator/api/v1alpha1" +) + +const DefaultLedgerConfigurationName = "default" + +type LedgerConfigurationSpec struct { + // Stacks on which the configuration is applied. Can contain `*` to + // indicate a wildcard, following the same convention as Settings. + // +optional + Stacks []string `json:"stacks,omitempty"` + + // Cluster is the base Ledger v3 Cluster specification. Stack-specific + // Settings and values owned by the Operator are applied on top of it. + // +optional + Cluster ledgerv1alpha1.ClusterSpec `json:"cluster,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Cluster +// LedgerConfiguration defines the base specification applied to every Ledger v3 +// Cluster targeted by spec.stacks. A configuration targeting a stack by name +// takes priority over a configuration targeting all stacks with `*`. +type LedgerConfiguration struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec LedgerConfigurationSpec `json:"spec,omitempty"` +} + +func (in *LedgerConfiguration) GetStacks() []string { + return in.Spec.Stacks +} + +func (in *LedgerConfiguration) IsWildcard() bool { + return len(in.Spec.Stacks) == 1 && in.Spec.Stacks[0] == "*" +} + +// +kubebuilder:object:root=true + +// LedgerConfigurationList contains a list of LedgerConfiguration +type LedgerConfigurationList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []LedgerConfiguration `json:"items"` +} + +func init() { + SchemeBuilder.Register(&LedgerConfiguration{}, &LedgerConfigurationList{}) +} diff --git a/api/formance.com/v1beta1/zz_generated.deepcopy.go b/api/formance.com/v1beta1/zz_generated.deepcopy.go index a1d01e41e..45a040053 100644 --- a/api/formance.com/v1beta1/zz_generated.deepcopy.go +++ b/api/formance.com/v1beta1/zz_generated.deepcopy.go @@ -1383,6 +1383,85 @@ func (in *Ledger) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LedgerConfiguration) DeepCopyInto(out *LedgerConfiguration) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LedgerConfiguration. +func (in *LedgerConfiguration) DeepCopy() *LedgerConfiguration { + if in == nil { + return nil + } + out := new(LedgerConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *LedgerConfiguration) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LedgerConfigurationList) DeepCopyInto(out *LedgerConfigurationList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]LedgerConfiguration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LedgerConfigurationList. +func (in *LedgerConfigurationList) DeepCopy() *LedgerConfigurationList { + if in == nil { + return nil + } + out := new(LedgerConfigurationList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *LedgerConfigurationList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LedgerConfigurationSpec) DeepCopyInto(out *LedgerConfigurationSpec) { + *out = *in + if in.Stacks != nil { + in, out := &in.Stacks, &out.Stacks + *out = make([]string, len(*in)) + copy(*out, *in) + } + in.Cluster.DeepCopyInto(&out.Cluster) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LedgerConfigurationSpec. +func (in *LedgerConfigurationSpec) DeepCopy() *LedgerConfigurationSpec { + if in == nil { + return nil + } + out := new(LedgerConfigurationSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LedgerList) DeepCopyInto(out *LedgerList) { *out = *in diff --git a/config/crd/bases/formance.com_ledgerconfigurations.yaml b/config/crd/bases/formance.com_ledgerconfigurations.yaml new file mode 100644 index 000000000..77f3d45f0 --- /dev/null +++ b/config/crd/bases/formance.com_ledgerconfigurations.yaml @@ -0,0 +1,4045 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.18.0 + name: ledgerconfigurations.formance.com +spec: + group: formance.com + names: + kind: LedgerConfiguration + listKind: LedgerConfigurationList + plural: ledgerconfigurations + singular: ledgerconfiguration + scope: Cluster + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: |- + LedgerConfiguration defines the base specification applied to every Ledger v3 + Cluster targeted by spec.stacks. A configuration targeting a stack by name + takes priority over a configuration targeting all stacks with `*`. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + cluster: + description: |- + Cluster is the base Ledger v3 Cluster specification. Stack-specific + Settings and values owned by the Operator are applied on top of it. + properties: + additionalLabels: + additionalProperties: + type: string + description: |- + AdditionalLabels are merged on top of the default selector labels + (app.kubernetes.io/name, app.kubernetes.io/instance) on every owned + resource AND on the pod template / Service selectors. Keys that collide + with a default override it; the app.kubernetes.io/managed-by ownership + label is dropped from the merge and stays operator-owned everywhere + (top-level objects AND pods). + + Use this to escape an unrelated Service whose selector accidentally + matches our pods. Selector fields on Service / StatefulSet are immutable + after creation: changing this field on an existing cluster will be + rejected by the operator (a SelectorImmutable=False condition is set). + type: object + admissionMetrics: + description: AdmissionMetrics enables admission path metrics. + type: boolean + affinity: + description: Affinity rules for pod scheduling. + properties: + nodeAffinity: + description: Describes node affinity scheduling rules for + the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated with + the corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching the + corresponding nodeSelectorTerm, in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. + The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling rules (e.g. + co-locate this pod in the same node, zone, etc. as some + other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling rules + (e.g. avoid putting this pod in the same node, zone, etc. + as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and subtracting + "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + auth: + description: Auth holds authentication and authorization configuration. + properties: + anonymousScopes: + description: |- + AnonymousScopes lists the granular scopes granted to requests that arrive + without a bearer token. Wildcards "*:read" / "*:write" expand to every + granular scope with that suffix. The canonical writes-only configuration + is `["*:read"]`: reads are public, writes still require a valid token. + Empty (default) preserves the strict behavior — every request must + authenticate. Invalid tokens are still rejected with 401 regardless of + this setting; only the *absence* of a token triggers the fallback. + items: + type: string + type: array + checkScopes: + description: CheckScopes enables scope checking. + type: boolean + enabled: + description: Enabled enables authentication. + type: boolean + issuer: + description: Issuer is the OIDC token issuer URL (for backward + compatibility). + type: string + issuers: + description: Issuers is a list of trusted OIDC issuers. + items: + type: string + type: array + readKeySetMaxRetries: + description: ReadKeySetMaxRetries is the maximum number of + retries for reading key sets. + format: int32 + type: integer + scopeMapping: + additionalProperties: + items: + type: string + type: array + description: |- + ScopeMapping maps virtual scopes (e.g. "ledger:read") to granular scopes. + When provided, overrides the default mapping and --auth-service is ignored for scope resolution. + type: object + service: + description: Service is the scope prefix (e.g. "ledger" for + "ledger:read"). + type: string + type: object + bindAddr: + default: 0.0.0.0:7777 + description: BindAddr is the Raft transport bind address. + type: string + x-kubernetes-validations: + - message: bindAddr is immutable once set + rule: oldSelf == '' || self == oldSelf + bloom: + description: Bloom filter configuration per attribute type. + properties: + boundaries: + description: Boundaries bloom filter configuration. + properties: + expectedKeys: + description: ExpectedKeys is the expected number of unique + keys (0 to disable this filter). + format: int64 + type: integer + fpRate: + description: FPRate is the false positive rate (0.0-1.0). + type: string + type: object + ledgerMetadata: + description: LedgerMetadata bloom filter configuration. + properties: + expectedKeys: + description: ExpectedKeys is the expected number of unique + keys (0 to disable this filter). + format: int64 + type: integer + fpRate: + description: FPRate is the false positive rate (0.0-1.0). + type: string + type: object + ledgers: + description: Ledgers bloom filter configuration. + properties: + expectedKeys: + description: ExpectedKeys is the expected number of unique + keys (0 to disable this filter). + format: int64 + type: integer + fpRate: + description: FPRate is the false positive rate (0.0-1.0). + type: string + type: object + metadata: + description: Metadata bloom filter configuration. + properties: + expectedKeys: + description: ExpectedKeys is the expected number of unique + keys (0 to disable this filter). + format: int64 + type: integer + fpRate: + description: FPRate is the false positive rate (0.0-1.0). + type: string + type: object + numscriptContents: + description: NumscriptContents bloom filter configuration. + properties: + expectedKeys: + description: ExpectedKeys is the expected number of unique + keys (0 to disable this filter). + format: int64 + type: integer + fpRate: + description: FPRate is the false positive rate (0.0-1.0). + type: string + type: object + numscriptVersions: + description: NumscriptVersions bloom filter configuration. + properties: + expectedKeys: + description: ExpectedKeys is the expected number of unique + keys (0 to disable this filter). + format: int64 + type: integer + fpRate: + description: FPRate is the false positive rate (0.0-1.0). + type: string + type: object + preparedQueries: + description: PreparedQueries bloom filter configuration. + properties: + expectedKeys: + description: ExpectedKeys is the expected number of unique + keys (0 to disable this filter). + format: int64 + type: integer + fpRate: + description: FPRate is the false positive rate (0.0-1.0). + type: string + type: object + references: + description: References bloom filter configuration. + properties: + expectedKeys: + description: ExpectedKeys is the expected number of unique + keys (0 to disable this filter). + format: int64 + type: integer + fpRate: + description: FPRate is the false positive rate (0.0-1.0). + type: string + type: object + sinkConfigs: + description: SinkConfigs bloom filter configuration. + properties: + expectedKeys: + description: ExpectedKeys is the expected number of unique + keys (0 to disable this filter). + format: int64 + type: integer + fpRate: + description: FPRate is the false positive rate (0.0-1.0). + type: string + type: object + transactions: + description: Transactions bloom filter configuration. + properties: + expectedKeys: + description: ExpectedKeys is the expected number of unique + keys (0 to disable this filter). + format: int64 + type: integer + fpRate: + description: FPRate is the false positive rate (0.0-1.0). + type: string + type: object + volumes: + description: Volumes bloom filter configuration. + properties: + expectedKeys: + description: ExpectedKeys is the expected number of unique + keys (0 to disable this filter). + format: int64 + type: integer + fpRate: + description: FPRate is the false positive rate (0.0-1.0). + type: string + type: object + type: object + cache: + description: Cache configuration. + properties: + rotationThreshold: + description: |- + RotationThreshold is the number of Raft log entries before rotating cache generations. + Changes trigger a rolling restart; convergence is deterministic via Raft (applyClusterConfig). + format: int32 + minimum: 1 + type: integer + type: object + clusterID: + default: default + description: ClusterID for inter-node communication validation. + type: string + x-kubernetes-validations: + - message: clusterID is immutable once set + rule: oldSelf == '' || self == oldSelf + coldStorage: + description: ColdStorage configuration for chapter archival. + properties: + bucketId: + description: BucketID is the shared namespace prefix for archives. + type: string + driver: + description: 'Driver is the storage driver: "filesystem" or + "s3".' + type: string + path: + description: Path is the base path for filesystem driver. + type: string + s3: + description: S3 configuration. + properties: + bucket: + description: Bucket is the S3 bucket name. + type: string + endpoint: + description: Endpoint is a custom S3 endpoint (for MinIO). + type: string + region: + description: Region is the AWS region. + type: string + type: object + type: object + dataDir: + default: /data/app + description: DataDir is the application data directory. + type: string + x-kubernetes-validations: + - message: dataDir is immutable once set + rule: oldSelf == '' || self == oldSelf + debug: + description: |- + Debug enables debug logging. Equivalent to LogLevel="debug". + LogLevel takes precedence when both are set; prefer LogLevel for new + manifests since it also unlocks the trace level. + type: boolean + dnsEndpoint: + description: DNSEndpoint configuration for ExternalDNS. + properties: + annotations: + additionalProperties: + type: string + description: Annotations to add to the DNSEndpoint resource. + type: object + enabled: + description: Enabled enables the DNSEndpoint resource. + type: boolean + endpoints: + description: Endpoints is the list of DNS endpoint entries. + items: + description: DNSEndpointEntry defines a single DNS endpoint. + properties: + dnsName: + description: DNSName is the hostname for the DNS record. + type: string + providerSpecific: + description: ProviderSpecific holds provider-specific + properties. + items: + description: ProviderSpecificProperty defines a provider-specific + key-value pair. + properties: + name: + description: Name is the property name. + type: string + value: + description: Value is the property value. + type: string + required: + - name + - value + type: object + type: array + recordTTL: + description: RecordTTL is the TTL in seconds for the + DNS record. + format: int64 + type: integer + recordType: + description: RecordType is the DNS record type (e.g., + CNAME, A). Defaults to CNAME. + type: string + targets: + description: Targets is the list of target hostnames + or IPs. + items: + type: string + type: array + required: + - dnsName + - targets + type: object + type: array + type: object + x-kubernetes-validations: + - message: endpoints are required when dnsEndpoint is enabled + rule: '!self.enabled || size(self.endpoints) > 0' + extraEnv: + description: ExtraEnv is a list of additional environment variables + to inject into ledger containers. + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the + exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + goMemLimitRatio: + description: |- + GoMemLimitRatio is the percentage of memory limit used for GOMEMLIMIT (0-100). + Defaults to 90 if not set. + format: int32 + maximum: 100 + minimum: 0 + type: integer + grpcCompression: + description: GrpcCompression enables gzip compression on gRPC + calls. + type: boolean + grpcPort: + default: 8888 + description: GrpcPort is the gRPC service port. + format: int32 + type: integer + grpcSlowThreshold: + description: GrpcSlowThreshold is the duration above which a gRPC + call is logged as slow. + type: string + hashAlgorithm: + description: |- + HashAlgorithm selects the hash algorithm for the log chain. + Supported values: "blake3" (cryptographic, default) or "xxh3" (non-cryptographic, faster). + enum: + - blake3 + - xxh3 + type: string + headlessService: + description: HeadlessService configuration for Raft peer discovery. + properties: + annotations: + additionalProperties: + type: string + description: Annotations to add to the headless service. + type: object + enabled: + default: true + description: Enabled enables the headless service. + type: boolean + type: object + health: + description: Health check configuration. + properties: + clockSkewThreshold: + description: ClockSkewThreshold is the maximum allowed clock + skew between nodes. + type: string + dataResumeThreshold: + description: |- + DataResumeThreshold is the data usage resume (low-water) threshold for + disk-usage hysteresis. Must be < DataThreshold. Maps to HEALTH_DATA_RESUME_THRESHOLD. + type: string + dataThreshold: + description: DataThreshold is the data volume usage threshold + (0.0-1.0). + type: string + interval: + description: Interval between health checks. + type: string + walResumeThreshold: + description: |- + WalResumeThreshold is the WAL usage resume (low-water) threshold for + disk-usage hysteresis. Must be < WalThreshold. Maps to HEALTH_WAL_RESUME_THRESHOLD. + type: string + walThreshold: + description: WalThreshold is the WAL volume usage threshold + (0.0-1.0). + type: string + type: object + httpPort: + default: 9000 + description: HttpPort is the HTTP service port. + format: int32 + type: integer + idempotencyEvictionInterval: + description: |- + IdempotencyEvictionInterval is how often the leader proposes idempotency eviction. + Default: 60s. + type: string + idempotencyTTL: + description: |- + IdempotencyTTL is the time-to-live for idempotency keys (0 = never expire). + Default: 24h. + type: string + image: + description: Image configuration for the ledger container. + properties: + pullPolicy: + description: PullPolicy for the container image. + type: string + repository: + description: Repository is the container image repository. + type: string + tag: + description: Tag is the container image tag. + type: string + type: object + imagePullSecrets: + description: ImagePullSecrets for private registries. + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + type: array + ingress: + description: Ingress configuration for HTTP access. + properties: + annotations: + additionalProperties: + type: string + description: Annotations to add to the ingress. + type: object + className: + description: ClassName is the ingress class name. + type: string + enabled: + description: Enabled enables the HTTP ingress. + type: boolean + hosts: + description: Hosts configuration. + items: + description: IngressHost defines an ingress host rule. + properties: + host: + description: Host is the hostname. + type: string + paths: + description: Paths are the path rules. + items: + description: IngressPath defines an ingress path rule. + properties: + path: + default: / + description: Path is the URL path. + type: string + pathType: + default: Prefix + description: PathType is the path matching type. + type: string + type: object + type: array + required: + - host + type: object + type: array + labels: + additionalProperties: + type: string + description: Labels to add to the ingress. + type: object + tls: + description: TLS configuration. + items: + description: IngressTLS defines ingress TLS configuration. + properties: + hosts: + description: Hosts are the TLS hostnames. + items: + type: string + type: array + secretName: + description: SecretName is the TLS secret. + type: string + type: object + type: array + type: object + x-kubernetes-validations: + - message: hosts are required when ingress is enabled + rule: '!self.enabled || size(self.hosts) > 0' + ingressGrpc: + description: IngressGrpc configuration for gRPC access. + properties: + annotations: + additionalProperties: + type: string + description: Annotations to add to the ingress. + type: object + className: + description: ClassName is the ingress class name (e.g., "nginx", + "traefik"). + type: string + enabled: + description: Enabled enables the gRPC ingress. + type: boolean + hosts: + description: Hosts configuration. + items: + description: IngressHost defines an ingress host rule. + properties: + host: + description: Host is the hostname. + type: string + paths: + description: Paths are the path rules. + items: + description: IngressPath defines an ingress path rule. + properties: + path: + default: / + description: Path is the URL path. + type: string + pathType: + default: Prefix + description: PathType is the path matching type. + type: string + type: object + type: array + required: + - host + type: object + type: array + labels: + additionalProperties: + type: string + description: Labels to add to the ingress. + type: object + serviceAnnotations: + additionalProperties: + type: string + description: ServiceAnnotations to add to the backing gRPC + Kubernetes Service. + type: object + tls: + description: TLS configuration. + items: + description: IngressTLS defines ingress TLS configuration. + properties: + hosts: + description: Hosts are the TLS hostnames. + items: + type: string + type: array + secretName: + description: SecretName is the TLS secret. + type: string + type: object + type: array + type: object + x-kubernetes-validations: + - message: hosts are required when gRPC ingress is enabled + rule: '!self.enabled || size(self.hosts) > 0' + livenessProbe: + description: LivenessProbe overrides the default liveness probe + for the ledger container. + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number must + be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + logLevel: + description: |- + LogLevel sets the server's log verbosity. One of trace|debug|info|error. + Trace records are stdout-only and never exported via OTLP. + When set, takes precedence over Debug. + enum: + - trace + - debug + - info + - error + type: string + maxExecutionPlanSize: + description: |- + MaxExecutionPlanSize caps the number of AttributePlan entries an + ExecutionPlan may carry. Admission rejects proposals beyond this + (0 = unlimited). Default: 4096. + format: int32 + type: integer + metricsNaming: + description: |- + MetricsNaming selects the convention for metric names emitted + by the server: "otel" (the default, dot-notation) preserves + the OpenTelemetry instrument names; "prom" rewrites every + metric the server emits with a `ledger_` prefix and dots + converted to underscores so the names are unambiguous after + an OTLP→Prometheus collector that sanitises dots. OTel + semantic-convention auto-instrumentation (`go.*`, `process.*`, + `system.*`, `http.*`) uses the global MeterProvider and is + never touched by this flag. + enum: + - otel + - prom + type: string + mirrorMaxBatchSize: + description: MirrorMaxBatchSize is the maximum allowed batch size + for mirror sync. + format: int32 + type: integer + monitoring: + description: Monitoring configuration (OpenTelemetry). + properties: + attributes: + description: Attributes are additional OTEL resource attributes. + type: string + flightRecorder: + description: FlightRecorder configuration for runtime execution + trace buffering. + properties: + enabled: + description: Enabled enables the runtime flight recorder. + type: boolean + maxBytes: + anyOf: + - type: integer + - type: string + description: |- + MaxBytes is the maximum memory for the flight recorder buffer (e.g. "10Mi"). + Accepts Kubernetes quantity format. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + minAge: + description: MinAge is the minimum duration of trace data + retained in the buffer. + type: string + type: object + logs: + description: Logs configuration. + properties: + enabled: + description: Enabled enables log exporting. + type: boolean + endpoint: + description: Endpoint for the log exporter. + type: string + exporter: + description: Exporter type. + type: string + insecure: + description: Insecure disables TLS for the exporter. + type: string + level: + description: Level is the log level. + type: string + mode: + description: Mode is the exporter mode. + type: string + port: + description: Port for the log exporter. + type: string + type: object + metrics: + description: Metrics configuration. + properties: + enabled: + description: Enabled enables metrics. + type: boolean + endpoint: + description: Endpoint for the metrics exporter. + type: string + exporter: + description: Exporter type. + type: string + exporterPushInterval: + description: ExporterPushInterval is the push interval. + type: string + insecure: + description: Insecure disables TLS for the exporter. + type: string + keepInMemory: + description: KeepInMemory keeps metrics in memory. + type: boolean + mode: + description: Mode is the exporter mode. + type: string + port: + description: Port for the metrics exporter. + type: string + runtime: + description: Runtime enables runtime metrics. + type: boolean + runtimeMinimumReadMemStatsInterval: + description: RuntimeMinimumReadMemStatsInterval is the + minimum interval for reading mem stats. + type: string + type: object + pyroscope: + description: Pyroscope continuous profiling configuration. + properties: + applicationName: + description: ApplicationName overrides the application + name. + type: string + authToken: + description: AuthToken for Pyroscope authentication. + type: string + basicAuthPassword: + description: BasicAuthPassword for basic authentication. + type: string + basicAuthUser: + description: BasicAuthUser for basic authentication. + type: string + blockProfileRate: + description: BlockProfileRate is the block profile rate. + format: int32 + type: integer + disableGCRuns: + description: DisableGCRuns disables GC runs. + type: boolean + enabled: + description: Enabled enables Pyroscope profiling. + type: boolean + mutexProfileFraction: + description: MutexProfileFraction is the mutex profile + fraction. + format: int32 + type: integer + profileTypes: + description: ProfileTypes to collect. + type: string + serverAddress: + description: ServerAddress is the Pyroscope server address. + type: string + tags: + description: Tags in key=value,key2=value2 format. + type: string + tenantId: + description: TenantID for multi-tenant Pyroscope. + type: string + uploadRate: + description: UploadRate is the upload interval. + type: string + type: object + serviceName: + default: ledger + description: ServiceName for monitoring. + type: string + traces: + description: Traces configuration. + properties: + batch: + description: Batch enables batch mode. + type: string + enabled: + description: Enabled enables tracing. + type: boolean + endpoint: + description: Endpoint for the trace exporter. + type: string + exporter: + description: Exporter type (e.g., "otlp"). + type: string + insecure: + description: Insecure disables TLS for the exporter. + type: string + mode: + description: Mode is the exporter mode (e.g., "grpc"). + type: string + port: + description: Port for the trace exporter. + type: string + sampling: + description: Sampling configuration. + properties: + enabled: + description: Enabled enables error-aware trace sampling. + type: boolean + successRatio: + description: SuccessRatio is the sampling ratio for + successful traces (0.0-1.0). + type: string + type: object + type: object + type: object + networkPolicy: + description: NetworkPolicy configuration for egress restrictions. + properties: + additionalEgress: + description: |- + AdditionalEgress appends custom egress rules to the generated NetworkPolicy. + Use this to allow traffic to cluster-internal services (e.g. databases, message brokers). + items: + description: |- + NetworkPolicyEgressRule describes a particular set of traffic that is allowed out of pods + matched by a NetworkPolicySpec's podSelector. The traffic must match both ports and to. + This type is beta-level in 1.8 + properties: + ports: + description: |- + ports is a list of destination ports for outgoing traffic. + Each item in this list is combined using a logical OR. If this field is + empty or missing, this rule matches all ports (traffic not restricted by port). + If this field is present and contains at least one item, then this rule allows + traffic only if the traffic matches at least one port in the list. + items: + description: NetworkPolicyPort describes a port to + allow traffic on + properties: + endPort: + description: |- + endPort indicates that the range of ports from port to endPort if set, inclusive, + should be allowed by the policy. This field cannot be defined if the port field + is not defined or if the port field is defined as a named (string) port. + The endPort must be equal or greater than port. + format: int32 + type: integer + port: + anyOf: + - type: integer + - type: string + description: |- + port represents the port on the given protocol. This can either be a numerical or named + port on a pod. If this field is not provided, this matches all port names and + numbers. + If present, only traffic on the specified protocol AND port will be matched. + x-kubernetes-int-or-string: true + protocol: + description: |- + protocol represents the protocol (TCP, UDP, or SCTP) which traffic must match. + If not specified, this field defaults to TCP. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + to: + description: |- + to is a list of destinations for outgoing traffic of pods selected for this rule. + Items in this list are combined using a logical OR operation. If this field is + empty or missing, this rule matches all destinations (traffic not restricted by + destination). If this field is present and contains at least one item, this rule + allows traffic only if the traffic matches at least one item in the to list. + items: + description: |- + NetworkPolicyPeer describes a peer to allow traffic to/from. Only certain combinations of + fields are allowed + properties: + ipBlock: + description: |- + ipBlock defines policy on a particular IPBlock. If this field is set then + neither of the other fields can be. + properties: + cidr: + description: |- + cidr is a string representing the IPBlock + Valid examples are "192.168.1.0/24" or "2001:db8::/64" + type: string + except: + description: |- + except is a slice of CIDRs that should not be included within an IPBlock + Valid examples are "192.168.1.0/24" or "2001:db8::/64" + Except values will be rejected if they are outside the cidr range + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - cidr + type: object + namespaceSelector: + description: |- + namespaceSelector selects namespaces using cluster-scoped labels. This field follows + standard label selector semantics; if present but empty, it selects all namespaces. + + If podSelector is also set, then the NetworkPolicyPeer as a whole selects + the pods matching podSelector in the namespaces selected by namespaceSelector. + Otherwise it selects all pods in the namespaces selected by namespaceSelector. + properties: + matchExpressions: + description: matchExpressions is a list of + label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + podSelector: + description: |- + podSelector is a label selector which selects pods. This field follows standard label + selector semantics; if present but empty, it selects all pods. + + If namespaceSelector is also set, then the NetworkPolicyPeer as a whole selects + the pods matching podSelector in the Namespaces selected by NamespaceSelector. + Otherwise it selects the pods matching podSelector in the policy's own namespace. + properties: + matchExpressions: + description: matchExpressions is a list of + label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: array + enabled: + description: Enabled enables the egress NetworkPolicy. + type: boolean + externalCIDRExcept: + description: |- + ExternalCIDRExcept overrides the default RFC1918 CIDR blocks excluded from external egress. + Defaults to [10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16]. + items: + type: string + type: array + type: object + nodeSelector: + additionalProperties: + type: string + description: NodeSelector for pod scheduling. + type: object + numscriptCacheSize: + description: NumscriptCacheSize is the maximum number of parsed + Numscript programs to cache (LRU eviction). + format: int32 + type: integer + pebble: + description: Pebble storage engine configuration. + properties: + bytesPerSync: + anyOf: + - type: integer + - type: string + description: |- + BytesPerSync is bytes written before sync during flush/compaction (e.g. "512Ki"). + Accepts Kubernetes quantity format. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + cacheSize: + anyOf: + - type: integer + - type: string + description: |- + CacheSize is the block cache size (e.g. "1Gi", "512Mi"). + Accepts Kubernetes quantity format. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + compression: + description: |- + Compression is the per-level compression algorithm (L0-L6, comma-separated). + Options: none, snappy, zstd, fastest, fast, balanced, good, default. + type: string + disableWAL: + description: DisableWAL disables WAL entirely. + type: boolean + incrementalCompactThreshold: + description: |- + IncrementalCompactThreshold is the number of new log entries before + triggering an incremental compaction of the new range. + Default: 100000. + format: int64 + type: integer + l0CompactionThreshold: + description: L0CompactionThreshold is the L0 file count to + trigger compaction. + format: int32 + type: integer + l0StopWritesThreshold: + description: L0StopWritesThreshold is the L0 file count before + writes stop. + format: int32 + type: integer + lBaseMaxBytes: + anyOf: + - type: integer + - type: string + description: |- + LBaseMaxBytes is the maximum size of L1 (e.g. "64Mi", "256Mi"). + Accepts Kubernetes quantity format. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + maxCheckpoints: + description: |- + MaxCheckpoints is the maximum number of Pebble checkpoints to keep. + Default: 10. + format: int32 + type: integer + maxConcurrentCompactions: + description: MaxConcurrentCompactions is the maximum concurrent + compactions. + format: int32 + type: integer + memTableSize: + anyOf: + - type: integer + - type: string + description: |- + MemTableSize is the MemTable size (e.g. "256Mi", "1Gi"). + Accepts Kubernetes quantity format. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memTableStopWritesThreshold: + description: MemTableStopWritesThreshold is the number of + memtables before writes stop. + format: int32 + type: integer + targetFileSize: + anyOf: + - type: integer + - type: string + description: |- + TargetFileSize is the target SST file size (e.g. "64Mi"). + Accepts Kubernetes quantity format. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + valueSeparation: + description: ValueSeparation configuration for large value + storage in blob files. + properties: + enabled: + description: Enabled enables value separation (large values + stored in blob files). + type: boolean + garbageRatio: + description: |- + GarbageRatio is the blob garbage ratio before rewrite (0.0-1.0). + Default: 0.20. + type: string + maxDepth: + description: |- + MaxDepth is the max blob reference depth per SSTable. + Default: 4. + format: int32 + type: integer + minSize: + anyOf: + - type: integer + - type: string + description: |- + MinSize is the minimum value size for separation (e.g. "256", "1Ki"). + Accepts Kubernetes quantity format. Default: 256. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + rewriteAge: + description: |- + RewriteAge is the minimum blob file age before rewrite. + Default: 1h. + type: string + type: object + walBytesPerSync: + anyOf: + - type: integer + - type: string + description: |- + WalBytesPerSync is WAL bytes written before sync (e.g. "512Ki"). + Accepts Kubernetes quantity format. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + walMinSyncInterval: + description: WalMinSyncInterval is the minimum interval between + WAL syncs. + type: string + type: object + persistence: + description: Persistence configuration for WAL and data volumes. + properties: + coldCache: + description: |- + ColdCache persistence configuration for cold storage read cache. + Uses a separate volume to avoid filling the data disk when reading archived chapters. + properties: + accessMode: + default: ReadWriteOnce + description: |- + AccessMode for the PVC. + Mutually exclusive with hostPath. + type: string + hostPath: + description: |- + HostPath configures a host-local volume instead of a PVC. + When set, no PersistentVolumeClaim is created for this volume. + The pod uses a hostPath volume mounted from the specified path on the node. + Each pod gets an isolated subdirectory named after its ordinal index. + This is intended for NVMe instance stores where Raft replication provides durability. + Mutually exclusive with storageClass, accessMode, and volumeAttributesClassName. + properties: + path: + description: |- + Path on the host node (e.g. "/mnt/nvme0/data"). + Each pod gets an isolated subdirectory: /. + type: string + type: + default: DirectoryOrCreate + description: Type is the hostPath type. + enum: + - Directory + - DirectoryOrCreate + type: string + required: + - path + type: object + size: + anyOf: + - type: integer + - type: string + description: Size of the volume. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + storageClass: + description: |- + StorageClass for the PVC. + Mutually exclusive with hostPath. + type: string + volumeAttributesClassName: + description: |- + VolumeAttributesClassName is the name of the VolumeAttributesClass to use for the PVC. + Requires the VolumeAttributesClass feature gate to be enabled (beta in K8s 1.31+). + Mutually exclusive with hostPath. + type: string + type: object + data: + description: Data persistence configuration. + properties: + accessMode: + default: ReadWriteOnce + description: |- + AccessMode for the PVC. + Mutually exclusive with hostPath. + type: string + hostPath: + description: |- + HostPath configures a host-local volume instead of a PVC. + When set, no PersistentVolumeClaim is created for this volume. + The pod uses a hostPath volume mounted from the specified path on the node. + Each pod gets an isolated subdirectory named after its ordinal index. + This is intended for NVMe instance stores where Raft replication provides durability. + Mutually exclusive with storageClass, accessMode, and volumeAttributesClassName. + properties: + path: + description: |- + Path on the host node (e.g. "/mnt/nvme0/data"). + Each pod gets an isolated subdirectory: /. + type: string + type: + default: DirectoryOrCreate + description: Type is the hostPath type. + enum: + - Directory + - DirectoryOrCreate + type: string + required: + - path + type: object + size: + anyOf: + - type: integer + - type: string + description: Size of the volume. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + storageClass: + description: |- + StorageClass for the PVC. + Mutually exclusive with hostPath. + type: string + volumeAttributesClassName: + description: |- + VolumeAttributesClassName is the name of the VolumeAttributesClass to use for the PVC. + Requires the VolumeAttributesClass feature gate to be enabled (beta in K8s 1.31+). + Mutually exclusive with hostPath. + type: string + type: object + deletionProtection: + default: true + description: |- + DeletionProtection opts this ledger's PVCs and bound PVs into the + cluster-scoped volume deletion-protection admission policy. It defaults to + true (protected): the operator stamps the + `ledger.formance.com/deletion-protection: enabled` label on the volumes so + the policy selects them and rejects accidental DELETEs unless the object + carries the allow-deletion annotation. Set it explicitly to false to opt + out — the label is then removed and protection is lifted. The policy itself + must be installed cluster-wide by an admin via the Helm value + `pvcProtection.enabled` (on by default) — otherwise this flag has no effect + and the operator emits a DeletionProtectionInactive warning on the CR. + type: boolean + retentionPolicy: + description: RetentionPolicy for PVCs. + properties: + whenDeleted: + default: Retain + description: WhenDeleted policy. + type: string + whenScaled: + default: Retain + description: WhenScaled policy. + type: string + type: object + wal: + description: WAL persistence configuration. + properties: + accessMode: + default: ReadWriteOnce + description: |- + AccessMode for the PVC. + Mutually exclusive with hostPath. + type: string + hostPath: + description: |- + HostPath configures a host-local volume instead of a PVC. + When set, no PersistentVolumeClaim is created for this volume. + The pod uses a hostPath volume mounted from the specified path on the node. + Each pod gets an isolated subdirectory named after its ordinal index. + This is intended for NVMe instance stores where Raft replication provides durability. + Mutually exclusive with storageClass, accessMode, and volumeAttributesClassName. + properties: + path: + description: |- + Path on the host node (e.g. "/mnt/nvme0/data"). + Each pod gets an isolated subdirectory: /. + type: string + type: + default: DirectoryOrCreate + description: Type is the hostPath type. + enum: + - Directory + - DirectoryOrCreate + type: string + required: + - path + type: object + size: + anyOf: + - type: integer + - type: string + description: Size of the volume. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + storageClass: + description: |- + StorageClass for the PVC. + Mutually exclusive with hostPath. + type: string + volumeAttributesClassName: + description: |- + VolumeAttributesClassName is the name of the VolumeAttributesClass to use for the PVC. + Requires the VolumeAttributesClass feature gate to be enabled (beta in K8s 1.31+). + Mutually exclusive with hostPath. + type: string + type: object + type: object + podAnnotations: + additionalProperties: + type: string + description: PodAnnotations to add to each pod. + type: object + podAntiAffinity: + description: PodAntiAffinity configuration. + properties: + enabled: + default: true + description: Enabled enables pod anti-affinity. + type: boolean + topologyKey: + default: kubernetes.io/hostname + description: TopologyKey for anti-affinity. + type: string + type: + default: soft + description: Type is "soft" or "hard". + enum: + - soft + - hard + type: string + weight: + default: 100 + description: Weight for soft anti-affinity (1-100). + format: int32 + maximum: 100 + minimum: 1 + type: integer + type: object + podSecurityContext: + description: PodSecurityContext for the pod. + properties: + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + fsGroup: + description: |- + A special supplemental group that applies to all containers in a pod. + Some volume types allow the Kubelet to change the ownership of that volume + to be owned by the pod: + + 1. The owning GID will be the FSGroup + 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) + 3. The permission bits are OR'd with rw-rw---- + + If unset, the Kubelet will not modify the ownership and permissions of any volume. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + fsGroupChangePolicy: + description: |- + fsGroupChangePolicy defines behavior of changing ownership and permission of the volume + before being exposed inside Pod. This field will only apply to + volume types which support fsGroup based ownership(and permissions). + It will have no effect on ephemeral volume types such as: secret, configmaps + and emptydir. + Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. + Note that this field cannot be set when spec.os.name is windows. + type: string + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxChangePolicy: + description: |- + seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod. + It has no effect on nodes that do not support SELinux or to volumes does not support SELinux. + Valid values are "MountOption" and "Recursive". + + "Recursive" means relabeling of all files on all Pod volumes by the container runtime. + This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node. + + "MountOption" mounts all eligible Pod volumes with `-o context` mount option. + This requires all Pods that share the same volume to use the same SELinux label. + It is not possible to share the same volume among privileged and unprivileged Pods. + Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes + whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their + CSIDriver instance. Other volumes are always re-labelled recursively. + "MountOption" value is allowed only when SELinuxMount feature gate is enabled. + + If not specified and SELinuxMount feature gate is enabled, "MountOption" is used. + If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes + and "Recursive" for all other volumes. + + This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers. + + All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state. + Note that this field cannot be set when spec.os.name is windows. + type: string + seLinuxOptions: + description: |- + The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in SecurityContext. If set in + both SecurityContext and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies + to the container. + type: string + role: + description: Role is a SELinux role label that applies + to the container. + type: string + type: + description: Type is a SELinux type label that applies + to the container. + type: string + user: + description: User is a SELinux user label that applies + to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + supplementalGroups: + description: |- + A list of groups applied to the first process run in each container, in + addition to the container's primary GID and fsGroup (if specified). If + the SupplementalGroupsPolicy feature is enabled, the + supplementalGroupsPolicy field determines whether these are in addition + to or instead of any group memberships defined in the container image. + If unspecified, no additional groups are added, though group memberships + defined in the container image may still be used, depending on the + supplementalGroupsPolicy field. + Note that this field cannot be set when spec.os.name is windows. + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + description: |- + Defines how supplemental groups of the first container processes are calculated. + Valid values are "Merge" and "Strict". If not specified, "Merge" is used. + (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled + and the container runtime must implement support for this feature. + Note that this field cannot be set when spec.os.name is windows. + type: string + sysctls: + description: |- + Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported + sysctls (by the container runtime) might fail to launch. + Note that this field cannot be set when spec.os.name is windows. + items: + description: Sysctl defines a kernel parameter to be set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options within a container's SecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the + GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + queryProfileThreshold: + description: QueryProfileThreshold logs and emits OTel attributes + for queries exceeding this duration (0 to disable). + type: string + raft: + description: Raft consensus configuration. + properties: + compactionMargin: + description: CompactionMargin is the compaction margin. + format: int32 + type: integer + electionTick: + description: ElectionTick is the election timeout in ticks. + format: int32 + type: integer + heartbeatTick: + description: HeartbeatTick is the heartbeat interval in ticks. + format: int32 + type: integer + learnerPromotionThreshold: + description: LearnerPromotionThreshold is the max log entry + lag before auto-promoting a learner. + format: int32 + type: integer + maintenanceInterval: + description: |- + MaintenanceInterval is the interval for background WAL snapshot + Pebble checkpoint. + Default: 30s. + type: string + maxInflightMsgs: + description: MaxInflightMsgs is the maximum number of in-flight + messages. + format: int32 + type: integer + maxSizePerMsg: + anyOf: + - type: integer + - type: string + description: |- + MaxSizePerMsg is the maximum size per message (e.g. "1Mi", "4Mi"). + Accepts Kubernetes quantity format. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + processingTickInterval: + description: |- + ProcessingTickInterval is the interval for processing committed entries. + Default: tickInterval/10. + type: string + proposeQueueCapacity: + description: ProposeQueueCapacity is the capacity of the propose + queue. + format: int32 + type: integer + replayBatchSize: + description: ReplayBatchSize is the number of Raft log entries + replayed per batch on startup. + format: int32 + type: integer + tickInterval: + description: TickInterval is the interval between Raft ticks. + type: string + transport: + description: Transport queue configuration. + properties: + bufferSize: + anyOf: + - type: integer + - type: string + description: |- + BufferSize is the per-peer send buffer capacity (e.g. "20Mi"). + Accepts Kubernetes quantity format. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + receptionQueues: + description: ReceptionQueues are the reception queue capacities + per priority. + items: + format: int32 + type: integer + type: array + sendQueues: + description: SendQueues are the send queue capacities + per priority. + items: + format: int32 + type: integer + type: array + type: object + type: object + readIndex: + description: ReadIndex configuration for the Pebble read index + store. + properties: + batchSize: + description: |- + BatchSize is the number of log entries per Pebble batch commit. + Higher values amortize commit overhead but use more memory. + Default: 1000. + format: int32 + type: integer + pebble: + description: |- + Pebble holds the common Pebble tunables for the read index. + Uses the same knobs as the primary store (cache size, memtable, L0 thresholds, etc.). + properties: + bytesPerSync: + anyOf: + - type: integer + - type: string + description: |- + BytesPerSync is bytes written before sync during flush/compaction (e.g. "512Ki"). + Accepts Kubernetes quantity format. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + cacheSize: + anyOf: + - type: integer + - type: string + description: |- + CacheSize is the block cache size (e.g. "1Gi", "512Mi"). + Accepts Kubernetes quantity format. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + compression: + description: |- + Compression is the per-level compression algorithm (L0-L6, comma-separated). + Options: none, snappy, zstd, fastest, fast, balanced, good, default. + type: string + disableWAL: + description: DisableWAL disables WAL entirely. + type: boolean + incrementalCompactThreshold: + description: |- + IncrementalCompactThreshold is the number of new log entries before + triggering an incremental compaction of the new range. + Default: 100000. + format: int64 + type: integer + l0CompactionThreshold: + description: L0CompactionThreshold is the L0 file count + to trigger compaction. + format: int32 + type: integer + l0StopWritesThreshold: + description: L0StopWritesThreshold is the L0 file count + before writes stop. + format: int32 + type: integer + lBaseMaxBytes: + anyOf: + - type: integer + - type: string + description: |- + LBaseMaxBytes is the maximum size of L1 (e.g. "64Mi", "256Mi"). + Accepts Kubernetes quantity format. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + maxCheckpoints: + description: |- + MaxCheckpoints is the maximum number of Pebble checkpoints to keep. + Default: 10. + format: int32 + type: integer + maxConcurrentCompactions: + description: MaxConcurrentCompactions is the maximum concurrent + compactions. + format: int32 + type: integer + memTableSize: + anyOf: + - type: integer + - type: string + description: |- + MemTableSize is the MemTable size (e.g. "256Mi", "1Gi"). + Accepts Kubernetes quantity format. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memTableStopWritesThreshold: + description: MemTableStopWritesThreshold is the number + of memtables before writes stop. + format: int32 + type: integer + targetFileSize: + anyOf: + - type: integer + - type: string + description: |- + TargetFileSize is the target SST file size (e.g. "64Mi"). + Accepts Kubernetes quantity format. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + valueSeparation: + description: ValueSeparation configuration for large value + storage in blob files. + properties: + enabled: + description: Enabled enables value separation (large + values stored in blob files). + type: boolean + garbageRatio: + description: |- + GarbageRatio is the blob garbage ratio before rewrite (0.0-1.0). + Default: 0.20. + type: string + maxDepth: + description: |- + MaxDepth is the max blob reference depth per SSTable. + Default: 4. + format: int32 + type: integer + minSize: + anyOf: + - type: integer + - type: string + description: |- + MinSize is the minimum value size for separation (e.g. "256", "1Ki"). + Accepts Kubernetes quantity format. Default: 256. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + rewriteAge: + description: |- + RewriteAge is the minimum blob file age before rewrite. + Default: 1h. + type: string + type: object + walBytesPerSync: + anyOf: + - type: integer + - type: string + description: |- + WalBytesPerSync is WAL bytes written before sync (e.g. "512Ki"). + Accepts Kubernetes quantity format. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + walMinSyncInterval: + description: WalMinSyncInterval is the minimum interval + between WAL syncs. + type: string + type: object + type: object + readinessProbe: + description: ReadinessProbe overrides the default readiness probe + for the ledger container. + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number must + be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + receiptSigning: + description: ReceiptSigning configures HMAC signing for JWT transaction + receipts. + properties: + secretKey: + default: key + description: SecretKey is the key in the secret containing + the HMAC key. + type: string + secretName: + description: SecretName is the Kubernetes secret containing + the HMAC key. + type: string + type: object + replicas: + default: 3 + description: Replicas is the number of Raft nodes. + format: int32 + minimum: 1 + type: integer + resources: + description: Resources for the ledger container. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + responseSigning: + description: ResponseSigning configuration (Ed25519). + properties: + enabled: + description: Enabled enables response signing. + type: boolean + secretKey: + default: seed + description: SecretKey is the key in the secret containing + the seed. + type: string + secretName: + description: SecretName is the Kubernetes secret containing + the Ed25519 seed. + type: string + type: object + restore: + description: Restore starts the server in restore mode. + type: boolean + securityContext: + description: SecurityContext for the container. + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies + to the container. + type: string + role: + description: Role is a SELinux role label that applies + to the container. + type: string + type: + description: Type is a SELinux type label that applies + to the container. + type: string + user: + description: User is a SELinux user label that applies + to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the + GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + sentinelMode: + description: |- + SentinelMode enables runtime volume consistency assertions + (monotonicity, delta/posting cross-check, post-commit cache/Pebble verification). + type: boolean + service: + description: Service configuration for the ClusterIP service. + properties: + annotations: + additionalProperties: + type: string + description: Annotations to add to the service. + type: object + grpcPort: + default: 8888 + description: GrpcPort is the gRPC service port. + format: int32 + type: integer + httpPort: + default: 9000 + description: HttpPort is the HTTP service port. + format: int32 + type: integer + raftPort: + default: 7777 + description: RaftPort is the Raft transport port. + format: int32 + type: integer + type: + default: ClusterIP + description: Type is the service type. + type: string + type: object + serviceAccount: + description: ServiceAccount configuration. + properties: + annotations: + additionalProperties: + type: string + description: Annotations to add to the service account. + type: object + create: + description: Create specifies whether to create a service + account. + type: boolean + name: + description: Name overrides the service account name. + type: string + type: object + snapshot: + description: Snapshot sync configuration for Raft snapshot transfers. + properties: + fileRetryCount: + description: |- + FileRetryCount is the per-file retry attempts during snapshot sync on transient stream errors. + Default: 3. + format: int32 + type: integer + parallelism: + description: |- + Parallelism is the number of parallel file fetch workers during snapshot sync. + Default: 4. + format: int32 + type: integer + retryCount: + description: |- + RetryCount is the session-level retry attempts for snapshot sync on transient errors. + Default: 5. + format: int32 + type: integer + sessionTTL: + description: |- + SessionTTL is the server-side session TTL for snapshot sync. + Default: 5m. + type: string + type: object + startupProbe: + description: StartupProbe overrides the default startup probe + for the ledger container. + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number must + be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + tls: + description: TLS configuration for gRPC connections. + properties: + caSecretKey: + description: CASecretKey is the key for the CA certificate + in the secret. + type: string + enabled: + description: Enabled enables TLS. + type: boolean + secretName: + description: SecretName is the Kubernetes secret containing + TLS certificate and key. + type: string + type: object + tolerations: + description: Tolerations for pod scheduling. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + topologySpreadConstraints: + description: |- + TopologySpreadConstraints describe how pods are spread across failure + domains (e.g. zones, nodes). See PodSpec.topologySpreadConstraints. The + pod label selector defaults to the Cluster's selector when omitted. + items: + description: TopologySpreadConstraint specifies how to spread + matching pods among the given topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + unsafeSkipConfigValidation: + description: |- + UnsafeSkipConfigValidation skips startup configuration safety checks. + DANGEROUS: allows node-id/cluster-id changes on existing data. + type: boolean + walDir: + default: /data/raft + description: WalDir is the WAL data directory. + type: string + x-kubernetes-validations: + - message: walDir is immutable once set + rule: oldSelf == '' || self == oldSelf + type: object + stacks: + description: |- + Stacks on which the configuration is applied. Can contain `*` to + indicate a wildcard, following the same convention as Settings. + items: + type: string + type: array + type: object + type: object + served: true + storage: true diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 35564ace8..6110f7150 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -29,6 +29,7 @@ resources: - bases/formance.com_transactionplanes.yaml - bases/formance.com_otelexporterendpoints.yaml +- bases/formance.com_ledgerconfigurations.yaml #+kubebuilder:scaffold:crdkustomizeresource # commonAnnotations: diff --git a/config/rbac/ledgerconfiguration_editor_role.yaml b/config/rbac/ledgerconfiguration_editor_role.yaml new file mode 100644 index 000000000..2e86d61e9 --- /dev/null +++ b/config/rbac/ledgerconfiguration_editor_role.yaml @@ -0,0 +1,25 @@ +# permissions for end users to edit ledgerconfigurations. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: ledgerconfiguration-editor-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: operatorv2 + app.kubernetes.io/part-of: operatorv2 + app.kubernetes.io/managed-by: kustomize + name: ledgerconfiguration-editor-role +rules: +- apiGroups: + - formance.com + resources: + - ledgerconfigurations + verbs: + - create + - delete + - get + - list + - patch + - update + - watch diff --git a/config/rbac/ledgerconfiguration_viewer_role.yaml b/config/rbac/ledgerconfiguration_viewer_role.yaml new file mode 100644 index 000000000..c2204a767 --- /dev/null +++ b/config/rbac/ledgerconfiguration_viewer_role.yaml @@ -0,0 +1,21 @@ +# permissions for end users to view ledgerconfigurations. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: ledgerconfiguration-viewer-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: operatorv2 + app.kubernetes.io/part-of: operatorv2 + app.kubernetes.io/managed-by: kustomize + name: ledgerconfiguration-viewer-role +rules: +- apiGroups: + - formance.com + resources: + - ledgerconfigurations + verbs: + - get + - list + - watch diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index ab8a14222..83d3b952a 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -193,6 +193,14 @@ rules: - patch - update - watch +- apiGroups: + - formance.com + resources: + - ledgerconfigurations + verbs: + - get + - list + - watch - apiGroups: - ledger.formance.com resources: diff --git a/config/samples/formance.com_v1beta1_ledgerconfiguration.yaml b/config/samples/formance.com_v1beta1_ledgerconfiguration.yaml new file mode 100644 index 000000000..ee032fd1c --- /dev/null +++ b/config/samples/formance.com_v1beta1_ledgerconfiguration.yaml @@ -0,0 +1,14 @@ +apiVersion: formance.com/v1beta1 +kind: LedgerConfiguration +metadata: + name: default +spec: + stacks: + - "*" + cluster: + monitoring: + pyroscope: + enabled: false + podAntiAffinity: + enabled: true + type: soft diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml index 17910c141..bb536ad23 100644 --- a/config/samples/kustomization.yaml +++ b/config/samples/kustomization.yaml @@ -21,4 +21,5 @@ resources: - formance.com_v1beta1_resourcereference.yaml - formance.com_v1beta1_brokerconsumer.yaml - formance.com_v1beta1_broker.yaml +- formance.com_v1beta1_ledgerconfiguration.yaml #+kubebuilder:scaffold:manifestskustomizesamples diff --git a/docs/09-Configuration reference/02-Custom Resource Definitions.md b/docs/09-Configuration reference/02-Custom Resource Definitions.md index 35bd4a8d8..4b72cdc84 100644 --- a/docs/09-Configuration reference/02-Custom Resource Definitions.md +++ b/docs/09-Configuration reference/02-Custom Resource Definitions.md @@ -41,6 +41,7 @@ Other resources : - [Database](#database) - [GatewayGRPCAPI](#gatewaygrpcapi) - [GatewayHTTPAPI](#gatewayhttpapi) +- [LedgerConfiguration](#ledgerconfiguration) - [OtelExporterEndpoint](#otelexporterendpoint) - [ResourceReference](#resourcereference) - [Versions](#versions) @@ -2501,6 +2502,65 @@ GatewayHTTPAPI is the Schema for the HTTPAPIs API | `ready` _boolean_ | | | | +#### LedgerConfiguration + + + +LedgerConfiguration defines the base specification applied to every Ledger v3 +Cluster targeted by spec.stacks. A configuration targeting a stack by name +takes priority over a configuration targeting all stacks with `*`. + + + + + + + + + + + + + + + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `formance.com/v1beta1` | | | +| `kind` _string_ | `LedgerConfiguration` | | | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `spec` _[LedgerConfigurationSpec](#ledgerconfigurationspec)_ | | | | + + + +##### LedgerConfigurationSpec + + + + + + + + + + + + + + + + + + + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `stacks` _string array_ | Stacks on which the configuration is applied. Can contain `*` to
indicate a wildcard, following the same convention as Settings. | | | +| `cluster` _[ClusterSpec](#clusterspec)_ | Cluster is the base Ledger v3 Cluster specification. Stack-specific
Settings and values owned by the Operator are applied on top of it. | | | + + + + #### OtelExporterEndpoint diff --git a/docs/09-Configuration reference/settings.catalog.json b/docs/09-Configuration reference/settings.catalog.json index 09f69d2a6..d8be02d3a 100644 --- a/docs/09-Configuration reference/settings.catalog.json +++ b/docs/09-Configuration reference/settings.catalog.json @@ -201,21 +201,21 @@ "key": "deployments.ledger.containers.ledger.resource-requirements.claims", "valueType": "string[]", "sources": [ - "internal/resources/ledgers/v3.go:254" + "internal/resources/ledgers/v3.go:290" ] }, { "key": "deployments.ledger.containers.ledger.resource-requirements.limits", "valueType": "map[string]string", "sources": [ - "internal/resources/ledgers/v3.go:254" + "internal/resources/ledgers/v3.go:290" ] }, { "key": "deployments.ledger.containers.ledger.resource-requirements.requests", "valueType": "map[string]string", "sources": [ - "internal/resources/ledgers/v3.go:254" + "internal/resources/ledgers/v3.go:290" ] }, { @@ -223,7 +223,7 @@ "valueType": "int32", "default": "3", "sources": [ - "internal/resources/ledgers/v3.go:240" + "internal/resources/ledgers/v3.go:276" ] }, { diff --git a/go.mod b/go.mod index 4c6280ea3..69c1e77dd 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,8 @@ module github.com/formancehq/operator/v3 -go 1.25.0 +go 1.26.0 -toolchain go1.25.5 +toolchain go1.26.1 require ( github.com/formancehq/go-libs/v5 v5.2.0 @@ -46,6 +46,7 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.13.0 // indirect github.com/evanphx/json-patch/v5 v5.9.11 // indirect + github.com/formancehq/ledger/misc/operator v0.0.0-20260715094310-76862ea0b681 github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect diff --git a/go.sum b/go.sum index eca6816b7..bedc0a293 100644 --- a/go.sum +++ b/go.sum @@ -18,6 +18,8 @@ github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjT github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= github.com/formancehq/go-libs/v5 v5.2.0 h1:TpS47F8X5g5cHhnecfD20TrcdBqUVGy/ezZv0oFaQjc= github.com/formancehq/go-libs/v5 v5.2.0/go.mod h1:ms6tCGw1yqB4qtEbAuqPOQegWo4rU48vDobNkK7Ak6U= +github.com/formancehq/ledger/misc/operator v0.0.0-20260715094310-76862ea0b681 h1:PzhfbpKZqMJd4opnKmEtFq0Hx/6Pxgq9GT7XzWcr+Ss= +github.com/formancehq/ledger/misc/operator v0.0.0-20260715094310-76862ea0b681/go.mod h1:tZa1TFBcXJxc1R0NqnsN6gOGoskd82a7wjmGqCBhOBU= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= diff --git a/helm/crds/templates/crds/apiextensions.k8s.io_v1_customresourcedefinition_ledgerconfigurations.formance.com.yaml b/helm/crds/templates/crds/apiextensions.k8s.io_v1_customresourcedefinition_ledgerconfigurations.formance.com.yaml new file mode 100644 index 000000000..a7f603b2a --- /dev/null +++ b/helm/crds/templates/crds/apiextensions.k8s.io_v1_customresourcedefinition_ledgerconfigurations.formance.com.yaml @@ -0,0 +1,4048 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.18.0 + helm.sh/resource-policy: keep + {{- with .Values.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: ledgerconfigurations.formance.com +spec: + group: formance.com + names: + kind: LedgerConfiguration + listKind: LedgerConfigurationList + plural: ledgerconfigurations + singular: ledgerconfiguration + scope: Cluster + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: |- + LedgerConfiguration defines the base specification applied to every Ledger v3 + Cluster targeted by spec.stacks. A configuration targeting a stack by name + takes priority over a configuration targeting all stacks with `*`. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + cluster: + description: |- + Cluster is the base Ledger v3 Cluster specification. Stack-specific + Settings and values owned by the Operator are applied on top of it. + properties: + additionalLabels: + additionalProperties: + type: string + description: |- + AdditionalLabels are merged on top of the default selector labels + (app.kubernetes.io/name, app.kubernetes.io/instance) on every owned + resource AND on the pod template / Service selectors. Keys that collide + with a default override it; the app.kubernetes.io/managed-by ownership + label is dropped from the merge and stays operator-owned everywhere + (top-level objects AND pods). + + Use this to escape an unrelated Service whose selector accidentally + matches our pods. Selector fields on Service / StatefulSet are immutable + after creation: changing this field on an existing cluster will be + rejected by the operator (a SelectorImmutable=False condition is set). + type: object + admissionMetrics: + description: AdmissionMetrics enables admission path metrics. + type: boolean + affinity: + description: Affinity rules for pod scheduling. + properties: + nodeAffinity: + description: Describes node affinity scheduling rules for + the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated with + the corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching the + corresponding nodeSelectorTerm, in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. + The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling rules (e.g. + co-locate this pod in the same node, zone, etc. as some + other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling rules + (e.g. avoid putting this pod in the same node, zone, etc. + as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and subtracting + "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + auth: + description: Auth holds authentication and authorization configuration. + properties: + anonymousScopes: + description: |- + AnonymousScopes lists the granular scopes granted to requests that arrive + without a bearer token. Wildcards "*:read" / "*:write" expand to every + granular scope with that suffix. The canonical writes-only configuration + is `["*:read"]`: reads are public, writes still require a valid token. + Empty (default) preserves the strict behavior — every request must + authenticate. Invalid tokens are still rejected with 401 regardless of + this setting; only the *absence* of a token triggers the fallback. + items: + type: string + type: array + checkScopes: + description: CheckScopes enables scope checking. + type: boolean + enabled: + description: Enabled enables authentication. + type: boolean + issuer: + description: Issuer is the OIDC token issuer URL (for backward + compatibility). + type: string + issuers: + description: Issuers is a list of trusted OIDC issuers. + items: + type: string + type: array + readKeySetMaxRetries: + description: ReadKeySetMaxRetries is the maximum number of + retries for reading key sets. + format: int32 + type: integer + scopeMapping: + additionalProperties: + items: + type: string + type: array + description: |- + ScopeMapping maps virtual scopes (e.g. "ledger:read") to granular scopes. + When provided, overrides the default mapping and --auth-service is ignored for scope resolution. + type: object + service: + description: Service is the scope prefix (e.g. "ledger" for + "ledger:read"). + type: string + type: object + bindAddr: + default: 0.0.0.0:7777 + description: BindAddr is the Raft transport bind address. + type: string + x-kubernetes-validations: + - message: bindAddr is immutable once set + rule: oldSelf == '' || self == oldSelf + bloom: + description: Bloom filter configuration per attribute type. + properties: + boundaries: + description: Boundaries bloom filter configuration. + properties: + expectedKeys: + description: ExpectedKeys is the expected number of unique + keys (0 to disable this filter). + format: int64 + type: integer + fpRate: + description: FPRate is the false positive rate (0.0-1.0). + type: string + type: object + ledgerMetadata: + description: LedgerMetadata bloom filter configuration. + properties: + expectedKeys: + description: ExpectedKeys is the expected number of unique + keys (0 to disable this filter). + format: int64 + type: integer + fpRate: + description: FPRate is the false positive rate (0.0-1.0). + type: string + type: object + ledgers: + description: Ledgers bloom filter configuration. + properties: + expectedKeys: + description: ExpectedKeys is the expected number of unique + keys (0 to disable this filter). + format: int64 + type: integer + fpRate: + description: FPRate is the false positive rate (0.0-1.0). + type: string + type: object + metadata: + description: Metadata bloom filter configuration. + properties: + expectedKeys: + description: ExpectedKeys is the expected number of unique + keys (0 to disable this filter). + format: int64 + type: integer + fpRate: + description: FPRate is the false positive rate (0.0-1.0). + type: string + type: object + numscriptContents: + description: NumscriptContents bloom filter configuration. + properties: + expectedKeys: + description: ExpectedKeys is the expected number of unique + keys (0 to disable this filter). + format: int64 + type: integer + fpRate: + description: FPRate is the false positive rate (0.0-1.0). + type: string + type: object + numscriptVersions: + description: NumscriptVersions bloom filter configuration. + properties: + expectedKeys: + description: ExpectedKeys is the expected number of unique + keys (0 to disable this filter). + format: int64 + type: integer + fpRate: + description: FPRate is the false positive rate (0.0-1.0). + type: string + type: object + preparedQueries: + description: PreparedQueries bloom filter configuration. + properties: + expectedKeys: + description: ExpectedKeys is the expected number of unique + keys (0 to disable this filter). + format: int64 + type: integer + fpRate: + description: FPRate is the false positive rate (0.0-1.0). + type: string + type: object + references: + description: References bloom filter configuration. + properties: + expectedKeys: + description: ExpectedKeys is the expected number of unique + keys (0 to disable this filter). + format: int64 + type: integer + fpRate: + description: FPRate is the false positive rate (0.0-1.0). + type: string + type: object + sinkConfigs: + description: SinkConfigs bloom filter configuration. + properties: + expectedKeys: + description: ExpectedKeys is the expected number of unique + keys (0 to disable this filter). + format: int64 + type: integer + fpRate: + description: FPRate is the false positive rate (0.0-1.0). + type: string + type: object + transactions: + description: Transactions bloom filter configuration. + properties: + expectedKeys: + description: ExpectedKeys is the expected number of unique + keys (0 to disable this filter). + format: int64 + type: integer + fpRate: + description: FPRate is the false positive rate (0.0-1.0). + type: string + type: object + volumes: + description: Volumes bloom filter configuration. + properties: + expectedKeys: + description: ExpectedKeys is the expected number of unique + keys (0 to disable this filter). + format: int64 + type: integer + fpRate: + description: FPRate is the false positive rate (0.0-1.0). + type: string + type: object + type: object + cache: + description: Cache configuration. + properties: + rotationThreshold: + description: |- + RotationThreshold is the number of Raft log entries before rotating cache generations. + Changes trigger a rolling restart; convergence is deterministic via Raft (applyClusterConfig). + format: int32 + minimum: 1 + type: integer + type: object + clusterID: + default: default + description: ClusterID for inter-node communication validation. + type: string + x-kubernetes-validations: + - message: clusterID is immutable once set + rule: oldSelf == '' || self == oldSelf + coldStorage: + description: ColdStorage configuration for chapter archival. + properties: + bucketId: + description: BucketID is the shared namespace prefix for archives. + type: string + driver: + description: 'Driver is the storage driver: "filesystem" or + "s3".' + type: string + path: + description: Path is the base path for filesystem driver. + type: string + s3: + description: S3 configuration. + properties: + bucket: + description: Bucket is the S3 bucket name. + type: string + endpoint: + description: Endpoint is a custom S3 endpoint (for MinIO). + type: string + region: + description: Region is the AWS region. + type: string + type: object + type: object + dataDir: + default: /data/app + description: DataDir is the application data directory. + type: string + x-kubernetes-validations: + - message: dataDir is immutable once set + rule: oldSelf == '' || self == oldSelf + debug: + description: |- + Debug enables debug logging. Equivalent to LogLevel="debug". + LogLevel takes precedence when both are set; prefer LogLevel for new + manifests since it also unlocks the trace level. + type: boolean + dnsEndpoint: + description: DNSEndpoint configuration for ExternalDNS. + properties: + annotations: + additionalProperties: + type: string + description: Annotations to add to the DNSEndpoint resource. + type: object + enabled: + description: Enabled enables the DNSEndpoint resource. + type: boolean + endpoints: + description: Endpoints is the list of DNS endpoint entries. + items: + description: DNSEndpointEntry defines a single DNS endpoint. + properties: + dnsName: + description: DNSName is the hostname for the DNS record. + type: string + providerSpecific: + description: ProviderSpecific holds provider-specific + properties. + items: + description: ProviderSpecificProperty defines a provider-specific + key-value pair. + properties: + name: + description: Name is the property name. + type: string + value: + description: Value is the property value. + type: string + required: + - name + - value + type: object + type: array + recordTTL: + description: RecordTTL is the TTL in seconds for the + DNS record. + format: int64 + type: integer + recordType: + description: RecordType is the DNS record type (e.g., + CNAME, A). Defaults to CNAME. + type: string + targets: + description: Targets is the list of target hostnames + or IPs. + items: + type: string + type: array + required: + - dnsName + - targets + type: object + type: array + type: object + x-kubernetes-validations: + - message: endpoints are required when dnsEndpoint is enabled + rule: '!self.enabled || size(self.endpoints) > 0' + extraEnv: + description: ExtraEnv is a list of additional environment variables + to inject into ledger containers. + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the + exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + goMemLimitRatio: + description: |- + GoMemLimitRatio is the percentage of memory limit used for GOMEMLIMIT (0-100). + Defaults to 90 if not set. + format: int32 + maximum: 100 + minimum: 0 + type: integer + grpcCompression: + description: GrpcCompression enables gzip compression on gRPC + calls. + type: boolean + grpcPort: + default: 8888 + description: GrpcPort is the gRPC service port. + format: int32 + type: integer + grpcSlowThreshold: + description: GrpcSlowThreshold is the duration above which a gRPC + call is logged as slow. + type: string + hashAlgorithm: + description: |- + HashAlgorithm selects the hash algorithm for the log chain. + Supported values: "blake3" (cryptographic, default) or "xxh3" (non-cryptographic, faster). + enum: + - blake3 + - xxh3 + type: string + headlessService: + description: HeadlessService configuration for Raft peer discovery. + properties: + annotations: + additionalProperties: + type: string + description: Annotations to add to the headless service. + type: object + enabled: + default: true + description: Enabled enables the headless service. + type: boolean + type: object + health: + description: Health check configuration. + properties: + clockSkewThreshold: + description: ClockSkewThreshold is the maximum allowed clock + skew between nodes. + type: string + dataResumeThreshold: + description: |- + DataResumeThreshold is the data usage resume (low-water) threshold for + disk-usage hysteresis. Must be < DataThreshold. Maps to HEALTH_DATA_RESUME_THRESHOLD. + type: string + dataThreshold: + description: DataThreshold is the data volume usage threshold + (0.0-1.0). + type: string + interval: + description: Interval between health checks. + type: string + walResumeThreshold: + description: |- + WalResumeThreshold is the WAL usage resume (low-water) threshold for + disk-usage hysteresis. Must be < WalThreshold. Maps to HEALTH_WAL_RESUME_THRESHOLD. + type: string + walThreshold: + description: WalThreshold is the WAL volume usage threshold + (0.0-1.0). + type: string + type: object + httpPort: + default: 9000 + description: HttpPort is the HTTP service port. + format: int32 + type: integer + idempotencyEvictionInterval: + description: |- + IdempotencyEvictionInterval is how often the leader proposes idempotency eviction. + Default: 60s. + type: string + idempotencyTTL: + description: |- + IdempotencyTTL is the time-to-live for idempotency keys (0 = never expire). + Default: 24h. + type: string + image: + description: Image configuration for the ledger container. + properties: + pullPolicy: + description: PullPolicy for the container image. + type: string + repository: + description: Repository is the container image repository. + type: string + tag: + description: Tag is the container image tag. + type: string + type: object + imagePullSecrets: + description: ImagePullSecrets for private registries. + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + type: array + ingress: + description: Ingress configuration for HTTP access. + properties: + annotations: + additionalProperties: + type: string + description: Annotations to add to the ingress. + type: object + className: + description: ClassName is the ingress class name. + type: string + enabled: + description: Enabled enables the HTTP ingress. + type: boolean + hosts: + description: Hosts configuration. + items: + description: IngressHost defines an ingress host rule. + properties: + host: + description: Host is the hostname. + type: string + paths: + description: Paths are the path rules. + items: + description: IngressPath defines an ingress path rule. + properties: + path: + default: / + description: Path is the URL path. + type: string + pathType: + default: Prefix + description: PathType is the path matching type. + type: string + type: object + type: array + required: + - host + type: object + type: array + labels: + additionalProperties: + type: string + description: Labels to add to the ingress. + type: object + tls: + description: TLS configuration. + items: + description: IngressTLS defines ingress TLS configuration. + properties: + hosts: + description: Hosts are the TLS hostnames. + items: + type: string + type: array + secretName: + description: SecretName is the TLS secret. + type: string + type: object + type: array + type: object + x-kubernetes-validations: + - message: hosts are required when ingress is enabled + rule: '!self.enabled || size(self.hosts) > 0' + ingressGrpc: + description: IngressGrpc configuration for gRPC access. + properties: + annotations: + additionalProperties: + type: string + description: Annotations to add to the ingress. + type: object + className: + description: ClassName is the ingress class name (e.g., "nginx", + "traefik"). + type: string + enabled: + description: Enabled enables the gRPC ingress. + type: boolean + hosts: + description: Hosts configuration. + items: + description: IngressHost defines an ingress host rule. + properties: + host: + description: Host is the hostname. + type: string + paths: + description: Paths are the path rules. + items: + description: IngressPath defines an ingress path rule. + properties: + path: + default: / + description: Path is the URL path. + type: string + pathType: + default: Prefix + description: PathType is the path matching type. + type: string + type: object + type: array + required: + - host + type: object + type: array + labels: + additionalProperties: + type: string + description: Labels to add to the ingress. + type: object + serviceAnnotations: + additionalProperties: + type: string + description: ServiceAnnotations to add to the backing gRPC + Kubernetes Service. + type: object + tls: + description: TLS configuration. + items: + description: IngressTLS defines ingress TLS configuration. + properties: + hosts: + description: Hosts are the TLS hostnames. + items: + type: string + type: array + secretName: + description: SecretName is the TLS secret. + type: string + type: object + type: array + type: object + x-kubernetes-validations: + - message: hosts are required when gRPC ingress is enabled + rule: '!self.enabled || size(self.hosts) > 0' + livenessProbe: + description: LivenessProbe overrides the default liveness probe + for the ledger container. + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number must + be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + logLevel: + description: |- + LogLevel sets the server's log verbosity. One of trace|debug|info|error. + Trace records are stdout-only and never exported via OTLP. + When set, takes precedence over Debug. + enum: + - trace + - debug + - info + - error + type: string + maxExecutionPlanSize: + description: |- + MaxExecutionPlanSize caps the number of AttributePlan entries an + ExecutionPlan may carry. Admission rejects proposals beyond this + (0 = unlimited). Default: 4096. + format: int32 + type: integer + metricsNaming: + description: |- + MetricsNaming selects the convention for metric names emitted + by the server: "otel" (the default, dot-notation) preserves + the OpenTelemetry instrument names; "prom" rewrites every + metric the server emits with a `ledger_` prefix and dots + converted to underscores so the names are unambiguous after + an OTLP→Prometheus collector that sanitises dots. OTel + semantic-convention auto-instrumentation (`go.*`, `process.*`, + `system.*`, `http.*`) uses the global MeterProvider and is + never touched by this flag. + enum: + - otel + - prom + type: string + mirrorMaxBatchSize: + description: MirrorMaxBatchSize is the maximum allowed batch size + for mirror sync. + format: int32 + type: integer + monitoring: + description: Monitoring configuration (OpenTelemetry). + properties: + attributes: + description: Attributes are additional OTEL resource attributes. + type: string + flightRecorder: + description: FlightRecorder configuration for runtime execution + trace buffering. + properties: + enabled: + description: Enabled enables the runtime flight recorder. + type: boolean + maxBytes: + anyOf: + - type: integer + - type: string + description: |- + MaxBytes is the maximum memory for the flight recorder buffer (e.g. "10Mi"). + Accepts Kubernetes quantity format. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + minAge: + description: MinAge is the minimum duration of trace data + retained in the buffer. + type: string + type: object + logs: + description: Logs configuration. + properties: + enabled: + description: Enabled enables log exporting. + type: boolean + endpoint: + description: Endpoint for the log exporter. + type: string + exporter: + description: Exporter type. + type: string + insecure: + description: Insecure disables TLS for the exporter. + type: string + level: + description: Level is the log level. + type: string + mode: + description: Mode is the exporter mode. + type: string + port: + description: Port for the log exporter. + type: string + type: object + metrics: + description: Metrics configuration. + properties: + enabled: + description: Enabled enables metrics. + type: boolean + endpoint: + description: Endpoint for the metrics exporter. + type: string + exporter: + description: Exporter type. + type: string + exporterPushInterval: + description: ExporterPushInterval is the push interval. + type: string + insecure: + description: Insecure disables TLS for the exporter. + type: string + keepInMemory: + description: KeepInMemory keeps metrics in memory. + type: boolean + mode: + description: Mode is the exporter mode. + type: string + port: + description: Port for the metrics exporter. + type: string + runtime: + description: Runtime enables runtime metrics. + type: boolean + runtimeMinimumReadMemStatsInterval: + description: RuntimeMinimumReadMemStatsInterval is the + minimum interval for reading mem stats. + type: string + type: object + pyroscope: + description: Pyroscope continuous profiling configuration. + properties: + applicationName: + description: ApplicationName overrides the application + name. + type: string + authToken: + description: AuthToken for Pyroscope authentication. + type: string + basicAuthPassword: + description: BasicAuthPassword for basic authentication. + type: string + basicAuthUser: + description: BasicAuthUser for basic authentication. + type: string + blockProfileRate: + description: BlockProfileRate is the block profile rate. + format: int32 + type: integer + disableGCRuns: + description: DisableGCRuns disables GC runs. + type: boolean + enabled: + description: Enabled enables Pyroscope profiling. + type: boolean + mutexProfileFraction: + description: MutexProfileFraction is the mutex profile + fraction. + format: int32 + type: integer + profileTypes: + description: ProfileTypes to collect. + type: string + serverAddress: + description: ServerAddress is the Pyroscope server address. + type: string + tags: + description: Tags in key=value,key2=value2 format. + type: string + tenantId: + description: TenantID for multi-tenant Pyroscope. + type: string + uploadRate: + description: UploadRate is the upload interval. + type: string + type: object + serviceName: + default: ledger + description: ServiceName for monitoring. + type: string + traces: + description: Traces configuration. + properties: + batch: + description: Batch enables batch mode. + type: string + enabled: + description: Enabled enables tracing. + type: boolean + endpoint: + description: Endpoint for the trace exporter. + type: string + exporter: + description: Exporter type (e.g., "otlp"). + type: string + insecure: + description: Insecure disables TLS for the exporter. + type: string + mode: + description: Mode is the exporter mode (e.g., "grpc"). + type: string + port: + description: Port for the trace exporter. + type: string + sampling: + description: Sampling configuration. + properties: + enabled: + description: Enabled enables error-aware trace sampling. + type: boolean + successRatio: + description: SuccessRatio is the sampling ratio for + successful traces (0.0-1.0). + type: string + type: object + type: object + type: object + networkPolicy: + description: NetworkPolicy configuration for egress restrictions. + properties: + additionalEgress: + description: |- + AdditionalEgress appends custom egress rules to the generated NetworkPolicy. + Use this to allow traffic to cluster-internal services (e.g. databases, message brokers). + items: + description: |- + NetworkPolicyEgressRule describes a particular set of traffic that is allowed out of pods + matched by a NetworkPolicySpec's podSelector. The traffic must match both ports and to. + This type is beta-level in 1.8 + properties: + ports: + description: |- + ports is a list of destination ports for outgoing traffic. + Each item in this list is combined using a logical OR. If this field is + empty or missing, this rule matches all ports (traffic not restricted by port). + If this field is present and contains at least one item, then this rule allows + traffic only if the traffic matches at least one port in the list. + items: + description: NetworkPolicyPort describes a port to + allow traffic on + properties: + endPort: + description: |- + endPort indicates that the range of ports from port to endPort if set, inclusive, + should be allowed by the policy. This field cannot be defined if the port field + is not defined or if the port field is defined as a named (string) port. + The endPort must be equal or greater than port. + format: int32 + type: integer + port: + anyOf: + - type: integer + - type: string + description: |- + port represents the port on the given protocol. This can either be a numerical or named + port on a pod. If this field is not provided, this matches all port names and + numbers. + If present, only traffic on the specified protocol AND port will be matched. + x-kubernetes-int-or-string: true + protocol: + description: |- + protocol represents the protocol (TCP, UDP, or SCTP) which traffic must match. + If not specified, this field defaults to TCP. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + to: + description: |- + to is a list of destinations for outgoing traffic of pods selected for this rule. + Items in this list are combined using a logical OR operation. If this field is + empty or missing, this rule matches all destinations (traffic not restricted by + destination). If this field is present and contains at least one item, this rule + allows traffic only if the traffic matches at least one item in the to list. + items: + description: |- + NetworkPolicyPeer describes a peer to allow traffic to/from. Only certain combinations of + fields are allowed + properties: + ipBlock: + description: |- + ipBlock defines policy on a particular IPBlock. If this field is set then + neither of the other fields can be. + properties: + cidr: + description: |- + cidr is a string representing the IPBlock + Valid examples are "192.168.1.0/24" or "2001:db8::/64" + type: string + except: + description: |- + except is a slice of CIDRs that should not be included within an IPBlock + Valid examples are "192.168.1.0/24" or "2001:db8::/64" + Except values will be rejected if they are outside the cidr range + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - cidr + type: object + namespaceSelector: + description: |- + namespaceSelector selects namespaces using cluster-scoped labels. This field follows + standard label selector semantics; if present but empty, it selects all namespaces. + + If podSelector is also set, then the NetworkPolicyPeer as a whole selects + the pods matching podSelector in the namespaces selected by namespaceSelector. + Otherwise it selects all pods in the namespaces selected by namespaceSelector. + properties: + matchExpressions: + description: matchExpressions is a list of + label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + podSelector: + description: |- + podSelector is a label selector which selects pods. This field follows standard label + selector semantics; if present but empty, it selects all pods. + + If namespaceSelector is also set, then the NetworkPolicyPeer as a whole selects + the pods matching podSelector in the Namespaces selected by NamespaceSelector. + Otherwise it selects the pods matching podSelector in the policy's own namespace. + properties: + matchExpressions: + description: matchExpressions is a list of + label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: array + enabled: + description: Enabled enables the egress NetworkPolicy. + type: boolean + externalCIDRExcept: + description: |- + ExternalCIDRExcept overrides the default RFC1918 CIDR blocks excluded from external egress. + Defaults to [10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16]. + items: + type: string + type: array + type: object + nodeSelector: + additionalProperties: + type: string + description: NodeSelector for pod scheduling. + type: object + numscriptCacheSize: + description: NumscriptCacheSize is the maximum number of parsed + Numscript programs to cache (LRU eviction). + format: int32 + type: integer + pebble: + description: Pebble storage engine configuration. + properties: + bytesPerSync: + anyOf: + - type: integer + - type: string + description: |- + BytesPerSync is bytes written before sync during flush/compaction (e.g. "512Ki"). + Accepts Kubernetes quantity format. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + cacheSize: + anyOf: + - type: integer + - type: string + description: |- + CacheSize is the block cache size (e.g. "1Gi", "512Mi"). + Accepts Kubernetes quantity format. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + compression: + description: |- + Compression is the per-level compression algorithm (L0-L6, comma-separated). + Options: none, snappy, zstd, fastest, fast, balanced, good, default. + type: string + disableWAL: + description: DisableWAL disables WAL entirely. + type: boolean + incrementalCompactThreshold: + description: |- + IncrementalCompactThreshold is the number of new log entries before + triggering an incremental compaction of the new range. + Default: 100000. + format: int64 + type: integer + l0CompactionThreshold: + description: L0CompactionThreshold is the L0 file count to + trigger compaction. + format: int32 + type: integer + l0StopWritesThreshold: + description: L0StopWritesThreshold is the L0 file count before + writes stop. + format: int32 + type: integer + lBaseMaxBytes: + anyOf: + - type: integer + - type: string + description: |- + LBaseMaxBytes is the maximum size of L1 (e.g. "64Mi", "256Mi"). + Accepts Kubernetes quantity format. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + maxCheckpoints: + description: |- + MaxCheckpoints is the maximum number of Pebble checkpoints to keep. + Default: 10. + format: int32 + type: integer + maxConcurrentCompactions: + description: MaxConcurrentCompactions is the maximum concurrent + compactions. + format: int32 + type: integer + memTableSize: + anyOf: + - type: integer + - type: string + description: |- + MemTableSize is the MemTable size (e.g. "256Mi", "1Gi"). + Accepts Kubernetes quantity format. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memTableStopWritesThreshold: + description: MemTableStopWritesThreshold is the number of + memtables before writes stop. + format: int32 + type: integer + targetFileSize: + anyOf: + - type: integer + - type: string + description: |- + TargetFileSize is the target SST file size (e.g. "64Mi"). + Accepts Kubernetes quantity format. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + valueSeparation: + description: ValueSeparation configuration for large value + storage in blob files. + properties: + enabled: + description: Enabled enables value separation (large values + stored in blob files). + type: boolean + garbageRatio: + description: |- + GarbageRatio is the blob garbage ratio before rewrite (0.0-1.0). + Default: 0.20. + type: string + maxDepth: + description: |- + MaxDepth is the max blob reference depth per SSTable. + Default: 4. + format: int32 + type: integer + minSize: + anyOf: + - type: integer + - type: string + description: |- + MinSize is the minimum value size for separation (e.g. "256", "1Ki"). + Accepts Kubernetes quantity format. Default: 256. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + rewriteAge: + description: |- + RewriteAge is the minimum blob file age before rewrite. + Default: 1h. + type: string + type: object + walBytesPerSync: + anyOf: + - type: integer + - type: string + description: |- + WalBytesPerSync is WAL bytes written before sync (e.g. "512Ki"). + Accepts Kubernetes quantity format. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + walMinSyncInterval: + description: WalMinSyncInterval is the minimum interval between + WAL syncs. + type: string + type: object + persistence: + description: Persistence configuration for WAL and data volumes. + properties: + coldCache: + description: |- + ColdCache persistence configuration for cold storage read cache. + Uses a separate volume to avoid filling the data disk when reading archived chapters. + properties: + accessMode: + default: ReadWriteOnce + description: |- + AccessMode for the PVC. + Mutually exclusive with hostPath. + type: string + hostPath: + description: |- + HostPath configures a host-local volume instead of a PVC. + When set, no PersistentVolumeClaim is created for this volume. + The pod uses a hostPath volume mounted from the specified path on the node. + Each pod gets an isolated subdirectory named after its ordinal index. + This is intended for NVMe instance stores where Raft replication provides durability. + Mutually exclusive with storageClass, accessMode, and volumeAttributesClassName. + properties: + path: + description: |- + Path on the host node (e.g. "/mnt/nvme0/data"). + Each pod gets an isolated subdirectory: /. + type: string + type: + default: DirectoryOrCreate + description: Type is the hostPath type. + enum: + - Directory + - DirectoryOrCreate + type: string + required: + - path + type: object + size: + anyOf: + - type: integer + - type: string + description: Size of the volume. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + storageClass: + description: |- + StorageClass for the PVC. + Mutually exclusive with hostPath. + type: string + volumeAttributesClassName: + description: |- + VolumeAttributesClassName is the name of the VolumeAttributesClass to use for the PVC. + Requires the VolumeAttributesClass feature gate to be enabled (beta in K8s 1.31+). + Mutually exclusive with hostPath. + type: string + type: object + data: + description: Data persistence configuration. + properties: + accessMode: + default: ReadWriteOnce + description: |- + AccessMode for the PVC. + Mutually exclusive with hostPath. + type: string + hostPath: + description: |- + HostPath configures a host-local volume instead of a PVC. + When set, no PersistentVolumeClaim is created for this volume. + The pod uses a hostPath volume mounted from the specified path on the node. + Each pod gets an isolated subdirectory named after its ordinal index. + This is intended for NVMe instance stores where Raft replication provides durability. + Mutually exclusive with storageClass, accessMode, and volumeAttributesClassName. + properties: + path: + description: |- + Path on the host node (e.g. "/mnt/nvme0/data"). + Each pod gets an isolated subdirectory: /. + type: string + type: + default: DirectoryOrCreate + description: Type is the hostPath type. + enum: + - Directory + - DirectoryOrCreate + type: string + required: + - path + type: object + size: + anyOf: + - type: integer + - type: string + description: Size of the volume. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + storageClass: + description: |- + StorageClass for the PVC. + Mutually exclusive with hostPath. + type: string + volumeAttributesClassName: + description: |- + VolumeAttributesClassName is the name of the VolumeAttributesClass to use for the PVC. + Requires the VolumeAttributesClass feature gate to be enabled (beta in K8s 1.31+). + Mutually exclusive with hostPath. + type: string + type: object + deletionProtection: + default: true + description: |- + DeletionProtection opts this ledger's PVCs and bound PVs into the + cluster-scoped volume deletion-protection admission policy. It defaults to + true (protected): the operator stamps the + `ledger.formance.com/deletion-protection: enabled` label on the volumes so + the policy selects them and rejects accidental DELETEs unless the object + carries the allow-deletion annotation. Set it explicitly to false to opt + out — the label is then removed and protection is lifted. The policy itself + must be installed cluster-wide by an admin via the Helm value + `pvcProtection.enabled` (on by default) — otherwise this flag has no effect + and the operator emits a DeletionProtectionInactive warning on the CR. + type: boolean + retentionPolicy: + description: RetentionPolicy for PVCs. + properties: + whenDeleted: + default: Retain + description: WhenDeleted policy. + type: string + whenScaled: + default: Retain + description: WhenScaled policy. + type: string + type: object + wal: + description: WAL persistence configuration. + properties: + accessMode: + default: ReadWriteOnce + description: |- + AccessMode for the PVC. + Mutually exclusive with hostPath. + type: string + hostPath: + description: |- + HostPath configures a host-local volume instead of a PVC. + When set, no PersistentVolumeClaim is created for this volume. + The pod uses a hostPath volume mounted from the specified path on the node. + Each pod gets an isolated subdirectory named after its ordinal index. + This is intended for NVMe instance stores where Raft replication provides durability. + Mutually exclusive with storageClass, accessMode, and volumeAttributesClassName. + properties: + path: + description: |- + Path on the host node (e.g. "/mnt/nvme0/data"). + Each pod gets an isolated subdirectory: /. + type: string + type: + default: DirectoryOrCreate + description: Type is the hostPath type. + enum: + - Directory + - DirectoryOrCreate + type: string + required: + - path + type: object + size: + anyOf: + - type: integer + - type: string + description: Size of the volume. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + storageClass: + description: |- + StorageClass for the PVC. + Mutually exclusive with hostPath. + type: string + volumeAttributesClassName: + description: |- + VolumeAttributesClassName is the name of the VolumeAttributesClass to use for the PVC. + Requires the VolumeAttributesClass feature gate to be enabled (beta in K8s 1.31+). + Mutually exclusive with hostPath. + type: string + type: object + type: object + podAnnotations: + additionalProperties: + type: string + description: PodAnnotations to add to each pod. + type: object + podAntiAffinity: + description: PodAntiAffinity configuration. + properties: + enabled: + default: true + description: Enabled enables pod anti-affinity. + type: boolean + topologyKey: + default: kubernetes.io/hostname + description: TopologyKey for anti-affinity. + type: string + type: + default: soft + description: Type is "soft" or "hard". + enum: + - soft + - hard + type: string + weight: + default: 100 + description: Weight for soft anti-affinity (1-100). + format: int32 + maximum: 100 + minimum: 1 + type: integer + type: object + podSecurityContext: + description: PodSecurityContext for the pod. + properties: + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + fsGroup: + description: |- + A special supplemental group that applies to all containers in a pod. + Some volume types allow the Kubelet to change the ownership of that volume + to be owned by the pod: + + 1. The owning GID will be the FSGroup + 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) + 3. The permission bits are OR'd with rw-rw---- + + If unset, the Kubelet will not modify the ownership and permissions of any volume. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + fsGroupChangePolicy: + description: |- + fsGroupChangePolicy defines behavior of changing ownership and permission of the volume + before being exposed inside Pod. This field will only apply to + volume types which support fsGroup based ownership(and permissions). + It will have no effect on ephemeral volume types such as: secret, configmaps + and emptydir. + Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. + Note that this field cannot be set when spec.os.name is windows. + type: string + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxChangePolicy: + description: |- + seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod. + It has no effect on nodes that do not support SELinux or to volumes does not support SELinux. + Valid values are "MountOption" and "Recursive". + + "Recursive" means relabeling of all files on all Pod volumes by the container runtime. + This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node. + + "MountOption" mounts all eligible Pod volumes with `-o context` mount option. + This requires all Pods that share the same volume to use the same SELinux label. + It is not possible to share the same volume among privileged and unprivileged Pods. + Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes + whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their + CSIDriver instance. Other volumes are always re-labelled recursively. + "MountOption" value is allowed only when SELinuxMount feature gate is enabled. + + If not specified and SELinuxMount feature gate is enabled, "MountOption" is used. + If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes + and "Recursive" for all other volumes. + + This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers. + + All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state. + Note that this field cannot be set when spec.os.name is windows. + type: string + seLinuxOptions: + description: |- + The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in SecurityContext. If set in + both SecurityContext and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies + to the container. + type: string + role: + description: Role is a SELinux role label that applies + to the container. + type: string + type: + description: Type is a SELinux type label that applies + to the container. + type: string + user: + description: User is a SELinux user label that applies + to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + supplementalGroups: + description: |- + A list of groups applied to the first process run in each container, in + addition to the container's primary GID and fsGroup (if specified). If + the SupplementalGroupsPolicy feature is enabled, the + supplementalGroupsPolicy field determines whether these are in addition + to or instead of any group memberships defined in the container image. + If unspecified, no additional groups are added, though group memberships + defined in the container image may still be used, depending on the + supplementalGroupsPolicy field. + Note that this field cannot be set when spec.os.name is windows. + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + description: |- + Defines how supplemental groups of the first container processes are calculated. + Valid values are "Merge" and "Strict". If not specified, "Merge" is used. + (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled + and the container runtime must implement support for this feature. + Note that this field cannot be set when spec.os.name is windows. + type: string + sysctls: + description: |- + Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported + sysctls (by the container runtime) might fail to launch. + Note that this field cannot be set when spec.os.name is windows. + items: + description: Sysctl defines a kernel parameter to be set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options within a container's SecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the + GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + queryProfileThreshold: + description: QueryProfileThreshold logs and emits OTel attributes + for queries exceeding this duration (0 to disable). + type: string + raft: + description: Raft consensus configuration. + properties: + compactionMargin: + description: CompactionMargin is the compaction margin. + format: int32 + type: integer + electionTick: + description: ElectionTick is the election timeout in ticks. + format: int32 + type: integer + heartbeatTick: + description: HeartbeatTick is the heartbeat interval in ticks. + format: int32 + type: integer + learnerPromotionThreshold: + description: LearnerPromotionThreshold is the max log entry + lag before auto-promoting a learner. + format: int32 + type: integer + maintenanceInterval: + description: |- + MaintenanceInterval is the interval for background WAL snapshot + Pebble checkpoint. + Default: 30s. + type: string + maxInflightMsgs: + description: MaxInflightMsgs is the maximum number of in-flight + messages. + format: int32 + type: integer + maxSizePerMsg: + anyOf: + - type: integer + - type: string + description: |- + MaxSizePerMsg is the maximum size per message (e.g. "1Mi", "4Mi"). + Accepts Kubernetes quantity format. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + processingTickInterval: + description: |- + ProcessingTickInterval is the interval for processing committed entries. + Default: tickInterval/10. + type: string + proposeQueueCapacity: + description: ProposeQueueCapacity is the capacity of the propose + queue. + format: int32 + type: integer + replayBatchSize: + description: ReplayBatchSize is the number of Raft log entries + replayed per batch on startup. + format: int32 + type: integer + tickInterval: + description: TickInterval is the interval between Raft ticks. + type: string + transport: + description: Transport queue configuration. + properties: + bufferSize: + anyOf: + - type: integer + - type: string + description: |- + BufferSize is the per-peer send buffer capacity (e.g. "20Mi"). + Accepts Kubernetes quantity format. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + receptionQueues: + description: ReceptionQueues are the reception queue capacities + per priority. + items: + format: int32 + type: integer + type: array + sendQueues: + description: SendQueues are the send queue capacities + per priority. + items: + format: int32 + type: integer + type: array + type: object + type: object + readIndex: + description: ReadIndex configuration for the Pebble read index + store. + properties: + batchSize: + description: |- + BatchSize is the number of log entries per Pebble batch commit. + Higher values amortize commit overhead but use more memory. + Default: 1000. + format: int32 + type: integer + pebble: + description: |- + Pebble holds the common Pebble tunables for the read index. + Uses the same knobs as the primary store (cache size, memtable, L0 thresholds, etc.). + properties: + bytesPerSync: + anyOf: + - type: integer + - type: string + description: |- + BytesPerSync is bytes written before sync during flush/compaction (e.g. "512Ki"). + Accepts Kubernetes quantity format. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + cacheSize: + anyOf: + - type: integer + - type: string + description: |- + CacheSize is the block cache size (e.g. "1Gi", "512Mi"). + Accepts Kubernetes quantity format. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + compression: + description: |- + Compression is the per-level compression algorithm (L0-L6, comma-separated). + Options: none, snappy, zstd, fastest, fast, balanced, good, default. + type: string + disableWAL: + description: DisableWAL disables WAL entirely. + type: boolean + incrementalCompactThreshold: + description: |- + IncrementalCompactThreshold is the number of new log entries before + triggering an incremental compaction of the new range. + Default: 100000. + format: int64 + type: integer + l0CompactionThreshold: + description: L0CompactionThreshold is the L0 file count + to trigger compaction. + format: int32 + type: integer + l0StopWritesThreshold: + description: L0StopWritesThreshold is the L0 file count + before writes stop. + format: int32 + type: integer + lBaseMaxBytes: + anyOf: + - type: integer + - type: string + description: |- + LBaseMaxBytes is the maximum size of L1 (e.g. "64Mi", "256Mi"). + Accepts Kubernetes quantity format. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + maxCheckpoints: + description: |- + MaxCheckpoints is the maximum number of Pebble checkpoints to keep. + Default: 10. + format: int32 + type: integer + maxConcurrentCompactions: + description: MaxConcurrentCompactions is the maximum concurrent + compactions. + format: int32 + type: integer + memTableSize: + anyOf: + - type: integer + - type: string + description: |- + MemTableSize is the MemTable size (e.g. "256Mi", "1Gi"). + Accepts Kubernetes quantity format. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memTableStopWritesThreshold: + description: MemTableStopWritesThreshold is the number + of memtables before writes stop. + format: int32 + type: integer + targetFileSize: + anyOf: + - type: integer + - type: string + description: |- + TargetFileSize is the target SST file size (e.g. "64Mi"). + Accepts Kubernetes quantity format. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + valueSeparation: + description: ValueSeparation configuration for large value + storage in blob files. + properties: + enabled: + description: Enabled enables value separation (large + values stored in blob files). + type: boolean + garbageRatio: + description: |- + GarbageRatio is the blob garbage ratio before rewrite (0.0-1.0). + Default: 0.20. + type: string + maxDepth: + description: |- + MaxDepth is the max blob reference depth per SSTable. + Default: 4. + format: int32 + type: integer + minSize: + anyOf: + - type: integer + - type: string + description: |- + MinSize is the minimum value size for separation (e.g. "256", "1Ki"). + Accepts Kubernetes quantity format. Default: 256. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + rewriteAge: + description: |- + RewriteAge is the minimum blob file age before rewrite. + Default: 1h. + type: string + type: object + walBytesPerSync: + anyOf: + - type: integer + - type: string + description: |- + WalBytesPerSync is WAL bytes written before sync (e.g. "512Ki"). + Accepts Kubernetes quantity format. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + walMinSyncInterval: + description: WalMinSyncInterval is the minimum interval + between WAL syncs. + type: string + type: object + type: object + readinessProbe: + description: ReadinessProbe overrides the default readiness probe + for the ledger container. + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number must + be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + receiptSigning: + description: ReceiptSigning configures HMAC signing for JWT transaction + receipts. + properties: + secretKey: + default: key + description: SecretKey is the key in the secret containing + the HMAC key. + type: string + secretName: + description: SecretName is the Kubernetes secret containing + the HMAC key. + type: string + type: object + replicas: + default: 3 + description: Replicas is the number of Raft nodes. + format: int32 + minimum: 1 + type: integer + resources: + description: Resources for the ledger container. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + responseSigning: + description: ResponseSigning configuration (Ed25519). + properties: + enabled: + description: Enabled enables response signing. + type: boolean + secretKey: + default: seed + description: SecretKey is the key in the secret containing + the seed. + type: string + secretName: + description: SecretName is the Kubernetes secret containing + the Ed25519 seed. + type: string + type: object + restore: + description: Restore starts the server in restore mode. + type: boolean + securityContext: + description: SecurityContext for the container. + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies + to the container. + type: string + role: + description: Role is a SELinux role label that applies + to the container. + type: string + type: + description: Type is a SELinux type label that applies + to the container. + type: string + user: + description: User is a SELinux user label that applies + to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the + GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + sentinelMode: + description: |- + SentinelMode enables runtime volume consistency assertions + (monotonicity, delta/posting cross-check, post-commit cache/Pebble verification). + type: boolean + service: + description: Service configuration for the ClusterIP service. + properties: + annotations: + additionalProperties: + type: string + description: Annotations to add to the service. + type: object + grpcPort: + default: 8888 + description: GrpcPort is the gRPC service port. + format: int32 + type: integer + httpPort: + default: 9000 + description: HttpPort is the HTTP service port. + format: int32 + type: integer + raftPort: + default: 7777 + description: RaftPort is the Raft transport port. + format: int32 + type: integer + type: + default: ClusterIP + description: Type is the service type. + type: string + type: object + serviceAccount: + description: ServiceAccount configuration. + properties: + annotations: + additionalProperties: + type: string + description: Annotations to add to the service account. + type: object + create: + description: Create specifies whether to create a service + account. + type: boolean + name: + description: Name overrides the service account name. + type: string + type: object + snapshot: + description: Snapshot sync configuration for Raft snapshot transfers. + properties: + fileRetryCount: + description: |- + FileRetryCount is the per-file retry attempts during snapshot sync on transient stream errors. + Default: 3. + format: int32 + type: integer + parallelism: + description: |- + Parallelism is the number of parallel file fetch workers during snapshot sync. + Default: 4. + format: int32 + type: integer + retryCount: + description: |- + RetryCount is the session-level retry attempts for snapshot sync on transient errors. + Default: 5. + format: int32 + type: integer + sessionTTL: + description: |- + SessionTTL is the server-side session TTL for snapshot sync. + Default: 5m. + type: string + type: object + startupProbe: + description: StartupProbe overrides the default startup probe + for the ledger container. + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number must + be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + tls: + description: TLS configuration for gRPC connections. + properties: + caSecretKey: + description: CASecretKey is the key for the CA certificate + in the secret. + type: string + enabled: + description: Enabled enables TLS. + type: boolean + secretName: + description: SecretName is the Kubernetes secret containing + TLS certificate and key. + type: string + type: object + tolerations: + description: Tolerations for pod scheduling. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + topologySpreadConstraints: + description: |- + TopologySpreadConstraints describe how pods are spread across failure + domains (e.g. zones, nodes). See PodSpec.topologySpreadConstraints. The + pod label selector defaults to the Cluster's selector when omitted. + items: + description: TopologySpreadConstraint specifies how to spread + matching pods among the given topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + unsafeSkipConfigValidation: + description: |- + UnsafeSkipConfigValidation skips startup configuration safety checks. + DANGEROUS: allows node-id/cluster-id changes on existing data. + type: boolean + walDir: + default: /data/raft + description: WalDir is the WAL data directory. + type: string + x-kubernetes-validations: + - message: walDir is immutable once set + rule: oldSelf == '' || self == oldSelf + type: object + stacks: + description: |- + Stacks on which the configuration is applied. Can contain `*` to + indicate a wildcard, following the same convention as Settings. + items: + type: string + type: array + type: object + type: object + served: true + storage: true diff --git a/helm/operator/templates/gen/rbac.authorization.k8s.io_v1_clusterrole_formance-manager-role.yaml b/helm/operator/templates/gen/rbac.authorization.k8s.io_v1_clusterrole_formance-manager-role.yaml index f1494fd7b..2057fa6a6 100644 --- a/helm/operator/templates/gen/rbac.authorization.k8s.io_v1_clusterrole_formance-manager-role.yaml +++ b/helm/operator/templates/gen/rbac.authorization.k8s.io_v1_clusterrole_formance-manager-role.yaml @@ -192,6 +192,14 @@ rules: - patch - update - watch +- apiGroups: + - formance.com + resources: + - ledgerconfigurations + verbs: + - get + - list + - watch - apiGroups: - ledger.formance.com resources: diff --git a/internal/core/setup.go b/internal/core/setup.go index 32064b8ed..49b049f8f 100644 --- a/internal/core/setup.go +++ b/internal/core/setup.go @@ -22,6 +22,9 @@ func Setup(mgr ctrl.Manager, platform Platform) error { if err := indexSettings(mgr); err != nil { return err } + if err := indexLedgerConfigurations(mgr); err != nil { + return err + } wrappedMgr := NewDefaultManager(mgr, platform) for _, initializer := range initializers { @@ -33,6 +36,17 @@ func Setup(mgr ctrl.Manager, platform Platform) error { return nil } +func indexLedgerConfigurations(mgr ctrl.Manager) error { + if err := mgr.GetFieldIndexer(). + IndexField(context.Background(), &v1beta1.LedgerConfiguration{}, "stack", func(object client.Object) []string { + return object.(*v1beta1.LedgerConfiguration).GetStacks() + }); err != nil { + mgr.GetLogger().Error(err, "indexing stack field", "type", &v1beta1.LedgerConfiguration{}) + return err + } + return nil +} + // indexStackDependentsObjects automatically add an index on `stack` property for all stack dependents objects func indexStackDependentsObjects(mgr ctrl.Manager) error { for _, rtype := range mgr.GetScheme().AllKnownTypes() { diff --git a/internal/resources/ledgers/init.go b/internal/resources/ledgers/init.go index c17118ffe..657cf1d1a 100644 --- a/internal/resources/ledgers/init.go +++ b/internal/resources/ledgers/init.go @@ -173,6 +173,7 @@ func init() { WithOwn[*v1beta1.Ledger](&corev1.ConfigMap{}), WithOwn[*v1beta1.Ledger](&v1beta1.BenthosStream{}), withLedgerV3ClusterWatch(), + withLedgerConfigurationWatch(), WithWatchSettings[*v1beta1.Ledger](), WithWatchDependency[*v1beta1.Ledger](&v1beta1.Auth{}), WithWatchDependency[*v1beta1.Ledger](&v1beta1.Search{}), diff --git a/internal/resources/ledgers/v3.go b/internal/resources/ledgers/v3.go index c4f535c10..d738ee58d 100644 --- a/internal/resources/ledgers/v3.go +++ b/internal/resources/ledgers/v3.go @@ -3,7 +3,6 @@ package ledgers import ( "fmt" "slices" - "strconv" "strings" "golang.org/x/mod/semver" @@ -17,8 +16,12 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + ledgerv1alpha1 "github.com/formancehq/ledger/misc/operator/api/v1alpha1" "github.com/formancehq/operator/v3/api/formance.com/v1beta1" "github.com/formancehq/operator/v3/internal/core" @@ -50,6 +53,7 @@ var ( ) //+kubebuilder:rbac:groups=ledger.formance.com,resources=clusters,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=formance.com,resources=ledgerconfigurations,verbs=get;list;watch func isLedgerV3(version string) bool { normalizedVersion := version @@ -82,6 +86,33 @@ func withLedgerV3ClusterWatch() core.ReconcilerOption[*v1beta1.Ledger] { } } +func withLedgerConfigurationWatch() core.ReconcilerOption[*v1beta1.Ledger] { + return core.WithWatch[*v1beta1.Ledger, *v1beta1.LedgerConfiguration]( + func(ctx core.Context, configuration *v1beta1.LedgerConfiguration) []reconcile.Request { + if configuration.IsWildcard() { + return core.BuildReconcileRequests( + ctx, + ctx.GetClient(), + ctx.GetScheme(), + &v1beta1.Ledger{}, + ) + } + + requests := make([]reconcile.Request, 0) + for _, stack := range configuration.GetStacks() { + requests = append(requests, core.BuildReconcileRequests( + ctx, + ctx.GetClient(), + ctx.GetScheme(), + &v1beta1.Ledger{}, + client.MatchingFields{"stack": stack}, + )...) + } + return requests + }, + ) +} + func watchLedgerV3Resource( ctx core.Context, b *builder.Builder, @@ -232,6 +263,11 @@ func normalizeLedgerV3Replicas(configured int32) (int32, bool, error) { } func createOrUpdateV3Cluster(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, version string, preview bool, tlsCAHash string) (*unstructured.Unstructured, error) { + baseSpec, err := ledgerV3BaseSpec(ctx, stack.Name) + if err != nil { + return nil, err + } + image, err := registries.GetFormanceImage(ctx, stack, "ledger", version) if err != nil { return nil, err @@ -265,20 +301,49 @@ func createOrUpdateV3Cluster(ctx core.Context, stack *v1beta1.Stack, ledger *v1b if err != nil { return nil, err } - extraEnv, err := buildV3ExtraEnv(stack, ledger) + serviceAccountName, err := settings.GetAWSServiceAccount(ctx, stack.Name) if err != nil { return nil, err } - - serviceAccountName, err := settings.GetAWSServiceAccount(ctx, stack.Name) + desiredSpec := composeLedgerV3ClusterSpec(baseSpec, ledgerV3SpecOverrides{ + ImageRepository: imageRepository(image), + ImageTag: image.Version, + ImagePullSecrets: image.PullSecrets, + Replicas: replicas, + ClusterID: stack.Name, + Debug: stack.Spec.Debug || ledger.Spec.Debug, + TLSSecretName: ledgerV3TLSName(stack.Name), + TLSCAHash: tlsCAHash, + Preview: preview, + Resources: resourceRequirements, + ExtraEnv: core.GetDevEnvVars(stack, ledger), + Monitoring: monitoringConfiguration, + Auth: authConfiguration, + ServiceAccountName: serviceAccountName, + }) + desiredSpecMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(desiredSpec) if err != nil { - return nil, err + return nil, fmt.Errorf("converting Ledger v3 Cluster spec: %w", err) + } + // ResourceRequirements is a value struct in the Ledger API. The + // unstructured converter therefore emits an empty resources object even + // though the JSON field is tagged omitempty. Remove it when neither the + // shared configuration nor Settings provided resources, preserving the + // historical absence of the field. + if !hasResourceRequirements(&baseSpec.Resources) && !hasResourceRequirements(resourceRequirements) { + delete(desiredSpecMap, "resources") } cluster := newV3Cluster() cluster.SetNamespace(stack.Name) cluster.SetName(stack.Name) _, err = controllerutil.CreateOrUpdate(ctx, ctx.GetClient(), cluster, func() error { + // Reset the desired spec to the shared configuration on every + // reconciliation. Stack-specific values below deliberately override it. + if err := unstructured.SetNestedMap(cluster.Object, desiredSpecMap, "spec"); err != nil { + return err + } + labels := cluster.GetLabels() if labels == nil { labels = map[string]string{} @@ -296,207 +361,49 @@ func createOrUpdateV3Cluster(ctx core.Context, stack *v1beta1.Stack, ledger *v1b return err } - if err := unstructured.SetNestedField(cluster.Object, imageRepository(image), "spec", "image", "repository"); err != nil { - return err - } - if err := unstructured.SetNestedField(cluster.Object, image.Version, "spec", "image", "tag"); err != nil { - return err - } - if err := unstructured.SetNestedField(cluster.Object, int64(replicas), "spec", "replicas"); err != nil { - return err - } - if err := unstructured.SetNestedField(cluster.Object, stack.Name, "spec", "clusterID"); err != nil { - return err - } - if err := unstructured.SetNestedField(cluster.Object, stack.Spec.Debug || ledger.Spec.Debug, "spec", "debug"); err != nil { - return err - } - // Configure TLS from the first Cluster revision. Pods can wait for the - // cert-manager Secret, but must never bootstrap a plaintext Raft cluster - // that is switched to TLS by a later StatefulSet revision. - if err := unstructured.SetNestedMap(cluster.Object, ledgerV3TLSSpec(stack.Name), "spec", "tls"); err != nil { - return err - } - podAnnotations, _, err := unstructured.NestedStringMap(cluster.Object, "spec", "podAnnotations") - if err != nil { - return err - } - if podAnnotations == nil { - podAnnotations = map[string]string{} - } - if tlsCAHash != "" { - podAnnotations[ledgerV3TLSCAHashAnnotation] = tlsCAHash - } else { - delete(podAnnotations, ledgerV3TLSCAHashAnnotation) - } - if len(podAnnotations) > 0 { - if err := unstructured.SetNestedStringMap(cluster.Object, podAnnotations, "spec", "podAnnotations"); err != nil { - return err - } - } else { - unstructured.RemoveNestedField(cluster.Object, "spec", "podAnnotations") - } - additionalLabels := map[string]string{} - if preview { - additionalLabels["app.kubernetes.io/name"] = "ledger-v3-preview" - additionalLabels[ledgerV3PreviewLabel] = "true" - } - if len(additionalLabels) > 0 { - if err := unstructured.SetNestedStringMap(cluster.Object, additionalLabels, "spec", "additionalLabels"); err != nil { - return err - } - } else { - unstructured.RemoveNestedField(cluster.Object, "spec", "additionalLabels") - } - - if hasResourceRequirements(resourceRequirements) { - resources, err := runtime.DefaultUnstructuredConverter.ToUnstructured(resourceRequirements) - if err != nil { - return err - } - if err := unstructured.SetNestedMap(cluster.Object, resources, "spec", "resources"); err != nil { - return err - } - } else { - unstructured.RemoveNestedField(cluster.Object, "spec", "resources") - } - - if len(image.PullSecrets) > 0 { - pullSecrets := make([]any, 0, len(image.PullSecrets)) - for _, pullSecret := range image.PullSecrets { - pullSecrets = append(pullSecrets, map[string]any{"name": pullSecret.Name}) - } - if err := unstructured.SetNestedSlice(cluster.Object, pullSecrets, "spec", "imagePullSecrets"); err != nil { - return err - } - } else { - unstructured.RemoveNestedField(cluster.Object, "spec", "imagePullSecrets") - } - - if len(extraEnv) > 0 { - if err := unstructured.SetNestedSlice(cluster.Object, extraEnv, "spec", "extraEnv"); err != nil { - return err - } - } else { - unstructured.RemoveNestedField(cluster.Object, "spec", "extraEnv") - } - - if monitoringConfiguration != nil { - if err := unstructured.SetNestedMap(cluster.Object, ledgerV3MonitoringSpec(monitoringConfiguration), "spec", "monitoring"); err != nil { - return err - } - } else { - unstructured.RemoveNestedField(cluster.Object, "spec", "monitoring") - } - - if authConfiguration != nil { - if err := unstructured.SetNestedMap(cluster.Object, ledgerV3AuthSpec(authConfiguration), "spec", "auth"); err != nil { - return err - } - } else { - unstructured.RemoveNestedField(cluster.Object, "spec", "auth") - } - - if serviceAccountName != "" { - if err := unstructured.SetNestedField(cluster.Object, false, "spec", "serviceAccount", "create"); err != nil { - return err - } - if err := unstructured.SetNestedField(cluster.Object, serviceAccountName, "spec", "serviceAccount", "name"); err != nil { - return err - } - } else { - unstructured.RemoveNestedField(cluster.Object, "spec", "serviceAccount") - } - return nil }) return cluster, err } -func hasResourceRequirements(resources *corev1.ResourceRequirements) bool { - return resources != nil && (len(resources.Limits) > 0 || len(resources.Requests) > 0 || len(resources.Claims) > 0) -} - -func imageRepository(image *registries.ImageConfiguration) string { - if image.Registry == "" { - return image.Image +func ledgerV3BaseSpec(ctx core.Context, stack string) (*ledgerv1alpha1.ClusterSpec, error) { + stackConfigurations := &v1beta1.LedgerConfigurationList{} + if err := ctx.GetClient().List(ctx, stackConfigurations, client.MatchingFields{"stack": stack}); err != nil { + return nil, fmt.Errorf("listing LedgerConfigurations for stack %q: %w", stack, err) } - return strings.TrimSuffix(image.Registry, "/") + "/" + image.Image -} -func buildV3ExtraEnv(stack *v1beta1.Stack, ledger *v1beta1.Ledger) ([]any, error) { - env := core.GetDevEnvVars(stack, ledger) - - ret := make([]any, 0, len(env)) - for i := range env { - value, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&env[i]) - if err != nil { - return nil, err - } - ret = append(ret, value) + wildcardConfigurations := &v1beta1.LedgerConfigurationList{} + if err := ctx.GetClient().List(ctx, wildcardConfigurations, client.MatchingFields{"stack": "*"}); err != nil { + return nil, fmt.Errorf("listing wildcard LedgerConfigurations: %w", err) } - return ret, nil -} -func ledgerV3AuthSpec(configuration *auths.ProtectedAuthConfiguration) map[string]any { - spec := map[string]any{ - "enabled": true, - "issuer": configuration.Issuer, - } - if len(configuration.Issuers) > 0 { - issuers := make([]any, 0, len(configuration.Issuers)) - for _, issuer := range configuration.Issuers { - issuers = append(issuers, issuer) + configurations := append(stackConfigurations.Items, wildcardConfigurations.Items...) + slices.SortStableFunc(configurations, func(a, b v1beta1.LedgerConfiguration) int { + switch { + case a.IsWildcard() && !b.IsWildcard(): + return 1 + case !a.IsWildcard() && b.IsWildcard(): + return -1 + default: + return strings.Compare(a.Name, b.Name) } - spec["issuers"] = issuers - } - if configuration.ReadKeySetMaxRetries != 0 { - spec["readKeySetMaxRetries"] = int64(configuration.ReadKeySetMaxRetries) - } - if configuration.CheckScopes { - spec["checkScopes"] = true - spec["service"] = configuration.Service + }) + if len(configurations) == 0 { + return &ledgerv1alpha1.ClusterSpec{}, nil } - return spec + + return configurations[0].Spec.Cluster.DeepCopy(), nil } -func ledgerV3MonitoringSpec(configuration *settings.OpenTelemetryConfiguration) map[string]any { - spec := map[string]any{ - "serviceName": configuration.ServiceName, - } - if len(configuration.Attributes) > 0 { - attributes := make([]string, 0, len(configuration.Attributes)) - for key, value := range configuration.Attributes { - attributes = append(attributes, fmt.Sprintf("%s=%s", key, value)) - } - slices.Sort(attributes) - spec["attributes"] = strings.Join(attributes, ",") - } - if configuration.Traces != nil { - traces := ledgerV3MonitoringSignalSpec(configuration.Traces) - traces["batch"] = "true" - spec["traces"] = traces - } - if configuration.Metrics != nil { - metrics := ledgerV3MonitoringSignalSpec(configuration.Metrics) - metrics["runtime"] = true - spec["metrics"] = metrics - } - if configuration.Logs != nil { - spec["logs"] = ledgerV3MonitoringSignalSpec(configuration.Logs) - } - return spec +func hasResourceRequirements(resources *corev1.ResourceRequirements) bool { + return resources != nil && (len(resources.Limits) > 0 || len(resources.Requests) > 0 || len(resources.Claims) > 0) } -func ledgerV3MonitoringSignalSpec(configuration *settings.OpenTelemetrySignalConfiguration) map[string]any { - return map[string]any{ - "enabled": true, - "exporter": "otlp", - "endpoint": configuration.Endpoint, - "port": configuration.Port, - "insecure": strconv.FormatBool(configuration.Insecure), - "mode": configuration.Mode, +func imageRepository(image *registries.ImageConfiguration) string { + if image.Registry == "" { + return image.Image } + return strings.TrimSuffix(image.Registry, "/") + "/" + image.Image } func isV3ClusterReady(cluster *unstructured.Unstructured) (bool, string, error) { diff --git a/internal/resources/ledgers/v3_spec.go b/internal/resources/ledgers/v3_spec.go new file mode 100644 index 000000000..514f70946 --- /dev/null +++ b/internal/resources/ledgers/v3_spec.go @@ -0,0 +1,185 @@ +package ledgers + +import ( + "slices" + "strconv" + "strings" + + corev1 "k8s.io/api/core/v1" + + ledgerv1alpha1 "github.com/formancehq/ledger/misc/operator/api/v1alpha1" + + "github.com/formancehq/operator/v3/internal/core" + "github.com/formancehq/operator/v3/internal/resources/auths" + "github.com/formancehq/operator/v3/internal/resources/settings" +) + +type ledgerV3SpecOverrides struct { + ImageRepository string + ImageTag string + ImagePullSecrets []corev1.LocalObjectReference + Replicas int32 + ClusterID string + Debug bool + TLSSecretName string + TLSCAHash string + Preview bool + Resources *corev1.ResourceRequirements + ExtraEnv []corev1.EnvVar + Monitoring *settings.OpenTelemetryConfiguration + Auth *auths.ProtectedAuthConfiguration + ServiceAccountName string +} + +// composeLedgerV3ClusterSpec applies the values owned by the Formance Operator +// to a shared ClusterSpec. Unmanaged fields remain entirely owned by the +// LedgerConfiguration. +func composeLedgerV3ClusterSpec(base *ledgerv1alpha1.ClusterSpec, overrides ledgerV3SpecOverrides) *ledgerv1alpha1.ClusterSpec { + spec := base.DeepCopy() + + spec.Image.Repository = overrides.ImageRepository + spec.Image.Tag = overrides.ImageTag + if len(overrides.ImagePullSecrets) > 0 { + spec.ImagePullSecrets = slices.Clone(overrides.ImagePullSecrets) + } + spec.Replicas = pointerTo(overrides.Replicas) + spec.ClusterID = overrides.ClusterID + spec.Debug = overrides.Debug + // TLS is configured from the first Cluster revision. Pods may wait for the + // cert-manager Secret, but must never bootstrap a plaintext Raft cluster. + spec.TLS = &ledgerv1alpha1.TLSConfig{ + Enabled: true, + SecretName: overrides.TLSSecretName, + CASecretKey: ledgerV3TLSCASecretKey, + } + + if spec.PodAnnotations == nil { + spec.PodAnnotations = map[string]string{} + } + if overrides.TLSCAHash != "" { + spec.PodAnnotations[ledgerV3TLSCAHashAnnotation] = overrides.TLSCAHash + } else { + delete(spec.PodAnnotations, ledgerV3TLSCAHashAnnotation) + } + if len(spec.PodAnnotations) == 0 { + spec.PodAnnotations = nil + } + + if overrides.Preview { + if spec.AdditionalLabels == nil { + spec.AdditionalLabels = map[string]string{} + } + spec.AdditionalLabels["app.kubernetes.io/name"] = "ledger-v3-preview" + spec.AdditionalLabels[ledgerV3PreviewLabel] = "true" + } + + if hasResourceRequirements(overrides.Resources) { + spec.Resources = *overrides.Resources.DeepCopy() + } + if len(overrides.ExtraEnv) > 0 { + spec.ExtraEnv = core.MergeEnvVars(spec.ExtraEnv, overrides.ExtraEnv) + } + applyLedgerV3Monitoring(spec, overrides.Monitoring) + applyLedgerV3Auth(spec, overrides.Auth) + + if overrides.ServiceAccountName != "" { + spec.ServiceAccount.Create = pointerTo(false) + spec.ServiceAccount.Name = overrides.ServiceAccountName + } + + return spec +} + +func applyLedgerV3Auth(spec *ledgerv1alpha1.ClusterSpec, configuration *auths.ProtectedAuthConfiguration) { + if configuration == nil { + return + } + if spec.Auth == nil { + spec.Auth = &ledgerv1alpha1.AuthorizationConfig{} + } + spec.Auth.Enabled = pointerTo(true) + spec.Auth.Issuer = configuration.Issuer + if len(configuration.Issuers) > 0 { + spec.Auth.Issuers = slices.Clone(configuration.Issuers) + } + spec.Auth.CheckScopes = pointerTo(configuration.CheckScopes) + if configuration.CheckScopes { + spec.Auth.Service = configuration.Service + } + if configuration.ReadKeySetMaxRetries != 0 { + spec.Auth.ReadKeySetMaxRetries = pointerTo(int32(configuration.ReadKeySetMaxRetries)) + } +} + +func applyLedgerV3Monitoring(spec *ledgerv1alpha1.ClusterSpec, configuration *settings.OpenTelemetryConfiguration) { + if configuration == nil { + return + } + if spec.Monitoring == nil { + spec.Monitoring = &ledgerv1alpha1.MonitoringConfig{} + } + spec.Monitoring.ServiceName = configuration.ServiceName + if len(configuration.Attributes) > 0 { + attributes := make([]string, 0, len(configuration.Attributes)) + for key, value := range configuration.Attributes { + attributes = append(attributes, key+"="+value) + } + slices.Sort(attributes) + spec.Monitoring.Attributes = strings.Join(attributes, ",") + } + if configuration.Traces != nil { + if spec.Monitoring.Traces == nil { + spec.Monitoring.Traces = &ledgerv1alpha1.TracesConfig{} + } + applyLedgerV3MonitoringSignal(configuration.Traces, + &spec.Monitoring.Traces.Enabled, + &spec.Monitoring.Traces.Exporter, + &spec.Monitoring.Traces.Endpoint, + &spec.Monitoring.Traces.Port, + &spec.Monitoring.Traces.Insecure, + &spec.Monitoring.Traces.Mode) + spec.Monitoring.Traces.Batch = "true" + } + if configuration.Metrics != nil { + if spec.Monitoring.Metrics == nil { + spec.Monitoring.Metrics = &ledgerv1alpha1.MetricsConfig{} + } + applyLedgerV3MonitoringSignal(configuration.Metrics, + &spec.Monitoring.Metrics.Enabled, + &spec.Monitoring.Metrics.Exporter, + &spec.Monitoring.Metrics.Endpoint, + &spec.Monitoring.Metrics.Port, + &spec.Monitoring.Metrics.Insecure, + &spec.Monitoring.Metrics.Mode) + spec.Monitoring.Metrics.Runtime = pointerTo(true) + } + if configuration.Logs != nil { + if spec.Monitoring.Logs == nil { + spec.Monitoring.Logs = &ledgerv1alpha1.LogsConfig{} + } + applyLedgerV3MonitoringSignal(configuration.Logs, + &spec.Monitoring.Logs.Enabled, + &spec.Monitoring.Logs.Exporter, + &spec.Monitoring.Logs.Endpoint, + &spec.Monitoring.Logs.Port, + &spec.Monitoring.Logs.Insecure, + &spec.Monitoring.Logs.Mode) + } +} + +func applyLedgerV3MonitoringSignal( + configuration *settings.OpenTelemetrySignalConfiguration, + enabled **bool, + exporter, endpoint, port, insecure, mode *string, +) { + *enabled = pointerTo(true) + *exporter = "otlp" + *endpoint = configuration.Endpoint + *port = configuration.Port + *insecure = strconv.FormatBool(configuration.Insecure) + *mode = configuration.Mode +} + +func pointerTo[T any](value T) *T { + return &value +} diff --git a/internal/resources/ledgers/v3_spec_test.go b/internal/resources/ledgers/v3_spec_test.go new file mode 100644 index 000000000..eda98bad9 --- /dev/null +++ b/internal/resources/ledgers/v3_spec_test.go @@ -0,0 +1,151 @@ +package ledgers + +import ( + "testing" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + + ledgerv1alpha1 "github.com/formancehq/ledger/misc/operator/api/v1alpha1" + + "github.com/formancehq/operator/v3/internal/resources/auths" + "github.com/formancehq/operator/v3/internal/resources/settings" +) + +func TestComposeLedgerV3ClusterSpec(t *testing.T) { + t.Parallel() + + base := &ledgerv1alpha1.ClusterSpec{ + Image: ledgerv1alpha1.ImageSpec{PullPolicy: corev1.PullAlways}, + Monitoring: &ledgerv1alpha1.MonitoringConfig{ + Pyroscope: &ledgerv1alpha1.PyroscopeConfig{Enabled: true, ServerAddress: "http://pyroscope"}, + FlightRecorder: &ledgerv1alpha1.FlightRecorderConfig{Enabled: true, MinAge: "30s"}, + Traces: &ledgerv1alpha1.TracesConfig{ + Sampling: &ledgerv1alpha1.TraceSamplingConfig{Enabled: true, SuccessRatio: "0.1"}, + }, + Metrics: &ledgerv1alpha1.MetricsConfig{KeepInMemory: pointerTo(true)}, + Logs: &ledgerv1alpha1.LogsConfig{Level: "warn"}, + }, + Auth: &ledgerv1alpha1.AuthorizationConfig{ + ScopeMapping: map[string][]string{"ledger:read": {"ledger:accounts:read"}}, + AnonymousScopes: []string{"*:read"}, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("256Mi")}, + }, + ExtraEnv: []corev1.EnvVar{ + {Name: "BASE_ONLY", Value: "kept"}, + {Name: "SHARED", Value: "base"}, + }, + PodAnnotations: map[string]string{"base": "kept"}, + AdditionalLabels: map[string]string{"base": "kept"}, + NodeSelector: map[string]string{"disk": "nvme"}, + ServiceAccount: ledgerv1alpha1.ServiceAccountSpec{ + Annotations: map[string]string{"eks.amazonaws.com/role-arn": "role"}, + }, + } + + actual := composeLedgerV3ClusterSpec(base, ledgerV3SpecOverrides{ + ImageRepository: "registry.example/ledger", + ImageTag: "v3.0.0", + ImagePullSecrets: []corev1.LocalObjectReference{{Name: "registry"}}, + Replicas: 5, + ClusterID: "stack0", + Debug: true, + TLSSecretName: "stack0-ledger-v3-tls", + TLSCAHash: "ca-hash", + Preview: true, + Resources: &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("2")}, + }, + ExtraEnv: []corev1.EnvVar{ + {Name: "SHARED", Value: "operator"}, + {Name: "OPERATOR_ONLY", Value: "added"}, + }, + Monitoring: &settings.OpenTelemetryConfiguration{ + ServiceName: "ledger", + Attributes: map[string]string{"stack": "stack0", "pod-name": "$(POD_NAME)"}, + Traces: &settings.OpenTelemetrySignalConfiguration{Endpoint: "otel", Port: "4318", Insecure: true, Mode: "http"}, + Metrics: &settings.OpenTelemetrySignalConfiguration{Endpoint: "otel", Port: "4318", Insecure: true, Mode: "http"}, + Logs: &settings.OpenTelemetrySignalConfiguration{Endpoint: "logs", Port: "4317", Mode: "grpc"}, + }, + Auth: &auths.ProtectedAuthConfiguration{ + Issuer: "https://auth.example", + Issuers: []string{"https://issuer.example"}, + ReadKeySetMaxRetries: 7, + CheckScopes: true, + Service: "ledger", + }, + ServiceAccountName: "ledger-aws", + }) + + require.Equal(t, "registry.example/ledger", actual.Image.Repository) + require.Equal(t, "v3.0.0", actual.Image.Tag) + require.Equal(t, corev1.PullAlways, actual.Image.PullPolicy) + require.Equal(t, int32(5), *actual.Replicas) + require.Equal(t, "stack0", actual.ClusterID) + require.True(t, actual.Debug) + require.Equal(t, "stack0-ledger-v3-tls", actual.TLS.SecretName) + require.Equal(t, "ca-hash", actual.PodAnnotations[ledgerV3TLSCAHashAnnotation]) + require.Equal(t, "kept", actual.PodAnnotations["base"]) + require.Equal(t, "kept", actual.AdditionalLabels["base"]) + require.Equal(t, "true", actual.AdditionalLabels[ledgerV3PreviewLabel]) + require.Equal(t, "nvme", actual.NodeSelector["disk"]) + require.Equal(t, "2", actual.Resources.Limits.Cpu().String()) + require.Empty(t, actual.Resources.Requests) + require.Equal(t, []corev1.EnvVar{ + {Name: "BASE_ONLY", Value: "kept"}, + {Name: "OPERATOR_ONLY", Value: "added"}, + {Name: "SHARED", Value: "operator"}, + }, actual.ExtraEnv) + + require.True(t, actual.Monitoring.Pyroscope.Enabled) + require.Equal(t, "30s", actual.Monitoring.FlightRecorder.MinAge) + require.Equal(t, "0.1", actual.Monitoring.Traces.Sampling.SuccessRatio) + require.True(t, *actual.Monitoring.Metrics.KeepInMemory) + require.Equal(t, "warn", actual.Monitoring.Logs.Level) + require.Equal(t, "ledger", actual.Monitoring.ServiceName) + require.Equal(t, "pod-name=$(POD_NAME),stack=stack0", actual.Monitoring.Attributes) + require.Equal(t, "otel", actual.Monitoring.Traces.Endpoint) + require.Equal(t, "true", actual.Monitoring.Traces.Insecure) + require.Equal(t, "true", actual.Monitoring.Traces.Batch) + require.True(t, *actual.Monitoring.Metrics.Runtime) + + require.Equal(t, []string{"*:read"}, actual.Auth.AnonymousScopes) + require.Equal(t, []string{"ledger:accounts:read"}, actual.Auth.ScopeMapping["ledger:read"]) + require.Equal(t, "https://auth.example", actual.Auth.Issuer) + require.True(t, *actual.Auth.Enabled) + require.True(t, *actual.Auth.CheckScopes) + require.Equal(t, int32(7), *actual.Auth.ReadKeySetMaxRetries) + require.Equal(t, "ledger-aws", actual.ServiceAccount.Name) + require.False(t, *actual.ServiceAccount.Create) + require.Equal(t, "role", actual.ServiceAccount.Annotations["eks.amazonaws.com/role-arn"]) + + // Composition must not mutate the shared configuration stored in Kubernetes. + require.Equal(t, "base", base.ExtraEnv[1].Value) + require.Empty(t, base.Image.Repository) +} + +func TestComposeLedgerV3ClusterSpecPreservesOptionalBaseValues(t *testing.T) { + t.Parallel() + + base := &ledgerv1alpha1.ClusterSpec{ + ImagePullSecrets: []corev1.LocalObjectReference{{Name: "base-registry"}}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("512Mi")}, + }, + Auth: &ledgerv1alpha1.AuthorizationConfig{Enabled: pointerTo(true), Issuer: "https://external.example"}, + } + actual := composeLedgerV3ClusterSpec(base, ledgerV3SpecOverrides{ + ImageRepository: "ledger", + ImageTag: "latest", + Replicas: 3, + ClusterID: "stack0", + TLSSecretName: "tls", + }) + + require.Equal(t, "base-registry", actual.ImagePullSecrets[0].Name) + require.Equal(t, "512Mi", actual.Resources.Requests.Memory().String()) + require.Equal(t, "https://external.example", actual.Auth.Issuer) +} diff --git a/internal/resources/ledgers/v3_tls.go b/internal/resources/ledgers/v3_tls.go index bdb294904..427eee336 100644 --- a/internal/resources/ledgers/v3_tls.go +++ b/internal/resources/ledgers/v3_tls.go @@ -189,11 +189,3 @@ func ledgerV3CertificateReady(certificate *unstructured.Unstructured) (bool, str } return false, fmt.Sprintf("Certificate %s/%s has no Ready condition", certificate.GetNamespace(), certificate.GetName()), nil } - -func ledgerV3TLSSpec(stackName string) map[string]any { - return map[string]any{ - "enabled": true, - "secretName": ledgerV3TLSName(stackName), - "caSecretKey": ledgerV3TLSCASecretKey, - } -} diff --git a/internal/tests/crds/ledger.formance.com_clusters.yaml b/internal/tests/crds/ledger.formance.com_clusters.yaml index ae8a95691..4c5ee908c 100644 --- a/internal/tests/crds/ledger.formance.com_clusters.yaml +++ b/internal/tests/crds/ledger.formance.com_clusters.yaml @@ -27,6 +27,13 @@ spec: spec: type: object x-kubernetes-preserve-unknown-fields: true + properties: + bindAddr: + type: string + default: 0.0.0.0:7777 + x-kubernetes-validations: + - rule: oldSelf == '' || self == oldSelf + message: bindAddr is immutable once set status: type: object x-kubernetes-preserve-unknown-fields: true diff --git a/internal/tests/ledger_v3_controller_test.go b/internal/tests/ledger_v3_controller_test.go index c60bd6aaa..81402045c 100644 --- a/internal/tests/ledger_v3_controller_test.go +++ b/internal/tests/ledger_v3_controller_test.go @@ -16,6 +16,8 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + ledgerv1alpha1 "github.com/formancehq/ledger/misc/operator/api/v1alpha1" + "github.com/formancehq/operator/v3/api/formance.com/v1beta1" "github.com/formancehq/operator/v3/internal/core" "github.com/formancehq/operator/v3/internal/resources/settings" @@ -885,3 +887,206 @@ var _ = Describe("Ledger v3 controller", func() { }) }) }) + +var _ = Describe("LedgerConfiguration", Serial, func() { + It("inherits the Ledger Cluster schema validation", func() { + invalidEnum := &v1beta1.LedgerConfiguration{ + ObjectMeta: metav1.ObjectMeta{Name: "invalid-enum"}, + Spec: v1beta1.LedgerConfigurationSpec{ + Stacks: []string{"*"}, + Cluster: ledgerv1alpha1.ClusterSpec{LogLevel: "verbose"}, + }, + } + Expect(apierrors.IsInvalid(Create(invalidEnum))).To(BeTrue()) + + invalidCEL := &v1beta1.LedgerConfiguration{ + ObjectMeta: metav1.ObjectMeta{Name: "invalid-cel"}, + Spec: v1beta1.LedgerConfigurationSpec{ + Stacks: []string{"*"}, + Cluster: ledgerv1alpha1.ClusterSpec{ + Ingress: &ledgerv1alpha1.IngressSpec{Enabled: true}, + }, + }, + } + Expect(apierrors.IsInvalid(Create(invalidCEL))).To(BeTrue()) + }) + + It("uses the default configuration as a live base for Ledger v3 Clusters", func() { + configuration := &v1beta1.LedgerConfiguration{ + ObjectMeta: metav1.ObjectMeta{Name: v1beta1.DefaultLedgerConfigurationName}, + Spec: v1beta1.LedgerConfigurationSpec{ + Stacks: []string{"*"}, + Cluster: ledgerv1alpha1.ClusterSpec{ + LogLevel: "info", + HashAlgorithm: "blake3", + NodeSelector: map[string]string{"disk": "nvme"}, + PodAnnotations: map[string]string{ + "configuration": "preserved", + }, + Monitoring: &ledgerv1alpha1.MonitoringConfig{ + Pyroscope: &ledgerv1alpha1.PyroscopeConfig{ + Enabled: true, + ServerAddress: "http://pyroscope.monitoring.svc:4040", + }, + }, + }, + }, + } + Expect(Create(configuration)).To(Succeed()) + DeferCleanup(func() { + Expect(client.IgnoreNotFound(Delete(configuration))).To(Succeed()) + }) + + stack := &v1beta1.Stack{ + ObjectMeta: RandObjectMeta(), + Spec: v1beta1.StackSpec{Version: "v3.0.0-alpha.1"}, + } + ledger := &v1beta1.Ledger{ + ObjectMeta: RandObjectMeta(), + Spec: v1beta1.LedgerSpec{ + StackDependency: v1beta1.StackDependency{Stack: stack.Name}, + }, + } + Expect(Create(stack)).To(Succeed()) + Expect(Create(ledger)).To(Succeed()) + DeferCleanup(func() { + Expect(client.IgnoreNotFound(Delete(ledger))).To(Succeed()) + Expect(client.IgnoreNotFound(Delete(stack))).To(Succeed()) + }) + + previewStack := &v1beta1.Stack{ + ObjectMeta: RandObjectMeta(), + Spec: v1beta1.StackSpec{Version: "v2.99.0"}, + } + previewLedger := &v1beta1.Ledger{ + ObjectMeta: RandObjectMeta(), + Spec: v1beta1.LedgerSpec{ + StackDependency: v1beta1.StackDependency{Stack: previewStack.Name}, + }, + } + previewVersion := settings.New(uuid.NewString(), "ledger.v3.preview-version", "v3.0.0-alpha.11", previewStack.Name) + previewDatabase := settings.New(uuid.NewString(), "postgres.*.uri", "postgresql://localhost", previewStack.Name) + Expect(Create(previewStack)).To(Succeed()) + Expect(Create(previewVersion)).To(Succeed()) + Expect(Create(previewDatabase)).To(Succeed()) + Expect(Create(previewLedger)).To(Succeed()) + DeferCleanup(func() { + Expect(client.IgnoreNotFound(Delete(previewLedger))).To(Succeed()) + Expect(client.IgnoreNotFound(Delete(previewVersion))).To(Succeed()) + Expect(client.IgnoreNotFound(Delete(previewDatabase))).To(Succeed()) + Expect(client.IgnoreNotFound(Delete(previewStack))).To(Succeed()) + }) + + cluster := newLedgerV3Cluster() + Eventually(func(g Gomega) map[string]any { + g.Expect(Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name}, cluster)).To(Succeed()) + spec, found, err := unstructured.NestedMap(cluster.Object, "spec") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(found).To(BeTrue()) + return spec + }).Should(SatisfyAll( + HaveKeyWithValue("logLevel", "info"), + HaveKeyWithValue("hashAlgorithm", "blake3"), + HaveKeyWithValue("nodeSelector", map[string]any{"disk": "nvme"}), + HaveKeyWithValue("monitoring", SatisfyAll( + HaveKeyWithValue("serviceName", "ledger"), + HaveKeyWithValue("pyroscope", SatisfyAll( + HaveKeyWithValue("enabled", true), + HaveKeyWithValue("serverAddress", "http://pyroscope.monitoring.svc:4040"), + )), + )), + )) + previewCluster := newLedgerV3Cluster() + Eventually(func(g Gomega) map[string]any { + g.Expect(Get(types.NamespacedName{Namespace: previewStack.Name, Name: previewStack.Name}, previewCluster)).To(Succeed()) + nodeSelector, found, err := unstructured.NestedMap(previewCluster.Object, "spec", "nodeSelector") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(found).To(BeTrue()) + return nodeSelector + }).Should(Equal(map[string]any{"disk": "nvme"})) + + configuration.Spec.Cluster.HashAlgorithm = "xxh3" + configuration.Spec.Cluster.Monitoring.Pyroscope.ServerAddress = "http://pyroscope-v2.monitoring.svc:4040" + Expect(Update(configuration)).To(Succeed()) + Eventually(func(g Gomega) string { + g.Expect(Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name}, cluster)).To(Succeed()) + hash, found, err := unstructured.NestedString(cluster.Object, "spec", "hashAlgorithm") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(found).To(BeTrue()) + return hash + }).Should(Equal("xxh3")) + Eventually(func(g Gomega) string { + g.Expect(Get(types.NamespacedName{Namespace: previewStack.Name, Name: previewStack.Name}, previewCluster)).To(Succeed()) + hash, found, err := unstructured.NestedString(previewCluster.Object, "spec", "hashAlgorithm") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(found).To(BeTrue()) + return hash + }).Should(Equal("xxh3")) + + specificConfiguration := &v1beta1.LedgerConfiguration{ + ObjectMeta: metav1.ObjectMeta{Name: "specific-" + stack.Name}, + Spec: v1beta1.LedgerConfigurationSpec{ + Stacks: []string{stack.Name}, + Cluster: ledgerv1alpha1.ClusterSpec{ + HashAlgorithm: "blake3", + LogLevel: "debug", + }, + }, + } + Expect(Create(specificConfiguration)).To(Succeed()) + Eventually(func(g Gomega) string { + g.Expect(Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name}, cluster)).To(Succeed()) + hash, found, err := unstructured.NestedString(cluster.Object, "spec", "hashAlgorithm") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(found).To(BeTrue()) + return hash + }).Should(Equal("blake3")) + Consistently(func(g Gomega) string { + g.Expect(Get(types.NamespacedName{Namespace: previewStack.Name, Name: previewStack.Name}, previewCluster)).To(Succeed()) + hash, found, err := unstructured.NestedString(previewCluster.Object, "spec", "hashAlgorithm") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(found).To(BeTrue()) + return hash + }).Should(Equal("xxh3")) + Expect(Delete(specificConfiguration)).To(Succeed()) + Eventually(func(g Gomega) string { + g.Expect(Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name}, cluster)).To(Succeed()) + hash, found, err := unstructured.NestedString(cluster.Object, "spec", "hashAlgorithm") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(found).To(BeTrue()) + return hash + }).Should(Equal("xxh3")) + + Expect(Delete(configuration)).To(Succeed()) + Eventually(func(g Gomega) bool { + g.Expect(Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name}, cluster)).To(Succeed()) + _, found, err := unstructured.NestedString(cluster.Object, "spec", "hashAlgorithm") + g.Expect(err).NotTo(HaveOccurred()) + return found + }).Should(BeFalse()) + Eventually(func(g Gomega) bool { + g.Expect(Get(types.NamespacedName{Namespace: previewStack.Name, Name: previewStack.Name}, previewCluster)).To(Succeed()) + _, found, err := unstructured.NestedString(previewCluster.Object, "spec", "hashAlgorithm") + g.Expect(err).NotTo(HaveOccurred()) + return found + }).Should(BeFalse()) + + configuration = &v1beta1.LedgerConfiguration{ + ObjectMeta: metav1.ObjectMeta{Name: v1beta1.DefaultLedgerConfigurationName}, + Spec: v1beta1.LedgerConfigurationSpec{ + Stacks: []string{"*"}, + Cluster: ledgerv1alpha1.ClusterSpec{ + BindAddr: "127.0.0.1:7777", + }, + }, + } + Expect(Create(configuration)).To(Succeed()) + Eventually(func(g Gomega) string { + g.Expect(LoadResource("", ledger.Name, ledger)).To(Succeed()) + condition := ledger.GetConditions().Get("LedgerV3ClusterReady") + g.Expect(condition).NotTo(BeNil()) + g.Expect(condition.Reason).To(Equal("ReconcileFailed")) + return condition.Message + }).Should(ContainSubstring("bindAddr")) + }) +}) diff --git a/tools/kubectl-stacks/go.mod b/tools/kubectl-stacks/go.mod index 9043846db..dc237148c 100644 --- a/tools/kubectl-stacks/go.mod +++ b/tools/kubectl-stacks/go.mod @@ -1,13 +1,11 @@ module github.com/formancehq/operator/tools/kubectl-stacks/v3 -go 1.25.0 - -toolchain go1.25.5 +go 1.26.0 require ( github.com/formancehq/go-libs/v5 v5.2.0 github.com/formancehq/operator/v3 v3.0.0-00010101000000-000000000000 - github.com/pterm/pterm v0.12.81 + github.com/pterm/pterm v0.12.82 github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 k8s.io/apimachinery v0.34.2 @@ -27,6 +25,7 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.13.0 // indirect github.com/evanphx/json-patch/v5 v5.9.11 // indirect + github.com/formancehq/ledger/misc/operator v0.0.0-20260715094310-76862ea0b681 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/go-errors/errors v1.4.2 // indirect github.com/go-logr/logr v1.4.3 // indirect diff --git a/tools/kubectl-stacks/go.sum b/tools/kubectl-stacks/go.sum index b789481aa..d903d82a8 100644 --- a/tools/kubectl-stacks/go.sum +++ b/tools/kubectl-stacks/go.sum @@ -42,6 +42,8 @@ github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjT github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= github.com/formancehq/go-libs/v5 v5.2.0 h1:TpS47F8X5g5cHhnecfD20TrcdBqUVGy/ezZv0oFaQjc= github.com/formancehq/go-libs/v5 v5.2.0/go.mod h1:ms6tCGw1yqB4qtEbAuqPOQegWo4rU48vDobNkK7Ak6U= +github.com/formancehq/ledger/misc/operator v0.0.0-20260715094310-76862ea0b681 h1:PzhfbpKZqMJd4opnKmEtFq0Hx/6Pxgq9GT7XzWcr+Ss= +github.com/formancehq/ledger/misc/operator v0.0.0-20260715094310-76862ea0b681/go.mod h1:tZa1TFBcXJxc1R0NqnsN6gOGoskd82a7wjmGqCBhOBU= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= @@ -144,8 +146,8 @@ github.com/pterm/pterm v0.12.31/go.mod h1:32ZAWZVXD7ZfG0s8qqHXePte42kdz8ECtRyEej github.com/pterm/pterm v0.12.33/go.mod h1:x+h2uL+n7CP/rel9+bImHD5lF3nM9vJj80k9ybiiTTE= github.com/pterm/pterm v0.12.36/go.mod h1:NjiL09hFhT/vWjQHSj1athJpx6H8cjpHXNAK5bUw8T8= github.com/pterm/pterm v0.12.40/go.mod h1:ffwPLwlbXxP+rxT0GsgDTzS3y3rmpAO1NMjUkGTYf8s= -github.com/pterm/pterm v0.12.81 h1:ju+j5I2++FO1jBKMmscgh5h5DPFDFMB7epEjSoKehKA= -github.com/pterm/pterm v0.12.81/go.mod h1:TyuyrPjnxfwP+ccJdBTeWHtd/e0ybQHkOS/TakajZCw= +github.com/pterm/pterm v0.12.82 h1:+D9wYhCaeaK0FIQoZtqbNQuNpe2lB2tajKKsTd5paVQ= +github.com/pterm/pterm v0.12.82/go.mod h1:TyuyrPjnxfwP+ccJdBTeWHtd/e0ybQHkOS/TakajZCw= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= From b5bf8385cca24150deca4f6853767167d28932c6 Mon Sep 17 00:00:00 2001 From: Maxence Maireaux Date: Wed, 15 Jul 2026 13:03:20 +0200 Subject: [PATCH 16/30] feat(ledger): map topology spread setting to v3 --- .../settings.catalog.json | 7 +++ internal/resources/ledgers/v3.go | 34 ++++++++----- internal/resources/ledgers/v3_spec.go | 51 ++++++++++++++----- internal/resources/ledgers/v3_spec_test.go | 34 ++++++++++++- 4 files changed, 97 insertions(+), 29 deletions(-) diff --git a/docs/09-Configuration reference/settings.catalog.json b/docs/09-Configuration reference/settings.catalog.json index d8be02d3a..6ebce3069 100644 --- a/docs/09-Configuration reference/settings.catalog.json +++ b/docs/09-Configuration reference/settings.catalog.json @@ -226,6 +226,13 @@ "internal/resources/ledgers/v3.go:276" ] }, + { + "key": "deployments.ledger.topology-spread-constraints", + "valueType": "bool", + "sources": [ + "internal/resources/ledgers/v3.go:308" + ] + }, { "key": "gateway.caddyfile.grace-period", "valueType": "string", diff --git a/internal/resources/ledgers/v3.go b/internal/resources/ledgers/v3.go index d738ee58d..8c7e3a536 100644 --- a/internal/resources/ledgers/v3.go +++ b/internal/resources/ledgers/v3.go @@ -305,21 +305,27 @@ func createOrUpdateV3Cluster(ctx core.Context, stack *v1beta1.Stack, ledger *v1b if err != nil { return nil, err } + topologySpreadConstraints, err := settings.GetBool(ctx, stack.Name, + "deployments", "ledger", "topology-spread-constraints") + if err != nil { + return nil, err + } desiredSpec := composeLedgerV3ClusterSpec(baseSpec, ledgerV3SpecOverrides{ - ImageRepository: imageRepository(image), - ImageTag: image.Version, - ImagePullSecrets: image.PullSecrets, - Replicas: replicas, - ClusterID: stack.Name, - Debug: stack.Spec.Debug || ledger.Spec.Debug, - TLSSecretName: ledgerV3TLSName(stack.Name), - TLSCAHash: tlsCAHash, - Preview: preview, - Resources: resourceRequirements, - ExtraEnv: core.GetDevEnvVars(stack, ledger), - Monitoring: monitoringConfiguration, - Auth: authConfiguration, - ServiceAccountName: serviceAccountName, + ImageRepository: imageRepository(image), + ImageTag: image.Version, + ImagePullSecrets: image.PullSecrets, + Replicas: replicas, + ClusterID: stack.Name, + Debug: stack.Spec.Debug || ledger.Spec.Debug, + TLSSecretName: ledgerV3TLSName(stack.Name), + TLSCAHash: tlsCAHash, + Preview: preview, + Resources: resourceRequirements, + ExtraEnv: core.GetDevEnvVars(stack, ledger), + Monitoring: monitoringConfiguration, + Auth: authConfiguration, + ServiceAccountName: serviceAccountName, + TopologySpreadConstraints: topologySpreadConstraints, }) desiredSpecMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(desiredSpec) if err != nil { diff --git a/internal/resources/ledgers/v3_spec.go b/internal/resources/ledgers/v3_spec.go index 514f70946..1b772be1f 100644 --- a/internal/resources/ledgers/v3_spec.go +++ b/internal/resources/ledgers/v3_spec.go @@ -15,20 +15,21 @@ import ( ) type ledgerV3SpecOverrides struct { - ImageRepository string - ImageTag string - ImagePullSecrets []corev1.LocalObjectReference - Replicas int32 - ClusterID string - Debug bool - TLSSecretName string - TLSCAHash string - Preview bool - Resources *corev1.ResourceRequirements - ExtraEnv []corev1.EnvVar - Monitoring *settings.OpenTelemetryConfiguration - Auth *auths.ProtectedAuthConfiguration - ServiceAccountName string + ImageRepository string + ImageTag string + ImagePullSecrets []corev1.LocalObjectReference + Replicas int32 + ClusterID string + Debug bool + TLSSecretName string + TLSCAHash string + Preview bool + Resources *corev1.ResourceRequirements + ExtraEnv []corev1.EnvVar + Monitoring *settings.OpenTelemetryConfiguration + Auth *auths.ProtectedAuthConfiguration + ServiceAccountName string + TopologySpreadConstraints *bool } // composeLedgerV3ClusterSpec applies the values owned by the Formance Operator @@ -86,10 +87,32 @@ func composeLedgerV3ClusterSpec(base *ledgerv1alpha1.ClusterSpec, overrides ledg spec.ServiceAccount.Create = pointerTo(false) spec.ServiceAccount.Name = overrides.ServiceAccountName } + if overrides.TopologySpreadConstraints != nil { + if *overrides.TopologySpreadConstraints { + spec.TopologySpreadConstraints = defaultLedgerV3TopologySpreadConstraints() + } else { + spec.TopologySpreadConstraints = nil + } + } return spec } +func defaultLedgerV3TopologySpreadConstraints() []corev1.TopologySpreadConstraint { + return []corev1.TopologySpreadConstraint{ + { + MaxSkew: 1, + TopologyKey: corev1.LabelTopologyZone, + WhenUnsatisfiable: corev1.ScheduleAnyway, + }, + { + MaxSkew: 1, + TopologyKey: corev1.LabelHostname, + WhenUnsatisfiable: corev1.DoNotSchedule, + }, + } +} + func applyLedgerV3Auth(spec *ledgerv1alpha1.ClusterSpec, configuration *auths.ProtectedAuthConfiguration) { if configuration == nil { return diff --git a/internal/resources/ledgers/v3_spec_test.go b/internal/resources/ledgers/v3_spec_test.go index eda98bad9..a24cad4a3 100644 --- a/internal/resources/ledgers/v3_spec_test.go +++ b/internal/resources/ledgers/v3_spec_test.go @@ -41,6 +41,11 @@ func TestComposeLedgerV3ClusterSpec(t *testing.T) { PodAnnotations: map[string]string{"base": "kept"}, AdditionalLabels: map[string]string{"base": "kept"}, NodeSelector: map[string]string{"disk": "nvme"}, + TopologySpreadConstraints: []corev1.TopologySpreadConstraint{{ + MaxSkew: 2, + TopologyKey: "configuration.example/failure-domain", + WhenUnsatisfiable: corev1.ScheduleAnyway, + }}, ServiceAccount: ledgerv1alpha1.ServiceAccountSpec{ Annotations: map[string]string{"eks.amazonaws.com/role-arn": "role"}, }, @@ -77,7 +82,8 @@ func TestComposeLedgerV3ClusterSpec(t *testing.T) { CheckScopes: true, Service: "ledger", }, - ServiceAccountName: "ledger-aws", + ServiceAccountName: "ledger-aws", + TopologySpreadConstraints: pointerTo(true), }) require.Equal(t, "registry.example/ledger", actual.Image.Repository) @@ -92,6 +98,7 @@ func TestComposeLedgerV3ClusterSpec(t *testing.T) { require.Equal(t, "kept", actual.AdditionalLabels["base"]) require.Equal(t, "true", actual.AdditionalLabels[ledgerV3PreviewLabel]) require.Equal(t, "nvme", actual.NodeSelector["disk"]) + require.Equal(t, defaultLedgerV3TopologySpreadConstraints(), actual.TopologySpreadConstraints) require.Equal(t, "2", actual.Resources.Limits.Cpu().String()) require.Empty(t, actual.Resources.Requests) require.Equal(t, []corev1.EnvVar{ @@ -136,6 +143,11 @@ func TestComposeLedgerV3ClusterSpecPreservesOptionalBaseValues(t *testing.T) { Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("512Mi")}, }, Auth: &ledgerv1alpha1.AuthorizationConfig{Enabled: pointerTo(true), Issuer: "https://external.example"}, + TopologySpreadConstraints: []corev1.TopologySpreadConstraint{{ + MaxSkew: 2, + TopologyKey: "configuration.example/failure-domain", + WhenUnsatisfiable: corev1.ScheduleAnyway, + }}, } actual := composeLedgerV3ClusterSpec(base, ledgerV3SpecOverrides{ ImageRepository: "ledger", @@ -148,4 +160,24 @@ func TestComposeLedgerV3ClusterSpecPreservesOptionalBaseValues(t *testing.T) { require.Equal(t, "base-registry", actual.ImagePullSecrets[0].Name) require.Equal(t, "512Mi", actual.Resources.Requests.Memory().String()) require.Equal(t, "https://external.example", actual.Auth.Issuer) + require.Equal(t, base.TopologySpreadConstraints, actual.TopologySpreadConstraints) +} + +func TestComposeLedgerV3ClusterSpecDisablesConfiguredTopologySpreadConstraints(t *testing.T) { + t.Parallel() + + base := &ledgerv1alpha1.ClusterSpec{ + TopologySpreadConstraints: defaultLedgerV3TopologySpreadConstraints(), + } + actual := composeLedgerV3ClusterSpec(base, ledgerV3SpecOverrides{ + ImageRepository: "ledger", + ImageTag: "latest", + Replicas: 3, + ClusterID: "stack0", + TLSSecretName: "tls", + TopologySpreadConstraints: pointerTo(false), + }) + + require.Nil(t, actual.TopologySpreadConstraints) + require.NotEmpty(t, base.TopologySpreadConstraints) } From 9ca431608644bd14d9f912eb048ea72f5880a9d1 Mon Sep 17 00:00:00 2001 From: Maxence Maireaux Date: Wed, 15 Jul 2026 13:13:30 +0200 Subject: [PATCH 17/30] fix(ledger): make v3 capability optional --- .../settings.catalog.json | 12 +- internal/resources/ledgers/v3.go | 15 +- internal/resources/ledgers/v3_preview.go | 7 + internal/resources/ledgers/v3_test.go | 135 +++++++++++++++++- 4 files changed, 161 insertions(+), 8 deletions(-) diff --git a/docs/09-Configuration reference/settings.catalog.json b/docs/09-Configuration reference/settings.catalog.json index 6ebce3069..d32e29d8f 100644 --- a/docs/09-Configuration reference/settings.catalog.json +++ b/docs/09-Configuration reference/settings.catalog.json @@ -201,21 +201,21 @@ "key": "deployments.ledger.containers.ledger.resource-requirements.claims", "valueType": "string[]", "sources": [ - "internal/resources/ledgers/v3.go:290" + "internal/resources/ledgers/v3.go:303" ] }, { "key": "deployments.ledger.containers.ledger.resource-requirements.limits", "valueType": "map[string]string", "sources": [ - "internal/resources/ledgers/v3.go:290" + "internal/resources/ledgers/v3.go:303" ] }, { "key": "deployments.ledger.containers.ledger.resource-requirements.requests", "valueType": "map[string]string", "sources": [ - "internal/resources/ledgers/v3.go:290" + "internal/resources/ledgers/v3.go:303" ] }, { @@ -223,14 +223,14 @@ "valueType": "int32", "default": "3", "sources": [ - "internal/resources/ledgers/v3.go:276" + "internal/resources/ledgers/v3.go:289" ] }, { "key": "deployments.ledger.topology-spread-constraints", "valueType": "bool", "sources": [ - "internal/resources/ledgers/v3.go:308" + "internal/resources/ledgers/v3.go:321" ] }, { @@ -473,7 +473,7 @@ "key": "ledger.v3.preview-version", "valueType": "string", "sources": [ - "internal/resources/ledgers/v3_preview.go:36" + "internal/resources/ledgers/v3_preview.go:43" ] }, { diff --git a/internal/resources/ledgers/v3.go b/internal/resources/ledgers/v3.go index 8c7e3a536..7a4ce143a 100644 --- a/internal/resources/ledgers/v3.go +++ b/internal/resources/ledgers/v3.go @@ -74,7 +74,11 @@ func withLedgerV3ClusterWatch() core.ReconcilerOption[*v1beta1.Ledger] { options.Raws = append(options.Raws, func(ctx core.Context, b *builder.Builder) error { crds := &apiextensionsv1.CustomResourceDefinitionList{} if err := ctx.GetAPIReader().List(ctx, crds); err != nil { - return err + ledgerV3ClusterAvailable = false + ledgerV3CertManagerAvailable = false + log.FromContext(ctx).Info("Ledger v3 capability is unavailable; continuing without it", + "error", err) + return nil } ledgerV3ClusterAvailable = watchLedgerV3Resource(ctx, b, options, crds, ledgerV3ClusterGVK) @@ -126,6 +130,15 @@ func watchLedgerV3Resource( } for _, version := range crd.Spec.Versions { if version.Name == gvk.Version && version.Served { + resourceList := &unstructured.UnstructuredList{} + resourceList.SetGroupVersionKind(gvk.GroupVersion().WithKind(gvk.Kind + "List")) + if err := ctx.GetAPIReader().List(ctx, resourceList, client.Limit(1)); err != nil { + log.FromContext(ctx).Info("Ledger v3 dependency is inaccessible; continuing without it", + "gvk", gvk, + "error", err) + return false + } + resource := newLedgerV3Resource(gvk) options.Owns[resource] = nil b.Owns(resource) diff --git a/internal/resources/ledgers/v3_preview.go b/internal/resources/ledgers/v3_preview.go index 0a86d8a4f..94523f0aa 100644 --- a/internal/resources/ledgers/v3_preview.go +++ b/internal/resources/ledgers/v3_preview.go @@ -33,6 +33,13 @@ import ( ) func ledgerV3PreviewVersion(ctx core.Context, stack *v1beta1.Stack) (string, error) { + // Preview is an optional capability. Ignore its Setting entirely when the + // Ledger Operator CRD could not be discovered, so Ledger v2 reconciliation + // remains unchanged and does not expose routes to a missing backend. + if !ledgerV3ClusterAvailable { + return "", nil + } + version, err := settings.GetStringOrEmpty(ctx, stack.Name, "ledger", "v3", "preview-version") if err != nil { return "", err diff --git a/internal/resources/ledgers/v3_test.go b/internal/resources/ledgers/v3_test.go index f284f00be..7a87aedab 100644 --- a/internal/resources/ledgers/v3_test.go +++ b/internal/resources/ledgers/v3_test.go @@ -1,6 +1,139 @@ package ledgers -import "testing" +import ( + "context" + "errors" + "testing" + + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/formancehq/operator/v3/api/formance.com/v1beta1" + "github.com/formancehq/operator/v3/internal/core" +) + +type failingLedgerV3DiscoveryReader struct { + err error +} + +func (r failingLedgerV3DiscoveryReader) Get(context.Context, client.ObjectKey, client.Object, ...client.GetOption) error { + return r.err +} + +func (r failingLedgerV3DiscoveryReader) List(context.Context, client.ObjectList, ...client.ListOption) error { + return r.err +} + +type inaccessibleLedgerV3ResourceReader struct { + err error +} + +func (r inaccessibleLedgerV3ResourceReader) Get(context.Context, client.ObjectKey, client.Object, ...client.GetOption) error { + return r.err +} + +func (r inaccessibleLedgerV3ResourceReader) List(_ context.Context, object client.ObjectList, _ ...client.ListOption) error { + switch list := object.(type) { + case *apiextensionsv1.CustomResourceDefinitionList: + list.Items = []apiextensionsv1.CustomResourceDefinition{{ + ObjectMeta: metav1.ObjectMeta{Name: "clusters.ledger.formance.com"}, + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ + Group: "ledger.formance.com", + Names: apiextensionsv1.CustomResourceDefinitionNames{Kind: "Cluster", Plural: "clusters"}, + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{{ + Name: "v1alpha1", Served: true, Storage: true, + }}, + }, + }} + return nil + case *unstructured.UnstructuredList: + return r.err + default: + return nil + } +} + +type ledgerV3DiscoveryContext struct { + context.Context + reader client.Reader +} + +func (c ledgerV3DiscoveryContext) GetClient() client.Client { return nil } +func (c ledgerV3DiscoveryContext) GetScheme() *runtime.Scheme { return nil } +func (c ledgerV3DiscoveryContext) GetAPIReader() client.Reader { return c.reader } +func (c ledgerV3DiscoveryContext) GetPlatform() core.Platform { return core.Platform{} } + +func TestLedgerV3DiscoveryFailureDisablesCapabilityWithoutFailing(t *testing.T) { + previousClusterAvailable := ledgerV3ClusterAvailable + previousCertManagerAvailable := ledgerV3CertManagerAvailable + ledgerV3ClusterAvailable = true + ledgerV3CertManagerAvailable = true + t.Cleanup(func() { + ledgerV3ClusterAvailable = previousClusterAvailable + ledgerV3CertManagerAvailable = previousCertManagerAvailable + }) + + options := core.ReconcilerOptions[*v1beta1.Ledger]{} + withLedgerV3ClusterWatch()(&options) + if len(options.Raws) != 1 { + t.Fatalf("withLedgerV3ClusterWatch() registered %d raw builders, want 1", len(options.Raws)) + } + + discoveryError := errors.New("CRD discovery forbidden") + ctx := ledgerV3DiscoveryContext{ + Context: context.Background(), + reader: failingLedgerV3DiscoveryReader{err: discoveryError}, + } + if err := options.Raws[0](ctx, nil); err != nil { + t.Fatalf("Ledger v3 discovery failure must not fail controller setup: %v", err) + } + if ledgerV3ClusterAvailable { + t.Fatal("Ledger v3 Cluster capability remains enabled after discovery failure") + } + if ledgerV3CertManagerAvailable { + t.Fatal("Ledger v3 cert-manager capability remains enabled after discovery failure") + } +} + +func TestLedgerV3PreviewVersionIgnoredWhenClusterUnavailable(t *testing.T) { + previous := ledgerV3ClusterAvailable + ledgerV3ClusterAvailable = false + t.Cleanup(func() { + ledgerV3ClusterAvailable = previous + }) + + version, err := ledgerV3PreviewVersion(nil, nil) + if err != nil { + t.Fatalf("ledgerV3PreviewVersion() returned error: %v", err) + } + if version != "" { + t.Fatalf("ledgerV3PreviewVersion() = %q, want an empty version", version) + } +} + +func TestLedgerV3ResourceAccessFailureDisablesCapabilityWithoutFailing(t *testing.T) { + previous := ledgerV3ClusterAvailable + ledgerV3ClusterAvailable = true + t.Cleanup(func() { + ledgerV3ClusterAvailable = previous + }) + + options := core.ReconcilerOptions[*v1beta1.Ledger]{} + withLedgerV3ClusterWatch()(&options) + ctx := ledgerV3DiscoveryContext{ + Context: context.Background(), + reader: inaccessibleLedgerV3ResourceReader{err: errors.New("Cluster list forbidden")}, + } + if err := options.Raws[0](ctx, nil); err != nil { + t.Fatalf("Ledger v3 resource access failure must not fail controller setup: %v", err) + } + if ledgerV3ClusterAvailable { + t.Fatal("Ledger v3 Cluster capability remains enabled when Cluster objects are inaccessible") + } +} func TestIsLedgerV3(t *testing.T) { t.Parallel() From 9e85a4fdfc0a2993437eafbce65e4313d73ef260 Mon Sep 17 00:00:00 2001 From: Maxence Maireaux Date: Wed, 15 Jul 2026 15:05:22 +0200 Subject: [PATCH 18/30] fix(operator): harden optional ledger v3 integration --- config/rbac/role.yaml | 6 ++ .../settings.catalog.json | 10 +-- ..._v1_clusterrole_formance-manager-role.yaml | 6 ++ internal/resources/gateways/Caddyfile.gotpl | 4 +- internal/resources/gateways/caddyfile.go | 36 ++++++--- internal/resources/gateways/caddyfile_test.go | 41 +++++++++++ internal/resources/ledgers/v3.go | 42 ++++++++++- internal/resources/ledgers/v3_preview.go | 6 ++ internal/resources/ledgers/v3_spec.go | 15 +++- internal/resources/ledgers/v3_spec_test.go | 27 ++++++- internal/resources/ledgers/v3_test.go | 73 ++++++++++++++++++- internal/tests/ledger_v3_controller_test.go | 19 +++-- .../configmap-with-audit.yaml | 2 +- ...igmap-with-ledger-and-another-service.yaml | 4 +- .../configmap-with-ledger-and-grpc.yaml | 2 +- .../configmap-with-ledger-only.yaml | 2 +- .../configmap-with-opentelemetry.yaml | 2 +- 17 files changed, 263 insertions(+), 34 deletions(-) create mode 100644 internal/resources/gateways/caddyfile_test.go diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 83d3b952a..e945ddd02 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -42,6 +42,12 @@ rules: - patch - update - watch +- apiGroups: + - authorization.k8s.io + resources: + - selfsubjectaccessreviews + verbs: + - create - apiGroups: - batch resources: diff --git a/docs/09-Configuration reference/settings.catalog.json b/docs/09-Configuration reference/settings.catalog.json index d32e29d8f..b98067c44 100644 --- a/docs/09-Configuration reference/settings.catalog.json +++ b/docs/09-Configuration reference/settings.catalog.json @@ -201,21 +201,21 @@ "key": "deployments.ledger.containers.ledger.resource-requirements.claims", "valueType": "string[]", "sources": [ - "internal/resources/ledgers/v3.go:303" + "internal/resources/ledgers/v3.go:343" ] }, { "key": "deployments.ledger.containers.ledger.resource-requirements.limits", "valueType": "map[string]string", "sources": [ - "internal/resources/ledgers/v3.go:303" + "internal/resources/ledgers/v3.go:343" ] }, { "key": "deployments.ledger.containers.ledger.resource-requirements.requests", "valueType": "map[string]string", "sources": [ - "internal/resources/ledgers/v3.go:303" + "internal/resources/ledgers/v3.go:343" ] }, { @@ -223,14 +223,14 @@ "valueType": "int32", "default": "3", "sources": [ - "internal/resources/ledgers/v3.go:289" + "internal/resources/ledgers/v3.go:329" ] }, { "key": "deployments.ledger.topology-spread-constraints", "valueType": "bool", "sources": [ - "internal/resources/ledgers/v3.go:321" + "internal/resources/ledgers/v3.go:361" ] }, { diff --git a/helm/operator/templates/gen/rbac.authorization.k8s.io_v1_clusterrole_formance-manager-role.yaml b/helm/operator/templates/gen/rbac.authorization.k8s.io_v1_clusterrole_formance-manager-role.yaml index 2057fa6a6..6473d5907 100644 --- a/helm/operator/templates/gen/rbac.authorization.k8s.io_v1_clusterrole_formance-manager-role.yaml +++ b/helm/operator/templates/gen/rbac.authorization.k8s.io_v1_clusterrole_formance-manager-role.yaml @@ -41,6 +41,12 @@ rules: - patch - update - watch +- apiGroups: + - authorization.k8s.io + resources: + - selfsubjectaccessreviews + verbs: + - create - apiGroups: - batch resources: diff --git a/internal/resources/gateways/Caddyfile.gotpl b/internal/resources/gateways/Caddyfile.gotpl index 769fd75fa..636ca9643 100644 --- a/internal/resources/gateways/Caddyfile.gotpl +++ b/internal/resources/gateways/Caddyfile.gotpl @@ -149,10 +149,10 @@ {{- if or (not (semver_is_valid $values.Gateway.Version)) (gt (semver_compare $values.Gateway.Version "v0.1.7") 0) }} {{ $service.Name }} { - http://{{ $service.Name }}:8080/_info http://{{ $service.Name }}:8080/{{ $service.HealthCheckEndpoint }} + http://{{ $service.HealthCheckBackend.Name }}:{{ $service.HealthCheckBackend.Port }}/_info http://{{ $service.HealthCheckBackend.Name }}:{{ $service.HealthCheckBackend.Port }}/{{ $healthCheckEndpoint }} } {{- else }} - {{ $service.Name }} http://{{ $service.Name }}:8080/_info http://{{ $service.Name }}:8080/{{ $service.HealthCheckEndpoint }} + {{ $service.Name }} http://{{ $service.HealthCheckBackend.Name }}:{{ $service.HealthCheckBackend.Port }}/_info http://{{ $service.HealthCheckBackend.Name }}:{{ $service.HealthCheckBackend.Port }}/{{ $healthCheckEndpoint }} {{- end }} {{- end }} } diff --git a/internal/resources/gateways/caddyfile.go b/internal/resources/gateways/caddyfile.go index 5758f8814..25aa353f5 100644 --- a/internal/resources/gateways/caddyfile.go +++ b/internal/resources/gateways/caddyfile.go @@ -14,20 +14,16 @@ import ( type CaddyOptions func(data map[string]any) error +type caddyHTTPService struct { + v1beta1.GatewayHTTPAPISpec + HealthCheckBackend v1beta1.GatewayBackendRef +} + func CreateCaddyfile(ctx core.Context, stack *v1beta1.Stack, gateway *v1beta1.Gateway, httpAPIs []*v1beta1.GatewayHTTPAPI, grpcAPIs []*v1beta1.GatewayGRPCAPI, broker *v1beta1.Broker, options ...CaddyOptions) (string, error) { - services := collectionutils.Map(httpAPIs, func(from *v1beta1.GatewayHTTPAPI) v1beta1.GatewayHTTPAPISpec { - spec := *from.Spec.DeepCopy() - for i := range spec.Rules { - normalizeBackendTLS(spec.Rules[i].BackendRef) - } - sort.SliceStable(spec.Rules, func(i, j int) bool { - return len(spec.Rules[i].Path) > len(spec.Rules[j].Path) - }) - return spec - }) + services := caddyHTTPServices(httpAPIs) data := map[string]any{ "Services": services, @@ -58,6 +54,26 @@ func CreateCaddyfile(ctx core.Context, stack *v1beta1.Stack, return caddy.ComputeCaddyfile(ctx, stack, Caddyfile, data) } +func caddyHTTPServices(httpAPIs []*v1beta1.GatewayHTTPAPI) []caddyHTTPService { + return collectionutils.Map(httpAPIs, func(from *v1beta1.GatewayHTTPAPI) caddyHTTPService { + spec := *from.Spec.DeepCopy() + healthCheckBackend := v1beta1.GatewayBackendRef{Name: spec.Name, Port: 8080} + for i := range spec.Rules { + normalizeBackendTLS(spec.Rules[i].BackendRef) + if spec.Rules[i].Path == "" && spec.Rules[i].BackendRef != nil { + healthCheckBackend = *spec.Rules[i].BackendRef.DeepCopy() + } + } + sort.SliceStable(spec.Rules, func(i, j int) bool { + return len(spec.Rules[i].Path) > len(spec.Rules[j].Path) + }) + return caddyHTTPService{ + GatewayHTTPAPISpec: spec, + HealthCheckBackend: healthCheckBackend, + } + }) +} + func normalizeBackendTLS(backendRef *v1beta1.GatewayBackendRef) { if backendRef != nil && backendRef.TLS != nil && backendRef.TLS.CASecretKey == "" { backendRef.TLS.CASecretKey = "ca.crt" diff --git a/internal/resources/gateways/caddyfile_test.go b/internal/resources/gateways/caddyfile_test.go new file mode 100644 index 000000000..261917f18 --- /dev/null +++ b/internal/resources/gateways/caddyfile_test.go @@ -0,0 +1,41 @@ +package gateways + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/formancehq/operator/v3/api/formance.com/v1beta1" +) + +func TestCaddyHTTPServicesUsesRootRuleBackendForHealthChecks(t *testing.T) { + t.Parallel() + + services := caddyHTTPServices([]*v1beta1.GatewayHTTPAPI{{ + Spec: v1beta1.GatewayHTTPAPISpec{ + Name: "ledger", + Rules: []v1beta1.GatewayHTTPAPIRule{ + {Path: "/v3", BackendRef: &v1beta1.GatewayBackendRef{Name: "ledger-preview", Port: 9000}}, + {BackendRef: &v1beta1.GatewayBackendRef{Name: "ledger-stack0", Port: 9000}}, + }, + }, + }}) + + require.Len(t, services, 1) + require.Equal(t, v1beta1.GatewayBackendRef{Name: "ledger-stack0", Port: 9000}, services[0].HealthCheckBackend) + require.Equal(t, "/v3", services[0].Rules[0].Path, "route specificity must still be preserved") +} + +func TestCaddyHTTPServicesKeepsHistoricalHealthCheckBackend(t *testing.T) { + t.Parallel() + + services := caddyHTTPServices([]*v1beta1.GatewayHTTPAPI{{ + Spec: v1beta1.GatewayHTTPAPISpec{ + Name: "ledger", + Rules: []v1beta1.GatewayHTTPAPIRule{{}}, + }, + }}) + + require.Len(t, services, 1) + require.Equal(t, v1beta1.GatewayBackendRef{Name: "ledger", Port: 8080}, services[0].HealthCheckBackend) +} diff --git a/internal/resources/ledgers/v3.go b/internal/resources/ledgers/v3.go index 7a4ce143a..2ff19851e 100644 --- a/internal/resources/ledgers/v3.go +++ b/internal/resources/ledgers/v3.go @@ -7,6 +7,7 @@ import ( "golang.org/x/mod/semver" appsv1 "k8s.io/api/apps/v1" + authorizationv1 "k8s.io/api/authorization/v1" corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -52,8 +53,11 @@ var ( ledgerV3CertManagerAvailable bool ) +var ledgerV3RequiredVerbs = []string{"get", "list", "watch", "create", "update", "patch", "delete"} + //+kubebuilder:rbac:groups=ledger.formance.com,resources=clusters,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=formance.com,resources=ledgerconfigurations,verbs=get;list;watch +//+kubebuilder:rbac:groups=authorization.k8s.io,resources=selfsubjectaccessreviews,verbs=create func isLedgerV3(version string) bool { normalizedVersion := version @@ -138,6 +142,9 @@ func watchLedgerV3Resource( "error", err) return false } + if !canAccessLedgerV3Resource(ctx, gvk, crd.Spec.Names.Plural) { + return false + } resource := newLedgerV3Resource(gvk) options.Owns[resource] = nil @@ -152,6 +159,36 @@ func watchLedgerV3Resource( return false } +func canAccessLedgerV3Resource(ctx core.Context, gvk schema.GroupVersionKind, resource string) bool { + for _, verb := range ledgerV3RequiredVerbs { + review := &authorizationv1.SelfSubjectAccessReview{ + Spec: authorizationv1.SelfSubjectAccessReviewSpec{ + ResourceAttributes: &authorizationv1.ResourceAttributes{ + Group: gvk.Group, + Version: gvk.Version, + Resource: resource, + Verb: verb, + }, + }, + } + if err := ctx.GetClient().Create(ctx, review); err != nil { + log.FromContext(ctx).Info("Ledger v3 dependency access review failed; continuing without it", + "gvk", gvk, + "verb", verb, + "error", err) + return false + } + if !review.Status.Allowed { + log.FromContext(ctx).Info("Ledger v3 dependency permission is unavailable; continuing without it", + "gvk", gvk, + "verb", verb, + "reason", review.Status.Reason) + return false + } + } + return true +} + func reconcileV3(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, version string) error { if !ledgerV3ClusterAvailable { setLedgerV3Condition(ledger, metav1.ConditionFalse, "OperatorUnavailable", "Ledger v3 Cluster CRD is not installed") @@ -184,7 +221,10 @@ func reconcileV3(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, setLedgerV3Condition(ledger, metav1.ConditionFalse, "TLSCertificatePending", tlsMessage) return core.NewPendingError().WithMessage("Ledger v3 TLS is not ready: %s", tlsMessage) } - if err := gatewayhttpapis.Create(ctx, ledger, gatewayhttpapis.WithHealthCheckEndpoint("livez")); err != nil { + if err := gatewayhttpapis.Create(ctx, ledger, + gatewayhttpapis.WithHealthCheckEndpoint("livez"), + gatewayhttpapis.WithRules(gatewayhttpapis.RuleSecuredWithBackend("", ledgerV3HTTPBackendRef(stack.Name))), + ); err != nil { return err } if err := gatewaygrpcapis.Create(ctx, ledger, diff --git a/internal/resources/ledgers/v3_preview.go b/internal/resources/ledgers/v3_preview.go index 94523f0aa..d9da5d779 100644 --- a/internal/resources/ledgers/v3_preview.go +++ b/internal/resources/ledgers/v3_preview.go @@ -140,6 +140,12 @@ func deleteLedgerV3Preview(ctx core.Context, stack *v1beta1.Stack) error { return err } } + // A missing or inaccessible cert-manager dependency must not block the + // legacy Ledger reconciliation. The Cluster cleanup above is still safe and + // useful because its capability is checked independently. + if !ledgerV3CertManagerAvailable { + return nil + } certificate := newLedgerV3Resource(ledgerV3CertificateGVK) err = ctx.GetClient().Get(ctx, types.NamespacedName{Namespace: stack.Name, Name: ledgerV3TLSName(stack.Name)}, certificate) diff --git a/internal/resources/ledgers/v3_spec.go b/internal/resources/ledgers/v3_spec.go index 1b772be1f..bf9891ea2 100644 --- a/internal/resources/ledgers/v3_spec.go +++ b/internal/resources/ledgers/v3_spec.go @@ -72,7 +72,17 @@ func composeLedgerV3ClusterSpec(base *ledgerv1alpha1.ClusterSpec, overrides ledg } spec.AdditionalLabels["app.kubernetes.io/name"] = "ledger-v3-preview" spec.AdditionalLabels[ledgerV3PreviewLabel] = "true" + } else { + if spec.AdditionalLabels == nil { + spec.AdditionalLabels = map[string]string{} + } + spec.AdditionalLabels["app.kubernetes.io/name"] = "ledger" + delete(spec.AdditionalLabels, ledgerV3PreviewLabel) } + // The Ledger Operator deliberately allows these labels to override its + // selectors. Keep the instance label aligned with the Cluster name because + // Formance Services and NetworkPolicies rely on that stable selector. + spec.AdditionalLabels["app.kubernetes.io/instance"] = overrides.ClusterID if hasResourceRequirements(overrides.Resources) { spec.Resources = *overrides.Resources.DeepCopy() @@ -135,13 +145,16 @@ func applyLedgerV3Auth(spec *ledgerv1alpha1.ClusterSpec, configuration *auths.Pr } func applyLedgerV3Monitoring(spec *ledgerv1alpha1.ClusterSpec, configuration *settings.OpenTelemetryConfiguration) { + if spec.Monitoring != nil { + spec.Monitoring.ServiceName = "ledger" + } if configuration == nil { return } if spec.Monitoring == nil { spec.Monitoring = &ledgerv1alpha1.MonitoringConfig{} } - spec.Monitoring.ServiceName = configuration.ServiceName + spec.Monitoring.ServiceName = "ledger" if len(configuration.Attributes) > 0 { attributes := make([]string, 0, len(configuration.Attributes)) for key, value := range configuration.Attributes { diff --git a/internal/resources/ledgers/v3_spec_test.go b/internal/resources/ledgers/v3_spec_test.go index a24cad4a3..2f1d351fd 100644 --- a/internal/resources/ledgers/v3_spec_test.go +++ b/internal/resources/ledgers/v3_spec_test.go @@ -38,9 +38,14 @@ func TestComposeLedgerV3ClusterSpec(t *testing.T) { {Name: "BASE_ONLY", Value: "kept"}, {Name: "SHARED", Value: "base"}, }, - PodAnnotations: map[string]string{"base": "kept"}, - AdditionalLabels: map[string]string{"base": "kept"}, - NodeSelector: map[string]string{"disk": "nvme"}, + PodAnnotations: map[string]string{"base": "kept"}, + AdditionalLabels: map[string]string{ + "base": "kept", + "app.kubernetes.io/name": "configuration-name", + "app.kubernetes.io/instance": "configuration-instance", + ledgerV3PreviewLabel: "false", + }, + NodeSelector: map[string]string{"disk": "nvme"}, TopologySpreadConstraints: []corev1.TopologySpreadConstraint{{ MaxSkew: 2, TopologyKey: "configuration.example/failure-domain", @@ -96,6 +101,8 @@ func TestComposeLedgerV3ClusterSpec(t *testing.T) { require.Equal(t, "ca-hash", actual.PodAnnotations[ledgerV3TLSCAHashAnnotation]) require.Equal(t, "kept", actual.PodAnnotations["base"]) require.Equal(t, "kept", actual.AdditionalLabels["base"]) + require.Equal(t, "ledger-v3-preview", actual.AdditionalLabels["app.kubernetes.io/name"]) + require.Equal(t, "stack0", actual.AdditionalLabels["app.kubernetes.io/instance"]) require.Equal(t, "true", actual.AdditionalLabels[ledgerV3PreviewLabel]) require.Equal(t, "nvme", actual.NodeSelector["disk"]) require.Equal(t, defaultLedgerV3TopologySpreadConstraints(), actual.TopologySpreadConstraints) @@ -143,6 +150,15 @@ func TestComposeLedgerV3ClusterSpecPreservesOptionalBaseValues(t *testing.T) { Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("512Mi")}, }, Auth: &ledgerv1alpha1.AuthorizationConfig{Enabled: pointerTo(true), Issuer: "https://external.example"}, + Monitoring: &ledgerv1alpha1.MonitoringConfig{ + ServiceName: "configuration-service", + Pyroscope: &ledgerv1alpha1.PyroscopeConfig{Enabled: true}, + }, + AdditionalLabels: map[string]string{ + "app.kubernetes.io/name": "configuration-name", + "app.kubernetes.io/instance": "configuration-instance", + ledgerV3PreviewLabel: "true", + }, TopologySpreadConstraints: []corev1.TopologySpreadConstraint{{ MaxSkew: 2, TopologyKey: "configuration.example/failure-domain", @@ -161,6 +177,11 @@ func TestComposeLedgerV3ClusterSpecPreservesOptionalBaseValues(t *testing.T) { require.Equal(t, "512Mi", actual.Resources.Requests.Memory().String()) require.Equal(t, "https://external.example", actual.Auth.Issuer) require.Equal(t, base.TopologySpreadConstraints, actual.TopologySpreadConstraints) + require.Equal(t, "ledger", actual.Monitoring.ServiceName) + require.True(t, actual.Monitoring.Pyroscope.Enabled) + require.Equal(t, "ledger", actual.AdditionalLabels["app.kubernetes.io/name"]) + require.Equal(t, "stack0", actual.AdditionalLabels["app.kubernetes.io/instance"]) + require.NotContains(t, actual.AdditionalLabels, ledgerV3PreviewLabel) } func TestComposeLedgerV3ClusterSpecDisablesConfiguredTopologySpreadConstraints(t *testing.T) { diff --git a/internal/resources/ledgers/v3_test.go b/internal/resources/ledgers/v3_test.go index 7a87aedab..e49b00217 100644 --- a/internal/resources/ledgers/v3_test.go +++ b/internal/resources/ledgers/v3_test.go @@ -5,10 +5,13 @@ import ( "errors" "testing" + authorizationv1 "k8s.io/api/authorization/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/formancehq/operator/v3/api/formance.com/v1beta1" @@ -59,13 +62,38 @@ func (r inaccessibleLedgerV3ResourceReader) List(_ context.Context, object clien type ledgerV3DiscoveryContext struct { context.Context reader client.Reader + client client.Client } -func (c ledgerV3DiscoveryContext) GetClient() client.Client { return nil } +func (c ledgerV3DiscoveryContext) GetClient() client.Client { return c.client } func (c ledgerV3DiscoveryContext) GetScheme() *runtime.Scheme { return nil } func (c ledgerV3DiscoveryContext) GetAPIReader() client.Reader { return c.reader } func (c ledgerV3DiscoveryContext) GetPlatform() core.Platform { return core.Platform{} } +type ledgerV3AccessReviewClient struct { + client.Client + deniedVerb string +} + +type ledgerV3CleanupClient struct { + client.Client + getCalls int +} + +func (c *ledgerV3CleanupClient) Get(_ context.Context, key client.ObjectKey, _ client.Object, _ ...client.GetOption) error { + c.getCalls++ + return apierrors.NewNotFound(schema.GroupResource{Group: ledgerV3ClusterGVK.Group, Resource: "clusters"}, key.Name) +} + +func (c ledgerV3AccessReviewClient) Create(_ context.Context, object client.Object, _ ...client.CreateOption) error { + review := object.(*authorizationv1.SelfSubjectAccessReview) + review.Status.Allowed = review.Spec.ResourceAttributes.Verb != c.deniedVerb + if !review.Status.Allowed { + review.Status.Reason = "denied by test" + } + return nil +} + func TestLedgerV3DiscoveryFailureDisablesCapabilityWithoutFailing(t *testing.T) { previousClusterAvailable := ledgerV3ClusterAvailable previousCertManagerAvailable := ledgerV3CertManagerAvailable @@ -114,6 +142,27 @@ func TestLedgerV3PreviewVersionIgnoredWhenClusterUnavailable(t *testing.T) { } } +func TestDeleteLedgerV3PreviewSkipsCertManagerResourcesWhenUnavailable(t *testing.T) { + previous := ledgerV3CertManagerAvailable + ledgerV3CertManagerAvailable = false + t.Cleanup(func() { + ledgerV3CertManagerAvailable = previous + }) + + kubernetesClient := &ledgerV3CleanupClient{} + ctx := ledgerV3DiscoveryContext{ + Context: context.Background(), + client: kubernetesClient, + } + stack := &v1beta1.Stack{ObjectMeta: metav1.ObjectMeta{Name: "stack0"}} + if err := deleteLedgerV3Preview(ctx, stack); err != nil { + t.Fatalf("deleteLedgerV3Preview() returned error without cert-manager: %v", err) + } + if kubernetesClient.getCalls != 1 { + t.Fatalf("deleteLedgerV3Preview() performed %d GETs, want only the Cluster lookup", kubernetesClient.getCalls) + } +} + func TestLedgerV3ResourceAccessFailureDisablesCapabilityWithoutFailing(t *testing.T) { previous := ledgerV3ClusterAvailable ledgerV3ClusterAvailable = true @@ -135,6 +184,28 @@ func TestLedgerV3ResourceAccessFailureDisablesCapabilityWithoutFailing(t *testin } } +func TestLedgerV3MissingWatchPermissionDisablesCapabilityWithoutFailing(t *testing.T) { + previous := ledgerV3ClusterAvailable + ledgerV3ClusterAvailable = true + t.Cleanup(func() { + ledgerV3ClusterAvailable = previous + }) + + options := core.ReconcilerOptions[*v1beta1.Ledger]{} + withLedgerV3ClusterWatch()(&options) + ctx := ledgerV3DiscoveryContext{ + Context: context.Background(), + reader: inaccessibleLedgerV3ResourceReader{}, + client: ledgerV3AccessReviewClient{deniedVerb: "watch"}, + } + if err := options.Raws[0](ctx, nil); err != nil { + t.Fatalf("Ledger v3 partial RBAC must not fail controller setup: %v", err) + } + if ledgerV3ClusterAvailable { + t.Fatal("Ledger v3 Cluster capability remains enabled without watch permission") + } +} + func TestIsLedgerV3(t *testing.T) { t.Parallel() diff --git a/internal/tests/ledger_v3_controller_test.go b/internal/tests/ledger_v3_controller_test.go index 81402045c..55e275bb4 100644 --- a/internal/tests/ledger_v3_controller_test.go +++ b/internal/tests/ledger_v3_controller_test.go @@ -114,8 +114,11 @@ var _ = Describe("Ledger v3 controller", func() { additionalLabels, found, err := unstructured.NestedStringMap(cluster.Object, "spec", "additionalLabels") Expect(err).NotTo(HaveOccurred()) - Expect(found).To(BeFalse()) - Expect(additionalLabels).To(BeNil()) + Expect(found).To(BeTrue()) + Expect(additionalLabels).To(Equal(map[string]string{ + "app.kubernetes.io/name": "ledger", + "app.kubernetes.io/instance": stack.Name, + })) Expect(cluster.GetLabels()).To(HaveKeyWithValue(v1beta1.LedgerV3Label, "true")) _, found, err = unstructured.NestedMap(cluster.Object, "spec", "auth") @@ -329,10 +332,15 @@ var _ = Describe("Ledger v3 controller", func() { }).Should(Equal(sha256Hex(rotatedCA))) httpAPI := &v1beta1.GatewayHTTPAPI{} - Eventually(func(g Gomega) { + Eventually(func(g Gomega) *v1beta1.GatewayBackendRef { g.Expect(Get(core.GetResourceName(core.GetObjectName(stack.Name, "ledger")), httpAPI)).To(Succeed()) - }).Should(Succeed()) - + g.Expect(httpAPI.Spec.Rules).To(HaveLen(1)) + g.Expect(httpAPI.Spec.Rules[0].Path).To(BeEmpty()) + return httpAPI.Spec.Rules[0].BackendRef + }).Should(Equal(&v1beta1.GatewayBackendRef{ + Name: "ledger-" + stack.Name, + Port: 9000, + })) grpcAPI := &v1beta1.GatewayGRPCAPI{} Eventually(func(g Gomega) *v1beta1.GatewayBackendRef { g.Expect(Get(core.GetResourceName(core.GetObjectName(stack.Name, "ledger")), grpcAPI)).To(Succeed()) @@ -761,6 +769,7 @@ var _ = Describe("Ledger v3 controller", func() { }, "additional": map[string]string{ "app.kubernetes.io/name": "ledger-v3-preview", + "app.kubernetes.io/instance": stack.Name, "formance.com/ledger-v3-preview": "true", }, })) diff --git a/internal/tests/testdata/resources/gateway-controller/configmap-with-audit.yaml b/internal/tests/testdata/resources/gateway-controller/configmap-with-audit.yaml index ed3a60487..62d3bd77e 100644 --- a/internal/tests/testdata/resources/gateway-controller/configmap-with-audit.yaml +++ b/internal/tests/testdata/resources/gateway-controller/configmap-with-audit.yaml @@ -64,7 +64,7 @@ env "staging" endpoints { ledger { - http://ledger:8080/_info http://ledger:8080/ + http://ledger:8080/_info http://ledger:8080/_healthcheck } } } diff --git a/internal/tests/testdata/resources/gateway-controller/configmap-with-ledger-and-another-service.yaml b/internal/tests/testdata/resources/gateway-controller/configmap-with-ledger-and-another-service.yaml index e56ef0ede..e136b9ff2 100644 --- a/internal/tests/testdata/resources/gateway-controller/configmap-with-ledger-and-another-service.yaml +++ b/internal/tests/testdata/resources/gateway-controller/configmap-with-ledger-and-another-service.yaml @@ -58,10 +58,10 @@ env "staging" endpoints { another { - http://another:8080/_info http://another:8080/ + http://another:8080/_info http://another:8080/_healthcheck } ledger { - http://ledger:8080/_info http://ledger:8080/ + http://ledger:8080/_info http://ledger:8080/_healthcheck } } } diff --git a/internal/tests/testdata/resources/gateway-controller/configmap-with-ledger-and-grpc.yaml b/internal/tests/testdata/resources/gateway-controller/configmap-with-ledger-and-grpc.yaml index 507cdb912..e66d01174 100644 --- a/internal/tests/testdata/resources/gateway-controller/configmap-with-ledger-and-grpc.yaml +++ b/internal/tests/testdata/resources/gateway-controller/configmap-with-ledger-and-grpc.yaml @@ -44,7 +44,7 @@ env "staging" endpoints { ledger { - http://ledger:8080/_info http://ledger:8080/ + http://ledger:8080/_info http://ledger:8080/_healthcheck } } } diff --git a/internal/tests/testdata/resources/gateway-controller/configmap-with-ledger-only.yaml b/internal/tests/testdata/resources/gateway-controller/configmap-with-ledger-only.yaml index e331ebb86..2bea51b82 100644 --- a/internal/tests/testdata/resources/gateway-controller/configmap-with-ledger-only.yaml +++ b/internal/tests/testdata/resources/gateway-controller/configmap-with-ledger-only.yaml @@ -43,7 +43,7 @@ env "staging" endpoints { ledger { - http://ledger:8080/_info http://ledger:8080/ + http://ledger:8080/_info http://ledger:8080/_healthcheck } } } diff --git a/internal/tests/testdata/resources/gateway-controller/configmap-with-opentelemetry.yaml b/internal/tests/testdata/resources/gateway-controller/configmap-with-opentelemetry.yaml index 7ee608a09..cec67a7c6 100644 --- a/internal/tests/testdata/resources/gateway-controller/configmap-with-opentelemetry.yaml +++ b/internal/tests/testdata/resources/gateway-controller/configmap-with-opentelemetry.yaml @@ -46,7 +46,7 @@ env "staging" endpoints { ledger { - http://ledger:8080/_info http://ledger:8080/ + http://ledger:8080/_info http://ledger:8080/_healthcheck } } } From cfbc18ee24b88c970fead779a5032fafad1803a4 Mon Sep 17 00:00:00 2001 From: Maxence Maireaux Date: Wed, 15 Jul 2026 15:34:11 +0200 Subject: [PATCH 19/30] fix(ci): stabilize ledger integration tests --- internal/tests/ledger_v3_controller_test.go | 5 +++-- tests/e2e/chainsaw/14-ledger-module/chainsaw-test.yaml | 4 ++-- tests/e2e/chainsaw/14-ledger-module/resources/database.yaml | 2 +- tests/e2e/chainsaw/14-ledger-module/resources/stack.yaml | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/internal/tests/ledger_v3_controller_test.go b/internal/tests/ledger_v3_controller_test.go index 55e275bb4..7adc3c58d 100644 --- a/internal/tests/ledger_v3_controller_test.go +++ b/internal/tests/ledger_v3_controller_test.go @@ -3,6 +3,7 @@ package tests_test import ( "crypto/sha256" "fmt" + "time" "github.com/google/uuid" . "github.com/onsi/ginkgo/v2" @@ -474,7 +475,7 @@ var _ = Describe("Ledger v3 controller", func() { _, found, err := unstructured.NestedMap(cluster.Object, "spec", "auth") g.Expect(err).NotTo(HaveOccurred()) return found - }).Should(BeFalse()) + }, time.Minute).Should(BeFalse()) }) It("combines a managed collector with logs configured in Settings", func() { @@ -587,7 +588,7 @@ var _ = Describe("Ledger v3 controller", func() { Eventually(func(g Gomega) bool { g.Expect(LoadResource("", ledger.Name, ledger)).To(Succeed()) return ledger.Status.Ready - }).Should(BeTrue()) + }, time.Minute).Should(BeTrue()) }) It("removes stale v2 readiness conditions", func() { diff --git a/tests/e2e/chainsaw/14-ledger-module/chainsaw-test.yaml b/tests/e2e/chainsaw/14-ledger-module/chainsaw-test.yaml index def0f091b..c7da5b174 100644 --- a/tests/e2e/chainsaw/14-ledger-module/chainsaw-test.yaml +++ b/tests/e2e/chainsaw/14-ledger-module/chainsaw-test.yaml @@ -46,8 +46,8 @@ spec: kubectl wait --for=create service/ledger -n chainsaw-ledger --timeout=2m kubectl wait --for=create service/ledger-worker -n chainsaw-ledger --timeout=2m kubectl wait --for=create pdb/ledger -n chainsaw-ledger --timeout=2m - kubectl wait --for=jsonpath='{.spec.template.spec.containers[0].image}'=ghcr.io/formancehq/ledger:v3.0.0 deployment/ledger -n chainsaw-ledger --timeout=2m - kubectl wait --for=jsonpath='{.spec.template.spec.containers[0].image}'=ghcr.io/formancehq/ledger:v3.0.0 deployment/ledger-worker -n chainsaw-ledger --timeout=2m + kubectl wait --for=jsonpath='{.spec.template.spec.containers[0].image}'=ghcr.io/formancehq/ledger:v3.0.0-alpha deployment/ledger -n chainsaw-ledger --timeout=2m + kubectl wait --for=jsonpath='{.spec.template.spec.containers[0].image}'=ghcr.io/formancehq/ledger:v3.0.0-alpha deployment/ledger-worker -n chainsaw-ledger --timeout=2m kubectl wait --for=jsonpath='{.spec.ports[0].port}'=8080 service/ledger -n chainsaw-ledger --timeout=2m kubectl wait --for=jsonpath='{.spec.ports[0].port}'=8081 service/ledger-worker -n chainsaw-ledger --timeout=2m kubectl wait --for=jsonpath='{.status.ready}'=true gatewayhttpapi/chainsaw-ledger-ledger --timeout=2m diff --git a/tests/e2e/chainsaw/14-ledger-module/resources/database.yaml b/tests/e2e/chainsaw/14-ledger-module/resources/database.yaml index 05bade79e..e4ff2912d 100644 --- a/tests/e2e/chainsaw/14-ledger-module/resources/database.yaml +++ b/tests/e2e/chainsaw/14-ledger-module/resources/database.yaml @@ -3,7 +3,7 @@ kind: Database metadata: name: chainsaw-ledger-ledger annotations: - formance.com/module-version: v3.0.0 + formance.com/module-version: v3.0.0-alpha spec: stack: chainsaw-ledger service: ledger diff --git a/tests/e2e/chainsaw/14-ledger-module/resources/stack.yaml b/tests/e2e/chainsaw/14-ledger-module/resources/stack.yaml index 801100420..0553567f9 100644 --- a/tests/e2e/chainsaw/14-ledger-module/resources/stack.yaml +++ b/tests/e2e/chainsaw/14-ledger-module/resources/stack.yaml @@ -3,4 +3,4 @@ kind: Stack metadata: name: chainsaw-ledger spec: - version: v3.0.0 + version: v3.0.0-alpha From e605fade708fe9ea73b047a5ae710bd3ee4bd971 Mon Sep 17 00:00:00 2001 From: David Ragot <35502263+Dav-14@users.noreply.github.com> Date: Thu, 16 Jul 2026 15:37:35 +0200 Subject: [PATCH 20/30] fix(ledger): give ledger v3 raft TLS secret a dedicated name (#491) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `ledgerV3TLSName` returned `-tls`, the same Secret name the gateway ingress already uses for its own TLS certificate (`internal/resources/gateways/ingress.go` names it `-tls`, and the gateway is named after the stack). When gateway ingress TLS is enabled, this duplicates an existing mechanism: two independent cert-manager clients target the *same* Secret — - the ledger v3 `Certificate` (self-signed, per-stack Issuer), and - the gateway ingress (ingress-shim, public ClusterIssuer) Each observes the other's issuance as `IncorrectIssuer` and reissues, producing an unbounded reissue race: the keypair rotates every few seconds, the ledger raft mTLS cert reloader repeatedly fails with `tls: private key does not match public key`, and the raft cluster never forms quorum (stuck Degraded). Rename the raft TLS Secret to `-ledger-v3-tls` so it no longer collides with the gateway ingress Secret. The gateway consumes this backend certificate by the `GatewayBackendTLSSecretLabel` label (not by name), so a dedicated name is safe. `ledgerV3TLSName` is the single source used by the Certificate/secretName, the ledger pod mount, the gateway backend ref and the preview cleanup, so the rename propagates consistently. This also matches the name already expected in `v3_spec_test.go` (`stack0-ledger-v3-tls`). Co-authored-by: David Ragot --- internal/resources/ledgers/v3_tls.go | 11 ++++++++++- internal/tests/ledger_v3_controller_test.go | 12 ++++++------ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/internal/resources/ledgers/v3_tls.go b/internal/resources/ledgers/v3_tls.go index 427eee336..335198560 100644 --- a/internal/resources/ledgers/v3_tls.go +++ b/internal/resources/ledgers/v3_tls.go @@ -122,7 +122,16 @@ func ledgerV3IssuerName(stackName string) string { } func ledgerV3TLSName(stackName string) string { - return stackName + "-tls" + // Must be unique to the ledger v3 raft cluster. The bare "-tls" + // name collides with the gateway ingress TLS secret + // (internal/resources/gateways/ingress.go names it "-tls", and + // the gateway is named after the stack). When gateway ingress TLS is + // enabled, both this self-signed Certificate and the gateway ingress + // (ingress-shim / public issuer) target the same Secret and continuously + // reissue it, churning the keypair and breaking raft mTLS. The gateway + // consumes this backend cert by the GatewayBackendTLSSecretLabel, not by + // name, so a dedicated name is safe. + return stackName + "-ledger-v3-tls" } func ledgerV3CertificateSpec(stackName string) map[string]any { diff --git a/internal/tests/ledger_v3_controller_test.go b/internal/tests/ledger_v3_controller_test.go index 7adc3c58d..9e974109a 100644 --- a/internal/tests/ledger_v3_controller_test.go +++ b/internal/tests/ledger_v3_controller_test.go @@ -212,7 +212,7 @@ var _ = Describe("Ledger v3 controller", func() { issuer := newLedgerV3Issuer() certificate := newLedgerV3Certificate() cluster := newLedgerV3Cluster() - tlsName := stack.Name + "-tls" + tlsName := stack.Name + "-ledger-v3-tls" Eventually(func(g Gomega) { g.Expect(Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name + "-selfsigned"}, issuer)).To(Succeed()) @@ -546,7 +546,7 @@ var _ = Describe("Ledger v3 controller", func() { It("mirrors Cluster readiness on the Formance Ledger", func() { certificate := newLedgerV3Certificate() - tlsName := stack.Name + "-tls" + tlsName := stack.Name + "-ledger-v3-tls" Eventually(func(g Gomega) error { g.Expect(Get(types.NamespacedName{Namespace: stack.Name, Name: tlsName}, certificate)).To(Succeed()) return nil @@ -791,7 +791,7 @@ var _ = Describe("Ledger v3 controller", func() { ))) certificate := newLedgerV3Certificate() - tlsName := stack.Name + "-tls" + tlsName := stack.Name + "-ledger-v3-tls" Eventually(func(g Gomega) error { g.Expect(Get(types.NamespacedName{Namespace: stack.Name, Name: tlsName}, certificate)).To(Succeed()) return nil @@ -826,7 +826,7 @@ var _ = Describe("Ledger v3 controller", func() { Name: "ledger-" + stack.Name, Port: 8888, TLS: &v1beta1.GatewayBackendTLS{ - SecretName: stack.Name + "-tls", + SecretName: stack.Name + "-ledger-v3-tls", CASecretKey: "ca.crt", ServerName: "ledger-" + stack.Name + "." + stack.Name + ".svc.cluster.local", }, @@ -862,7 +862,7 @@ var _ = Describe("Ledger v3 controller", func() { certificate := newLedgerV3Certificate() issuer := newLedgerV3Issuer() Eventually(func(g Gomega) { - g.Expect(Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name + "-tls"}, certificate)).To(Succeed()) + g.Expect(Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name + "-ledger-v3-tls"}, certificate)).To(Succeed()) g.Expect(Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name + "-selfsigned"}, issuer)).To(Succeed()) }).Should(Succeed()) @@ -872,7 +872,7 @@ var _ = Describe("Ledger v3 controller", func() { return Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name}, cluster) }).Should(BeNotFound()) Eventually(func() error { - return Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name + "-tls"}, certificate) + return Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name + "-ledger-v3-tls"}, certificate) }).Should(BeNotFound()) Eventually(func() error { return Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name + "-selfsigned"}, issuer) From 647f730d7f9829eee374be109b04ee9e635fa3ae Mon Sep 17 00:00:00 2001 From: Maxence Maireaux Date: Wed, 22 Jul 2026 13:20:08 +0200 Subject: [PATCH 21/30] refactor(ledger): remove unreachable preview cleanup --- internal/resources/ledgers/init.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/internal/resources/ledgers/init.go b/internal/resources/ledgers/init.go index 657cf1d1a..72aa613a9 100644 --- a/internal/resources/ledgers/init.go +++ b/internal/resources/ledgers/init.go @@ -89,12 +89,6 @@ func reconcileLegacy(ctx Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, }, httpRules...) } - if previewVersion != "" && !ledgerV3ClusterAvailable { - if err := DeleteIfExists[*v1beta1.GatewayGRPCAPI](ctx, GetResourceName(GetObjectName(stack.Name, "ledger"))); err != nil { - return err - } - } - database, err := databases.Create(ctx, stack, ledger) if err != nil { return err From f94cf172f8cece3a1971e0f29dba7bb5570ead14 Mon Sep 17 00:00:00 2001 From: Maxence Maireaux Date: Wed, 22 Jul 2026 13:39:36 +0200 Subject: [PATCH 22/30] fix(ledger): address v3 integration review findings --- Dockerfile | 3 + .../v1beta1/gatewaygrpcapi_types.go | 2 - .../v1beta1/gatewayhttpapi_types.go | 2 - deployment/operator/.gitignore | 2 +- deployment/operator/Pulumi.yaml | 3 +- deployment/operator/helpers.go | 23 ++++---- deployment/operator/main.go | 36 ++++++----- docs.config.yaml | 3 + docs/04-Modules/03-Ledger.md | 9 +-- .../02-Custom Resource Definitions.md | 4 +- .../settings.catalog.json | 12 ++-- internal/resources/gateways/Caddyfile.gotpl | 31 ++++++++++ internal/resources/ledgers/init.go | 9 +-- internal/resources/ledgers/v3.go | 40 +++++++------ internal/resources/ledgers/v3_preview.go | 52 ++++++++++++---- internal/resources/ledgers/v3_spec.go | 18 ++++-- internal/resources/ledgers/v3_spec_test.go | 18 +++++- internal/resources/ledgers/v3_test.go | 31 ++++++++-- internal/resources/ledgers/v3_tls.go | 14 +++-- internal/resources/stacks/networkpolicies.go | 10 ++-- internal/tests/ledger_v3_controller_test.go | 59 +++++++++++++------ .../tests/networkpolicy_controller_test.go | 7 ++- 22 files changed, 256 insertions(+), 132 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6733ff7d3..2ebe19ad3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,7 +28,10 @@ RUN set -e; \ FROM alpine:3.20 RUN apk update && apk add --no-cache ca-certificates curl +RUN addgroup -S operator && adduser -S -G operator operator ENTRYPOINT ["/usr/bin/operator"] COPY --from=builder /usr/bin/operator /usr/bin/operator + +USER operator diff --git a/api/formance.com/v1beta1/gatewaygrpcapi_types.go b/api/formance.com/v1beta1/gatewaygrpcapi_types.go index 9b9d7a4a8..97202f32b 100644 --- a/api/formance.com/v1beta1/gatewaygrpcapi_types.go +++ b/api/formance.com/v1beta1/gatewaygrpcapi_types.go @@ -38,8 +38,6 @@ type GatewayGRPCAPISpec struct { type GatewayGRPCAPIStatus struct { Status `json:",inline"` - //+optional - Ready bool `json:"ready,omitempty"` } //+kubebuilder:object:root=true diff --git a/api/formance.com/v1beta1/gatewayhttpapi_types.go b/api/formance.com/v1beta1/gatewayhttpapi_types.go index 35200248c..1f9a44851 100644 --- a/api/formance.com/v1beta1/gatewayhttpapi_types.go +++ b/api/formance.com/v1beta1/gatewayhttpapi_types.go @@ -44,8 +44,6 @@ type GatewayHTTPAPISpec struct { type GatewayHTTPAPIStatus struct { Status `json:",inline"` - //+optional - Ready bool `json:"ready,omitempty"` } //+kubebuilder:object:root=true diff --git a/deployment/operator/.gitignore b/deployment/operator/.gitignore index 7dc315b66..8871a5977 100644 --- a/deployment/operator/.gitignore +++ b/deployment/operator/.gitignore @@ -1,2 +1,2 @@ -deployment/operator/bin/ +bin/ operator diff --git a/deployment/operator/Pulumi.yaml b/deployment/operator/Pulumi.yaml index 0b8aac981..0f4343f57 100644 --- a/deployment/operator/Pulumi.yaml +++ b/deployment/operator/Pulumi.yaml @@ -27,8 +27,7 @@ x-plr-config: description: Image tag (defaults to git version) arch: type: string - default: amd64 - description: Target CPU architecture + description: Target CPU architecture (defaults to amd64 and arm64) registry-username: type: string secret: true diff --git a/deployment/operator/helpers.go b/deployment/operator/helpers.go index 44effc7eb..60a2910e5 100644 --- a/deployment/operator/helpers.go +++ b/deployment/operator/helpers.go @@ -92,17 +92,17 @@ func newDockerConfig(ctx *pulumi.Context, cfg *config.Config) *dockerConfig { } arch := cfg.Get("arch") - if arch == "" { - arch = "amd64" - } - platforms := make([]string, 0, len(allPlatforms)) - for _, p := range allPlatforms { - if strings.HasSuffix(p, arch) { - platforms = append(platforms, p) + platforms := append([]string(nil), allPlatforms...) + if arch != "" { + platforms = platforms[:0] + for _, p := range allPlatforms { + if strings.HasSuffix(p, arch) { + platforms = append(platforms, p) + } + } + if len(platforms) == 0 { + platforms = []string{"linux-" + arch} } - } - if len(platforms) == 0 { - platforms = []string{"linux-" + arch} } return &dockerConfig{ @@ -159,7 +159,8 @@ func (dc *dockerConfig) buildImage( CacheTo: dockerbuild.CacheToArray{ dockerbuild.CacheToArgs{ Registry: dockerbuild.CacheToRegistryArgs{ - Ref: pulumi.Sprintf("%s/%s:buildcache-%s,mode=max", dc.Registry, name, platform), + Ref: pulumi.Sprintf("%s/%s:buildcache-%s", dc.Registry, name, platform), + Mode: dockerbuild.CacheModeMax, }, }, }, diff --git a/deployment/operator/main.go b/deployment/operator/main.go index 0c8ddb1c0..7d997125c 100644 --- a/deployment/operator/main.go +++ b/deployment/operator/main.go @@ -5,8 +5,8 @@ import ( "path/filepath" "strings" - k8syaml "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/yaml" "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/helm/v3" + k8syaml "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/yaml" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" "github.com/pulumi/pulumi/sdk/v3/go/pulumi/config" ) @@ -38,6 +38,9 @@ func main() { if err != nil { return fmt.Errorf("failed to glob CRD files: %w", err) } + if len(crdFiles) == 0 { + return fmt.Errorf("no CRD manifests found under ../../config/crd/bases") + } var crds []pulumi.Resource for _, crdFile := range crdFiles { @@ -62,30 +65,25 @@ func main() { } // Licence configuration - licenceValues := pulumi.Map{} - licenceToken := cfg.Get("licence-token") - if licenceToken != "" { - licenceIssuer := cfg.Get("licence-issuer") - if licenceIssuer == "" { - licenceIssuer = "https://license.formance.cloud/keys" - } - licenceValues = pulumi.Map{ - "createSecret": pulumi.Bool(true), - "token": pulumi.String(licenceToken), - "issuer": pulumi.String(licenceIssuer), - } - } else { - licenceValues = pulumi.Map{ - "createSecret": pulumi.Bool(false), - } + licenceIssuer := cfg.Get("licence-issuer") + if licenceIssuer == "" { + licenceIssuer = "https://license.formance.cloud/keys" + } + licenceToken := config.GetSecret(ctx, "licence-token") + licenceValues := pulumi.Map{ + "createSecret": licenceToken.ApplyT(func(token string) bool { + return token != "" + }).(pulumi.BoolOutput), + "token": licenceToken, + "issuer": pulumi.String(licenceIssuer), } // Deploy operator via Helm operatorChartPath := filepath.Join("..", "..", "helm", "operator") operatorRelease, err := helm.NewRelease(ctx, "formance-operator", &helm.ReleaseArgs{ - Name: pulumi.String("formance-operator"), - Chart: pulumi.String(operatorChartPath), + Name: pulumi.String("formance-operator"), + Chart: pulumi.String(operatorChartPath), Namespace: pulumi.String(namespace), CreateNamespace: pulumi.Bool(true), Values: pulumi.Map{ diff --git a/docs.config.yaml b/docs.config.yaml index 82f93b1c7..a1737b792 100644 --- a/docs.config.yaml +++ b/docs.config.yaml @@ -20,3 +20,6 @@ render: - name: SecretObjectReference package: sigs.k8s.io/gateway-api/apis/v1beta1 link: https://gateway-api.sigs.k8s.io/references/spec/#gateway.networking.k8s.io/v1beta1.SecretObjectReference + - name: ClusterSpec + package: github.com/formancehq/ledger/misc/operator/api/v1alpha1 + link: https://github.com/formancehq/ledger/blob/release/v3.0/misc/operator/api/v1alpha1/cluster_types.go diff --git a/docs/04-Modules/03-Ledger.md b/docs/04-Modules/03-Ledger.md index 1cbb6e249..45ee82e1b 100644 --- a/docs/04-Modules/03-Ledger.md +++ b/docs/04-Modules/03-Ledger.md @@ -4,12 +4,13 @@ Formance Ledger is a real-time money tracking microservice that lets you model a Ledger versions up to and including `v3.0.0-alpha` require: -- **PostgreSQL**: See configuration guide [here](../05-Infrastructure%20services/01-PostgreSQL.md). -- (Optional) **Broker**: See configuration guide [here](../05-Infrastructure%20services/02-Message%20broker.md). +- See the [PostgreSQL configuration](../05-Infrastructure%20services/01-PostgreSQL.md). +- (Optional) See the [message broker configuration](../05-Infrastructure%20services/02-Message%20broker.md). Ledger versions newer than `v3.0.0-alpha` require the Ledger Operator and its -`ledger.formance.com/v1alpha1` CRDs to be installed in the cluster. They use the -Ledger v3 native storage and do not require a PostgreSQL `Database` resource. +`ledger.formance.com/v1alpha1` CRDs, plus cert-manager with its `Issuer` and +`Certificate` CRDs, to be installed in the cluster. They use Ledger v3 native +storage and do not require a PostgreSQL `Database` resource. ## Ledger Object diff --git a/docs/09-Configuration reference/02-Custom Resource Definitions.md b/docs/09-Configuration reference/02-Custom Resource Definitions.md index 4b72cdc84..8fdb089c2 100644 --- a/docs/09-Configuration reference/02-Custom Resource Definitions.md +++ b/docs/09-Configuration reference/02-Custom Resource Definitions.md @@ -2384,7 +2384,6 @@ GatewayBackendTLS configures TLS when Gateway connects to a backend. | --- | --- | --- | --- | | `ready` _boolean_ | Ready indicates if the resource is seen as completely reconciled | | | | `info` _string_ | Info can contain any additional like reconciliation errors | | | -| `ready` _boolean_ | | | | #### GatewayHTTPAPI @@ -2499,7 +2498,6 @@ GatewayHTTPAPI is the Schema for the HTTPAPIs API | --- | --- | --- | --- | | `ready` _boolean_ | Ready indicates if the resource is seen as completely reconciled | | | | `info` _string_ | Info can contain any additional like reconciliation errors | | | -| `ready` _boolean_ | | | | #### LedgerConfiguration @@ -2556,7 +2554,7 @@ takes priority over a configuration targeting all stacks with `*`. | Field | Description | Default | Validation | | --- | --- | --- | --- | | `stacks` _string array_ | Stacks on which the configuration is applied. Can contain `*` to
indicate a wildcard, following the same convention as Settings. | | | -| `cluster` _[ClusterSpec](#clusterspec)_ | Cluster is the base Ledger v3 Cluster specification. Stack-specific
Settings and values owned by the Operator are applied on top of it. | | | +| `cluster` _[ClusterSpec](https://github.com/formancehq/ledger/blob/release/v3.0/misc/operator/api/v1alpha1/cluster_types.go)_ | Cluster is the base Ledger v3 Cluster specification. Stack-specific
Settings and values owned by the Operator are applied on top of it. | | | diff --git a/docs/09-Configuration reference/settings.catalog.json b/docs/09-Configuration reference/settings.catalog.json index b98067c44..0bc89d032 100644 --- a/docs/09-Configuration reference/settings.catalog.json +++ b/docs/09-Configuration reference/settings.catalog.json @@ -201,21 +201,21 @@ "key": "deployments.ledger.containers.ledger.resource-requirements.claims", "valueType": "string[]", "sources": [ - "internal/resources/ledgers/v3.go:343" + "internal/resources/ledgers/v3.go:346" ] }, { "key": "deployments.ledger.containers.ledger.resource-requirements.limits", "valueType": "map[string]string", "sources": [ - "internal/resources/ledgers/v3.go:343" + "internal/resources/ledgers/v3.go:346" ] }, { "key": "deployments.ledger.containers.ledger.resource-requirements.requests", "valueType": "map[string]string", "sources": [ - "internal/resources/ledgers/v3.go:343" + "internal/resources/ledgers/v3.go:346" ] }, { @@ -223,14 +223,14 @@ "valueType": "int32", "default": "3", "sources": [ - "internal/resources/ledgers/v3.go:329" + "internal/resources/ledgers/v3.go:332" ] }, { "key": "deployments.ledger.topology-spread-constraints", "valueType": "bool", "sources": [ - "internal/resources/ledgers/v3.go:361" + "internal/resources/ledgers/v3.go:364" ] }, { @@ -473,7 +473,7 @@ "key": "ledger.v3.preview-version", "valueType": "string", "sources": [ - "internal/resources/ledgers/v3_preview.go:43" + "internal/resources/ledgers/v3_preview.go:44" ] }, { diff --git a/internal/resources/gateways/Caddyfile.gotpl b/internal/resources/gateways/Caddyfile.gotpl index 636ca9643..9dc23c4f0 100644 --- a/internal/resources/gateways/Caddyfile.gotpl +++ b/internal/resources/gateways/Caddyfile.gotpl @@ -136,6 +136,29 @@ {{- end }} {{- end }} + {{- range $i, $service := .Services }} + {{- if $service.HealthCheckBackend.TLS }} + {{- $healthCheckEndpoint := $service.HealthCheckEndpoint }} + {{- if eq $healthCheckEndpoint "" }} + {{- $healthCheckEndpoint = "_healthcheck" }} + {{- end }} + @tls_health_{{ $i }} { + remote_ip 127.0.0.1 ::1 + path /_gateway-health/{{ $service.Name }}/_info /_gateway-health/{{ $service.Name }}/{{ $healthCheckEndpoint }} + } + handle @tls_health_{{ $i }} { + uri strip_prefix /_gateway-health/{{ $service.Name }} + reverse_proxy https://{{ $service.HealthCheckBackend.Name }}:{{ $service.HealthCheckBackend.Port }} { + transport http { + tls + tls_trust_pool file /etc/gateway/tls/{{ $service.HealthCheckBackend.TLS.SecretName }}/{{ $service.HealthCheckBackend.TLS.CASecretKey }} + tls_server_name {{ $service.HealthCheckBackend.TLS.ServerName }} + } + } + } + {{- end }} + {{- end }} + handle /versions { versions { region "{{ .Platform.Region }}" @@ -149,12 +172,20 @@ {{- if or (not (semver_is_valid $values.Gateway.Version)) (gt (semver_compare $values.Gateway.Version "v0.1.7") 0) }} {{ $service.Name }} { + {{- if $service.HealthCheckBackend.TLS }} + http://127.0.0.1:8080/_gateway-health/{{ $service.Name }}/_info http://127.0.0.1:8080/_gateway-health/{{ $service.Name }}/{{ $healthCheckEndpoint }} + {{- else }} http://{{ $service.HealthCheckBackend.Name }}:{{ $service.HealthCheckBackend.Port }}/_info http://{{ $service.HealthCheckBackend.Name }}:{{ $service.HealthCheckBackend.Port }}/{{ $healthCheckEndpoint }} + {{- end }} } {{- else }} + {{- if $service.HealthCheckBackend.TLS }} + {{ $service.Name }} http://127.0.0.1:8080/_gateway-health/{{ $service.Name }}/_info http://127.0.0.1:8080/_gateway-health/{{ $service.Name }}/{{ $healthCheckEndpoint }} + {{- else }} {{ $service.Name }} http://{{ $service.HealthCheckBackend.Name }}:{{ $service.HealthCheckBackend.Port }}/_info http://{{ $service.HealthCheckBackend.Name }}:{{ $service.HealthCheckBackend.Port }}/{{ $healthCheckEndpoint }} {{- end }} {{- end }} + {{- end }} } } } diff --git a/internal/resources/ledgers/init.go b/internal/resources/ledgers/init.go index 72aa613a9..8432d5c7a 100644 --- a/internal/resources/ledgers/init.go +++ b/internal/resources/ledgers/init.go @@ -82,13 +82,6 @@ func reconcileLegacy(ctx Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, } } - httpRules := []v1beta1.GatewayHTTPAPIRule{gatewayhttpapis.RuleSecured()} - if previewVersion != "" { - httpRules = append([]v1beta1.GatewayHTTPAPIRule{ - gatewayhttpapis.RuleSecuredWithBackend("/v3", ledgerV3HTTPBackendRef(stack.Name)), - }, httpRules...) - } - database, err := databases.Create(ctx, stack, ledger) if err != nil { return err @@ -101,7 +94,7 @@ func reconcileLegacy(ctx Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, if err := gatewayhttpapis.Create(ctx, ledger, gatewayhttpapis.WithHealthCheckEndpoint("_healthcheck"), - gatewayhttpapis.WithRules(httpRules...), + gatewayhttpapis.WithRules(gatewayhttpapis.RuleSecured()), ); err != nil { return err } diff --git a/internal/resources/ledgers/v3.go b/internal/resources/ledgers/v3.go index 2ff19851e..86f117678 100644 --- a/internal/resources/ledgers/v3.go +++ b/internal/resources/ledgers/v3.go @@ -212,25 +212,28 @@ func reconcileV3(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, return err } - cluster, err := createOrUpdateV3Cluster(ctx, stack, ledger, version, false, tlsCAHash) + cluster, clusterSpec, err := createOrUpdateV3Cluster(ctx, stack, ledger, version, false, tlsCAHash) if err != nil { setLedgerV3Condition(ledger, metav1.ConditionFalse, "ReconcileFailed", err.Error()) return err } if !tlsReady { + if err := core.DeleteIfExists[*v1beta1.GatewayGRPCAPI](ctx, core.GetResourceName(core.GetObjectName(stack.Name, "ledger"))); err != nil { + return err + } setLedgerV3Condition(ledger, metav1.ConditionFalse, "TLSCertificatePending", tlsMessage) return core.NewPendingError().WithMessage("Ledger v3 TLS is not ready: %s", tlsMessage) } if err := gatewayhttpapis.Create(ctx, ledger, gatewayhttpapis.WithHealthCheckEndpoint("livez"), - gatewayhttpapis.WithRules(gatewayhttpapis.RuleSecuredWithBackend("", ledgerV3HTTPBackendRef(stack.Name))), + gatewayhttpapis.WithRules(gatewayhttpapis.RuleSecuredWithBackend("", ledgerV3HTTPBackendRef(stack.Name, clusterSpec.Service.HttpPort))), ); err != nil { return err } if err := gatewaygrpcapis.Create(ctx, ledger, gatewaygrpcapis.WithGRPCServices(ledgerV3PublicGRPCService), - gatewaygrpcapis.WithPort(ledgerV3GRPCPort), - gatewaygrpcapis.WithBackendRef(ledgerV3GRPCBackendRef(stack.Name)), + gatewaygrpcapis.WithPort(ledgerV3ServicePort(clusterSpec.Service.GrpcPort, ledgerV3GRPCPort)), + gatewaygrpcapis.WithBackendRef(ledgerV3GRPCBackendRef(stack.Name, clusterSpec.Service.GrpcPort)), ); err != nil { return err } @@ -315,24 +318,24 @@ func normalizeLedgerV3Replicas(configured int32) (int32, bool, error) { return configured, false, nil } -func createOrUpdateV3Cluster(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, version string, preview bool, tlsCAHash string) (*unstructured.Unstructured, error) { +func createOrUpdateV3Cluster(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, version string, preview bool, tlsCAHash string) (*unstructured.Unstructured, *ledgerv1alpha1.ClusterSpec, error) { baseSpec, err := ledgerV3BaseSpec(ctx, stack.Name) if err != nil { - return nil, err + return nil, nil, err } image, err := registries.GetFormanceImage(ctx, stack, "ledger", version) if err != nil { - return nil, err + return nil, nil, err } configuredReplicas, err := settings.GetInt32OrDefault(ctx, stack.Name, 3, "deployments", "ledger", "replicas") if err != nil { - return nil, err + return nil, nil, err } replicas, normalized, err := normalizeLedgerV3Replicas(configuredReplicas) if err != nil { - return nil, err + return nil, nil, err } if normalized { log.FromContext(ctx).Info("Normalized Ledger v3 replicas to an odd number", @@ -343,27 +346,27 @@ func createOrUpdateV3Cluster(ctx core.Context, stack *v1beta1.Stack, ledger *v1b resourceRequirements, err := settings.GetResourceRequirements(ctx, stack.Name, "deployments", "ledger", "containers", "ledger", "resource-requirements") if err != nil { - return nil, err + return nil, nil, err } monitoringConfiguration, err := settings.GetOpenTelemetryConfiguration(ctx, stack.Name, "ledger") if err != nil { - return nil, err + return nil, nil, err } authConfiguration, err := auths.GetProtectedConfiguration(ctx, stack, "ledger", ledger.Spec.Auth) if err != nil { - return nil, err + return nil, nil, err } serviceAccountName, err := settings.GetAWSServiceAccount(ctx, stack.Name) if err != nil { - return nil, err + return nil, nil, err } topologySpreadConstraints, err := settings.GetBool(ctx, stack.Name, "deployments", "ledger", "topology-spread-constraints") if err != nil { - return nil, err + return nil, nil, err } - desiredSpec := composeLedgerV3ClusterSpec(baseSpec, ledgerV3SpecOverrides{ + desiredSpec, err := composeLedgerV3ClusterSpec(baseSpec, ledgerV3SpecOverrides{ ImageRepository: imageRepository(image), ImageTag: image.Version, ImagePullSecrets: image.PullSecrets, @@ -380,9 +383,12 @@ func createOrUpdateV3Cluster(ctx core.Context, stack *v1beta1.Stack, ledger *v1b ServiceAccountName: serviceAccountName, TopologySpreadConstraints: topologySpreadConstraints, }) + if err != nil { + return nil, nil, fmt.Errorf("composing Ledger v3 Cluster spec: %w", err) + } desiredSpecMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(desiredSpec) if err != nil { - return nil, fmt.Errorf("converting Ledger v3 Cluster spec: %w", err) + return nil, nil, fmt.Errorf("converting Ledger v3 Cluster spec: %w", err) } // ResourceRequirements is a value struct in the Ledger API. The // unstructured converter therefore emits an empty resources object even @@ -422,7 +428,7 @@ func createOrUpdateV3Cluster(ctx core.Context, stack *v1beta1.Stack, ledger *v1b return nil }) - return cluster, err + return cluster, desiredSpec, err } func ledgerV3BaseSpec(ctx core.Context, stack string) (*ledgerv1alpha1.ClusterSpec, error) { diff --git a/internal/resources/ledgers/v3_preview.go b/internal/resources/ledgers/v3_preview.go index d9da5d779..0df08df0d 100644 --- a/internal/resources/ledgers/v3_preview.go +++ b/internal/resources/ledgers/v3_preview.go @@ -29,6 +29,7 @@ import ( "github.com/formancehq/operator/v3/api/formance.com/v1beta1" "github.com/formancehq/operator/v3/internal/core" "github.com/formancehq/operator/v3/internal/resources/gatewaygrpcapis" + "github.com/formancehq/operator/v3/internal/resources/gatewayhttpapis" "github.com/formancehq/operator/v3/internal/resources/settings" ) @@ -61,7 +62,7 @@ func reconcileV3Preview(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1. setLedgerV3PreviewCondition(ledger, metav1.ConditionFalse, "TLSReconcileFailed", err.Error()) return err } - cluster, err := createOrUpdateV3Cluster(ctx, stack, ledger, version, true, tlsCAHash) + cluster, clusterSpec, err := createOrUpdateV3Cluster(ctx, stack, ledger, version, true, tlsCAHash) if err != nil { setLedgerV3PreviewCondition(ledger, metav1.ConditionFalse, "ReconcileFailed", err.Error()) return err @@ -74,10 +75,21 @@ func reconcileV3Preview(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1. return core.NewPendingError().WithMessage("Ledger v3 preview TLS is not ready: %s", tlsMessage) } + if err := gatewayhttpapis.Create(ctx, ledger, + gatewayhttpapis.WithHealthCheckEndpoint("_healthcheck"), + gatewayhttpapis.WithRules( + gatewayhttpapis.RuleSecuredWithBackend("/v3", ledgerV3HTTPBackendRef(stack.Name, clusterSpec.Service.HttpPort)), + gatewayhttpapis.RuleSecured(), + ), + ); err != nil { + setLedgerV3PreviewCondition(ledger, metav1.ConditionFalse, "GatewayReconcileFailed", err.Error()) + return err + } + if err := gatewaygrpcapis.Create(ctx, ledger, gatewaygrpcapis.WithGRPCServices(ledgerV3PublicGRPCService), - gatewaygrpcapis.WithPort(ledgerV3GRPCPort), - gatewaygrpcapis.WithBackendRef(ledgerV3GRPCBackendRef(stack.Name)), + gatewaygrpcapis.WithPort(ledgerV3ServicePort(clusterSpec.Service.GrpcPort, ledgerV3GRPCPort)), + gatewaygrpcapis.WithBackendRef(ledgerV3GRPCBackendRef(stack.Name, clusterSpec.Service.GrpcPort)), ); err != nil { setLedgerV3PreviewCondition(ledger, metav1.ConditionFalse, "GatewayReconcileFailed", err.Error()) return err @@ -96,17 +108,17 @@ func reconcileV3Preview(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1. return nil } -func ledgerV3HTTPBackendRef(stackName string) v1beta1.GatewayBackendRef { +func ledgerV3HTTPBackendRef(stackName string, port int32) v1beta1.GatewayBackendRef { return v1beta1.GatewayBackendRef{ Name: "ledger-" + stackName, - Port: ledgerV3HTTPPort, + Port: ledgerV3ServicePort(port, ledgerV3HTTPPort), } } -func ledgerV3GRPCBackendRef(stackName string) v1beta1.GatewayBackendRef { +func ledgerV3GRPCBackendRef(stackName string, port int32) v1beta1.GatewayBackendRef { return v1beta1.GatewayBackendRef{ Name: "ledger-" + stackName, - Port: ledgerV3GRPCPort, + Port: ledgerV3ServicePort(port, ledgerV3GRPCPort), TLS: &v1beta1.GatewayBackendTLS{ SecretName: ledgerV3TLSName(stackName), CASecretKey: ledgerV3TLSCASecretKey, @@ -115,6 +127,13 @@ func ledgerV3GRPCBackendRef(stackName string) v1beta1.GatewayBackendRef { } } +func ledgerV3ServicePort(configured, defaultPort int32) int32 { + if configured == 0 { + return defaultPort + } + return configured +} + func setLedgerV3PreviewCondition(ledger *v1beta1.Ledger, status metav1.ConditionStatus, reason, message string) { ledger.GetConditions().AppendOrReplace(v1beta1.Condition{ Type: ledgerV3PreviewReadyCondition, @@ -140,9 +159,20 @@ func deleteLedgerV3Preview(ctx core.Context, stack *v1beta1.Stack) error { return err } } + + secret := &corev1.Secret{} + err = ctx.GetClient().Get(ctx, types.NamespacedName{Namespace: stack.Name, Name: ledgerV3TLSName(stack.Name)}, secret) + if err == nil && secret.GetLabels()[ledgerV3PreviewLabel] == "true" { + if err := ctx.GetClient().Delete(ctx, secret); err != nil && !apierrors.IsNotFound(err) { + return err + } + } else if err != nil && !apierrors.IsNotFound(err) { + return err + } + // A missing or inaccessible cert-manager dependency must not block the - // legacy Ledger reconciliation. The Cluster cleanup above is still safe and - // useful because its capability is checked independently. + // legacy Ledger reconciliation. Cluster and Secret cleanup above remain + // safe because they use preview-specific labels and do not require the CRDs. if !ledgerV3CertManagerAvailable { return nil } @@ -153,10 +183,6 @@ func deleteLedgerV3Preview(ctx core.Context, stack *v1beta1.Stack) error { if err := ctx.GetClient().Delete(ctx, certificate); err != nil && !apierrors.IsNotFound(err) { return err } - secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: stack.Name, Name: ledgerV3TLSName(stack.Name)}} - if err := ctx.GetClient().Delete(ctx, secret); err != nil && !apierrors.IsNotFound(err) { - return err - } } else if err != nil && !apierrors.IsNotFound(err) { return err } diff --git a/internal/resources/ledgers/v3_spec.go b/internal/resources/ledgers/v3_spec.go index bf9891ea2..95912c8e7 100644 --- a/internal/resources/ledgers/v3_spec.go +++ b/internal/resources/ledgers/v3_spec.go @@ -1,6 +1,8 @@ package ledgers import ( + "fmt" + "math" "slices" "strconv" "strings" @@ -35,7 +37,7 @@ type ledgerV3SpecOverrides struct { // composeLedgerV3ClusterSpec applies the values owned by the Formance Operator // to a shared ClusterSpec. Unmanaged fields remain entirely owned by the // LedgerConfiguration. -func composeLedgerV3ClusterSpec(base *ledgerv1alpha1.ClusterSpec, overrides ledgerV3SpecOverrides) *ledgerv1alpha1.ClusterSpec { +func composeLedgerV3ClusterSpec(base *ledgerv1alpha1.ClusterSpec, overrides ledgerV3SpecOverrides) (*ledgerv1alpha1.ClusterSpec, error) { spec := base.DeepCopy() spec.Image.Repository = overrides.ImageRepository @@ -91,7 +93,9 @@ func composeLedgerV3ClusterSpec(base *ledgerv1alpha1.ClusterSpec, overrides ledg spec.ExtraEnv = core.MergeEnvVars(spec.ExtraEnv, overrides.ExtraEnv) } applyLedgerV3Monitoring(spec, overrides.Monitoring) - applyLedgerV3Auth(spec, overrides.Auth) + if err := applyLedgerV3Auth(spec, overrides.Auth); err != nil { + return nil, err + } if overrides.ServiceAccountName != "" { spec.ServiceAccount.Create = pointerTo(false) @@ -105,7 +109,7 @@ func composeLedgerV3ClusterSpec(base *ledgerv1alpha1.ClusterSpec, overrides ledg } } - return spec + return spec, nil } func defaultLedgerV3TopologySpreadConstraints() []corev1.TopologySpreadConstraint { @@ -123,9 +127,9 @@ func defaultLedgerV3TopologySpreadConstraints() []corev1.TopologySpreadConstrain } } -func applyLedgerV3Auth(spec *ledgerv1alpha1.ClusterSpec, configuration *auths.ProtectedAuthConfiguration) { +func applyLedgerV3Auth(spec *ledgerv1alpha1.ClusterSpec, configuration *auths.ProtectedAuthConfiguration) error { if configuration == nil { - return + return nil } if spec.Auth == nil { spec.Auth = &ledgerv1alpha1.AuthorizationConfig{} @@ -140,8 +144,12 @@ func applyLedgerV3Auth(spec *ledgerv1alpha1.ClusterSpec, configuration *auths.Pr spec.Auth.Service = configuration.Service } if configuration.ReadKeySetMaxRetries != 0 { + if configuration.ReadKeySetMaxRetries > math.MaxInt32 || configuration.ReadKeySetMaxRetries < math.MinInt32 { + return fmt.Errorf("auth read key set max retries must fit in int32, got %d", configuration.ReadKeySetMaxRetries) + } spec.Auth.ReadKeySetMaxRetries = pointerTo(int32(configuration.ReadKeySetMaxRetries)) } + return nil } func applyLedgerV3Monitoring(spec *ledgerv1alpha1.ClusterSpec, configuration *settings.OpenTelemetryConfiguration) { diff --git a/internal/resources/ledgers/v3_spec_test.go b/internal/resources/ledgers/v3_spec_test.go index 2f1d351fd..ddfbee148 100644 --- a/internal/resources/ledgers/v3_spec_test.go +++ b/internal/resources/ledgers/v3_spec_test.go @@ -56,7 +56,7 @@ func TestComposeLedgerV3ClusterSpec(t *testing.T) { }, } - actual := composeLedgerV3ClusterSpec(base, ledgerV3SpecOverrides{ + actual, err := composeLedgerV3ClusterSpec(base, ledgerV3SpecOverrides{ ImageRepository: "registry.example/ledger", ImageTag: "v3.0.0", ImagePullSecrets: []corev1.LocalObjectReference{{Name: "registry"}}, @@ -90,6 +90,7 @@ func TestComposeLedgerV3ClusterSpec(t *testing.T) { ServiceAccountName: "ledger-aws", TopologySpreadConstraints: pointerTo(true), }) + require.NoError(t, err) require.Equal(t, "registry.example/ledger", actual.Image.Repository) require.Equal(t, "v3.0.0", actual.Image.Tag) @@ -165,13 +166,14 @@ func TestComposeLedgerV3ClusterSpecPreservesOptionalBaseValues(t *testing.T) { WhenUnsatisfiable: corev1.ScheduleAnyway, }}, } - actual := composeLedgerV3ClusterSpec(base, ledgerV3SpecOverrides{ + actual, err := composeLedgerV3ClusterSpec(base, ledgerV3SpecOverrides{ ImageRepository: "ledger", ImageTag: "latest", Replicas: 3, ClusterID: "stack0", TLSSecretName: "tls", }) + require.NoError(t, err) require.Equal(t, "base-registry", actual.ImagePullSecrets[0].Name) require.Equal(t, "512Mi", actual.Resources.Requests.Memory().String()) @@ -190,7 +192,7 @@ func TestComposeLedgerV3ClusterSpecDisablesConfiguredTopologySpreadConstraints(t base := &ledgerv1alpha1.ClusterSpec{ TopologySpreadConstraints: defaultLedgerV3TopologySpreadConstraints(), } - actual := composeLedgerV3ClusterSpec(base, ledgerV3SpecOverrides{ + actual, err := composeLedgerV3ClusterSpec(base, ledgerV3SpecOverrides{ ImageRepository: "ledger", ImageTag: "latest", Replicas: 3, @@ -198,7 +200,17 @@ func TestComposeLedgerV3ClusterSpecDisablesConfiguredTopologySpreadConstraints(t TLSSecretName: "tls", TopologySpreadConstraints: pointerTo(false), }) + require.NoError(t, err) require.Nil(t, actual.TopologySpreadConstraints) require.NotEmpty(t, base.TopologySpreadConstraints) } + +func TestComposeLedgerV3ClusterSpecRejectsOversizedAuthRetries(t *testing.T) { + t.Parallel() + + _, err := composeLedgerV3ClusterSpec(&ledgerv1alpha1.ClusterSpec{}, ledgerV3SpecOverrides{ + Auth: &auths.ProtectedAuthConfiguration{ReadKeySetMaxRetries: int(^uint32(0)>>1) + 1}, + }) + require.ErrorContains(t, err, "must fit in int32") +} diff --git a/internal/resources/ledgers/v3_test.go b/internal/resources/ledgers/v3_test.go index e49b00217..c420d6286 100644 --- a/internal/resources/ledgers/v3_test.go +++ b/internal/resources/ledgers/v3_test.go @@ -6,6 +6,7 @@ import ( "testing" authorizationv1 "k8s.io/api/authorization/v1" + corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -77,14 +78,25 @@ type ledgerV3AccessReviewClient struct { type ledgerV3CleanupClient struct { client.Client - getCalls int + getCalls int + deleteCalls int + secret *corev1.Secret } -func (c *ledgerV3CleanupClient) Get(_ context.Context, key client.ObjectKey, _ client.Object, _ ...client.GetOption) error { +func (c *ledgerV3CleanupClient) Get(_ context.Context, key client.ObjectKey, object client.Object, _ ...client.GetOption) error { c.getCalls++ + if secret, ok := object.(*corev1.Secret); ok && c.secret != nil { + c.secret.DeepCopyInto(secret) + return nil + } return apierrors.NewNotFound(schema.GroupResource{Group: ledgerV3ClusterGVK.Group, Resource: "clusters"}, key.Name) } +func (c *ledgerV3CleanupClient) Delete(_ context.Context, _ client.Object, _ ...client.DeleteOption) error { + c.deleteCalls++ + return nil +} + func (c ledgerV3AccessReviewClient) Create(_ context.Context, object client.Object, _ ...client.CreateOption) error { review := object.(*authorizationv1.SelfSubjectAccessReview) review.Status.Allowed = review.Spec.ResourceAttributes.Verb != c.deniedVerb @@ -142,14 +154,18 @@ func TestLedgerV3PreviewVersionIgnoredWhenClusterUnavailable(t *testing.T) { } } -func TestDeleteLedgerV3PreviewSkipsCertManagerResourcesWhenUnavailable(t *testing.T) { +func TestDeleteLedgerV3PreviewRemovesSecretWhenCertManagerIsUnavailable(t *testing.T) { previous := ledgerV3CertManagerAvailable ledgerV3CertManagerAvailable = false t.Cleanup(func() { ledgerV3CertManagerAvailable = previous }) - kubernetesClient := &ledgerV3CleanupClient{} + kubernetesClient := &ledgerV3CleanupClient{secret: &corev1.Secret{ObjectMeta: metav1.ObjectMeta{ + Namespace: "stack0", + Name: ledgerV3TLSName("stack0"), + Labels: map[string]string{ledgerV3PreviewLabel: "true"}, + }}} ctx := ledgerV3DiscoveryContext{ Context: context.Background(), client: kubernetesClient, @@ -158,8 +174,11 @@ func TestDeleteLedgerV3PreviewSkipsCertManagerResourcesWhenUnavailable(t *testin if err := deleteLedgerV3Preview(ctx, stack); err != nil { t.Fatalf("deleteLedgerV3Preview() returned error without cert-manager: %v", err) } - if kubernetesClient.getCalls != 1 { - t.Fatalf("deleteLedgerV3Preview() performed %d GETs, want only the Cluster lookup", kubernetesClient.getCalls) + if kubernetesClient.getCalls != 2 { + t.Fatalf("deleteLedgerV3Preview() performed %d GETs, want Cluster and Secret lookups", kubernetesClient.getCalls) + } + if kubernetesClient.deleteCalls != 1 { + t.Fatalf("deleteLedgerV3Preview() performed %d deletes, want the preview Secret deleted", kubernetesClient.deleteCalls) } } diff --git a/internal/resources/ledgers/v3_tls.go b/internal/resources/ledgers/v3_tls.go index 335198560..20dc42ef6 100644 --- a/internal/resources/ledgers/v3_tls.go +++ b/internal/resources/ledgers/v3_tls.go @@ -70,7 +70,7 @@ func createOrUpdateV3TLSResources(ctx core.Context, stack *v1beta1.Stack, ledger if err := controllerutil.SetControllerReference(ledger, certificate, ctx.GetScheme()); err != nil { return err } - certificate.Object["spec"] = ledgerV3CertificateSpec(stack.Name) + certificate.Object["spec"] = ledgerV3CertificateSpec(stack.Name, preview) return nil }); err != nil { return false, "", "", err @@ -134,17 +134,21 @@ func ledgerV3TLSName(stackName string) string { return stackName + "-ledger-v3-tls" } -func ledgerV3CertificateSpec(stackName string) map[string]any { +func ledgerV3CertificateSpec(stackName string, preview bool) map[string]any { serviceName := "ledger-" + stackName + secretLabels := map[string]any{ + v1beta1.GatewayBackendTLSSecretLabel: "true", + } + if preview { + secretLabels[ledgerV3PreviewLabel] = "true" + } return map[string]any{ "secretName": ledgerV3TLSName(stackName), "duration": ledgerV3TLSCertificateDuration, "renewBefore": ledgerV3TLSCertificateRenewBefore, "isCA": true, "secretTemplate": map[string]any{ - "labels": map[string]any{ - v1beta1.GatewayBackendTLSSecretLabel: "true", - }, + "labels": secretLabels, }, "issuerRef": map[string]any{ "name": ledgerV3IssuerName(stackName), diff --git a/internal/resources/stacks/networkpolicies.go b/internal/resources/stacks/networkpolicies.go index ad2fb3397..3fb385d75 100644 --- a/internal/resources/stacks/networkpolicies.go +++ b/internal/resources/stacks/networkpolicies.go @@ -98,9 +98,9 @@ func createNetworkPolicies(ctx Context, stack *v1beta1.Stack) error { return err } - // 4. allow-ledger-v3-cluster: allow Raft and service gRPC between direct - // Ledger v3 replicas. These labels are part of the Ledger Operator's stable - // selector and also work for clusters created before this policy. + // 4. allow-ledger-v3-cluster: allow traffic between direct Ledger v3 + // replicas. Ports are intentionally unrestricted within the cluster so + // LedgerConfiguration service port overrides cannot break Raft or gRPC. if _, _, err := CreateOrUpdate[*networkingv1.NetworkPolicy](ctx, types.NamespacedName{ Namespace: stack.Name, @@ -116,7 +116,6 @@ func createNetworkPolicies(ctx Context, stack *v1beta1.Stack) error { From: []networkingv1.NetworkPolicyPeer{ {PodSelector: &selector}, }, - Ports: networkPolicyTCPPorts(7777, 8888), }, }, } @@ -140,8 +139,7 @@ func createNetworkPolicies(ctx Context, stack *v1beta1.Stack) error { PodSelector: selector, PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeIngress}, Ingress: []networkingv1.NetworkPolicyIngressRule{{ - From: []networkingv1.NetworkPolicyPeer{{PodSelector: &selector}}, - Ports: networkPolicyTCPPorts(7777, 8888), + From: []networkingv1.NetworkPolicyPeer{{PodSelector: &selector}}, }}, } return nil diff --git a/internal/tests/ledger_v3_controller_test.go b/internal/tests/ledger_v3_controller_test.go index 9e974109a..0224322b3 100644 --- a/internal/tests/ledger_v3_controller_test.go +++ b/internal/tests/ledger_v3_controller_test.go @@ -209,6 +209,22 @@ var _ = Describe("Ledger v3 controller", func() { }) It("configures TLS before the managed certificate is ready", func() { + configuration := &v1beta1.LedgerConfiguration{ + ObjectMeta: metav1.ObjectMeta{Name: "ports-" + stack.Name}, + Spec: v1beta1.LedgerConfigurationSpec{ + Stacks: []string{stack.Name}, + Cluster: ledgerv1alpha1.ClusterSpec{Service: ledgerv1alpha1.ServiceSpec{ + HttpPort: 19000, + GrpcPort: 18888, + RaftPort: 17777, + }}, + }, + } + Expect(Create(configuration)).To(Succeed()) + DeferCleanup(func() { + Expect(client.IgnoreNotFound(Delete(configuration))).To(Succeed()) + }) + issuer := newLedgerV3Issuer() certificate := newLedgerV3Certificate() cluster := newLedgerV3Cluster() @@ -340,7 +356,7 @@ var _ = Describe("Ledger v3 controller", func() { return httpAPI.Spec.Rules[0].BackendRef }).Should(Equal(&v1beta1.GatewayBackendRef{ Name: "ledger-" + stack.Name, - Port: 9000, + Port: 19000, })) grpcAPI := &v1beta1.GatewayGRPCAPI{} Eventually(func(g Gomega) *v1beta1.GatewayBackendRef { @@ -350,13 +366,23 @@ var _ = Describe("Ledger v3 controller", func() { return grpcAPI.Spec.BackendRef }).Should(Equal(&v1beta1.GatewayBackendRef{ Name: "ledger-" + stack.Name, - Port: 8888, + Port: 18888, TLS: &v1beta1.GatewayBackendTLS{ SecretName: tlsName, CASecretKey: "ca.crt", ServerName: "ledger-" + stack.Name + "." + stack.Name + ".svc.cluster.local", }, })) + + certificate.Object["status"] = map[string]any{ + "conditions": []any{ + map[string]any{"type": "Ready", "status": "False", "reason": "SecretMissing"}, + }, + } + Expect(TestContext().GetClient().Status().Update(TestContext(), certificate)).To(Succeed()) + Eventually(func() error { + return Get(core.GetResourceName(core.GetObjectName(stack.Name, "ledger")), &v1beta1.GatewayGRPCAPI{}) + }).Should(BeNotFound()) }) It("configures authentication and monitoring from the existing stack settings", func() { @@ -575,17 +601,16 @@ var _ = Describe("Ledger v3 controller", func() { return Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name}, cluster) }).Should(Succeed()) - Eventually(func(g Gomega) error { + Eventually(func(g Gomega) bool { g.Expect(Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name}, cluster)).To(Succeed()) cluster.Object["status"] = map[string]any{ "phase": "Running", "readyReplicas": int64(3), "observedGeneration": cluster.GetGeneration(), } - return TestContext().GetClient().Status().Update(TestContext(), cluster) - }).Should(Succeed()) - - Eventually(func(g Gomega) bool { + if err := TestContext().GetClient().Status().Update(TestContext(), cluster); err != nil { + return false + } g.Expect(LoadResource("", ledger.Name, ledger)).To(Succeed()) return ledger.Status.Ready }, time.Minute).Should(BeTrue()) @@ -780,16 +805,6 @@ var _ = Describe("Ledger v3 controller", func() { return Get(core.GetResourceName(core.GetObjectName(stack.Name, "ledger")), &v1beta1.Database{}) }).Should(Succeed()) - httpAPI := &v1beta1.GatewayHTTPAPI{} - Eventually(func(g Gomega) []v1beta1.GatewayHTTPAPIRule { - g.Expect(Get(core.GetResourceName(core.GetObjectName(stack.Name, "ledger")), httpAPI)).To(Succeed()) - return httpAPI.Spec.Rules - }).Should(ContainElement(SatisfyAll( - HaveField("Path", "/v3"), - HaveField("BackendRef.Name", "ledger-"+stack.Name), - HaveField("BackendRef.Port", int32(9000)), - ))) - certificate := newLedgerV3Certificate() tlsName := stack.Name + "-ledger-v3-tls" Eventually(func(g Gomega) error { @@ -817,6 +832,16 @@ var _ = Describe("Ledger v3 controller", func() { } Expect(TestContext().GetClient().Status().Update(TestContext(), certificate)).To(Succeed()) + httpAPI := &v1beta1.GatewayHTTPAPI{} + Eventually(func(g Gomega) []v1beta1.GatewayHTTPAPIRule { + g.Expect(Get(core.GetResourceName(core.GetObjectName(stack.Name, "ledger")), httpAPI)).To(Succeed()) + return httpAPI.Spec.Rules + }).Should(ContainElement(SatisfyAll( + HaveField("Path", "/v3"), + HaveField("BackendRef.Name", "ledger-"+stack.Name), + HaveField("BackendRef.Port", int32(9000)), + ))) + grpcAPI := &v1beta1.GatewayGRPCAPI{} Eventually(func(g Gomega) *v1beta1.GatewayBackendRef { g.Expect(Get(core.GetResourceName(core.GetObjectName(stack.Name, "ledger")), grpcAPI)).To(Succeed()) diff --git a/internal/tests/networkpolicy_controller_test.go b/internal/tests/networkpolicy_controller_test.go index a4597a9f3..e9413adfe 100644 --- a/internal/tests/networkpolicy_controller_test.go +++ b/internal/tests/networkpolicy_controller_test.go @@ -81,7 +81,7 @@ var _ = Describe("NetworkPolicyController", func() { g.Expect(np.Spec.Ingress[0].From[0].PodSelector.MatchLabels).To(HaveKeyWithValue("app.kubernetes.io/name", "gateway")) }).Should(Succeed()) - // Direct Ledger v3 replicas can communicate over Raft and service gRPC. + // Direct Ledger v3 replicas can communicate on configured service ports. Eventually(func(g Gomega) { np := &networkingv1.NetworkPolicy{} g.Expect(LoadResource(stack.Name, "allow-ledger-v3-cluster", np)).To(Succeed()) @@ -94,7 +94,7 @@ var _ = Describe("NetworkPolicyController", func() { g.Expect(np.Spec.Ingress[0].From).To(HaveLen(1)) g.Expect(np.Spec.Ingress[0].From[0].NamespaceSelector).To(BeNil()) g.Expect(np.Spec.Ingress[0].From[0].PodSelector.MatchLabels).To(Equal(np.Spec.PodSelector.MatchLabels)) - g.Expect(np.Spec.Ingress[0].Ports).To(HaveLen(2)) + g.Expect(np.Spec.Ingress[0].Ports).To(BeEmpty()) }).Should(Succeed()) // Existing preview clusters keep their historical immutable selector. @@ -103,7 +103,10 @@ var _ = Describe("NetworkPolicyController", func() { g.Expect(LoadResource(stack.Name, "allow-ledger-v3-preview-cluster", np)).To(Succeed()) g.Expect(np).To(BeControlledBy(stack)) g.Expect(np.Spec.PodSelector.MatchLabels).To(HaveKeyWithValue("formance.com/ledger-v3-preview", "true")) + g.Expect(np.Spec.Ingress).To(HaveLen(1)) + g.Expect(np.Spec.Ingress[0].From).To(HaveLen(1)) g.Expect(np.Spec.Ingress[0].From[0].PodSelector.MatchLabels).To(Equal(np.Spec.PodSelector.MatchLabels)) + g.Expect(np.Spec.Ingress[0].Ports).To(BeEmpty()) }).Should(Succeed()) // Ledger v3 can reach only the legacy Ledger from the same namespace. From 91328c78cd5dbc42c81efe98f8139beeb7ec2fc6 Mon Sep 17 00:00:00 2001 From: Maxence Maireaux Date: Wed, 22 Jul 2026 13:46:24 +0200 Subject: [PATCH 23/30] test(gateway): require a renewed backend TLS hash --- internal/tests/gateway_controller_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/tests/gateway_controller_test.go b/internal/tests/gateway_controller_test.go index 7aacda66a..82437a137 100644 --- a/internal/tests/gateway_controller_test.go +++ b/internal/tests/gateway_controller_test.go @@ -418,7 +418,7 @@ var _ = Describe("GatewayController", func() { Eventually(func(g Gomega) string { g.Expect(LoadResource(stack.Name, "gateway", deployment)).To(Succeed()) return deployment.Spec.Template.Annotations["formance.com/backend-tls-secrets-hash"] - }).ShouldNot(Equal(initialHash)) + }).Should(SatisfyAll(Not(BeEmpty()), Not(Equal(initialHash)))) }) }) Context("With a consumer on gateway", func() { From b8fefe14f3069123fd609bd5af47da749823213d Mon Sep 17 00:00:00 2001 From: Maxence Maireaux Date: Wed, 22 Jul 2026 13:52:53 +0200 Subject: [PATCH 24/30] fix(ledger): reject mixed wildcard configuration selectors --- api/formance.com/v1beta1/ledgerconfiguration_types.go | 1 + config/crd/bases/formance.com_ledgerconfigurations.yaml | 4 ++++ ...ourcedefinition_ledgerconfigurations.formance.com.yaml | 4 ++++ internal/tests/ledger_v3_controller_test.go | 8 ++++++++ 4 files changed, 17 insertions(+) diff --git a/api/formance.com/v1beta1/ledgerconfiguration_types.go b/api/formance.com/v1beta1/ledgerconfiguration_types.go index 80ca33841..9b4f8c2d0 100644 --- a/api/formance.com/v1beta1/ledgerconfiguration_types.go +++ b/api/formance.com/v1beta1/ledgerconfiguration_types.go @@ -28,6 +28,7 @@ type LedgerConfigurationSpec struct { // Stacks on which the configuration is applied. Can contain `*` to // indicate a wildcard, following the same convention as Settings. // +optional + // +kubebuilder:validation:XValidation:rule="size(self) == 1 || !self.exists(stack, stack == '*')",message="the wildcard stack selector '*' cannot be combined with explicit stack names" Stacks []string `json:"stacks,omitempty"` // Cluster is the base Ledger v3 Cluster specification. Stack-specific diff --git a/config/crd/bases/formance.com_ledgerconfigurations.yaml b/config/crd/bases/formance.com_ledgerconfigurations.yaml index 77f3d45f0..c4e87596b 100644 --- a/config/crd/bases/formance.com_ledgerconfigurations.yaml +++ b/config/crd/bases/formance.com_ledgerconfigurations.yaml @@ -4039,6 +4039,10 @@ spec: items: type: string type: array + x-kubernetes-validations: + - message: the wildcard stack selector '*' cannot be combined with + explicit stack names + rule: size(self) == 1 || !self.exists(stack, stack == '*') type: object type: object served: true diff --git a/helm/crds/templates/crds/apiextensions.k8s.io_v1_customresourcedefinition_ledgerconfigurations.formance.com.yaml b/helm/crds/templates/crds/apiextensions.k8s.io_v1_customresourcedefinition_ledgerconfigurations.formance.com.yaml index a7f603b2a..e9c22a7d6 100644 --- a/helm/crds/templates/crds/apiextensions.k8s.io_v1_customresourcedefinition_ledgerconfigurations.formance.com.yaml +++ b/helm/crds/templates/crds/apiextensions.k8s.io_v1_customresourcedefinition_ledgerconfigurations.formance.com.yaml @@ -4042,6 +4042,10 @@ spec: items: type: string type: array + x-kubernetes-validations: + - message: the wildcard stack selector '*' cannot be combined with + explicit stack names + rule: size(self) == 1 || !self.exists(stack, stack == '*') type: object type: object served: true diff --git a/internal/tests/ledger_v3_controller_test.go b/internal/tests/ledger_v3_controller_test.go index 0224322b3..3dfb2c599 100644 --- a/internal/tests/ledger_v3_controller_test.go +++ b/internal/tests/ledger_v3_controller_test.go @@ -944,6 +944,14 @@ var _ = Describe("LedgerConfiguration", Serial, func() { }, } Expect(apierrors.IsInvalid(Create(invalidCEL))).To(BeTrue()) + + mixedWildcard := &v1beta1.LedgerConfiguration{ + ObjectMeta: metav1.ObjectMeta{Name: "invalid-mixed-wildcard"}, + Spec: v1beta1.LedgerConfigurationSpec{ + Stacks: []string{"*", "explicit-stack"}, + }, + } + Expect(apierrors.IsInvalid(Create(mixedWildcard))).To(BeTrue()) }) It("uses the default configuration as a live base for Ledger v3 Clusters", func() { From 2a77ffeb713f7d39c58a9bdb95b587e27eb25218 Mon Sep 17 00:00:00 2001 From: Maxence Maireaux Date: Wed, 22 Jul 2026 13:59:17 +0200 Subject: [PATCH 25/30] docs(gateway): document backend TLS rotation label --- api/formance.com/v1beta1/gatewaybackend_types.go | 2 ++ config/crd/bases/formance.com_gatewaygrpcapis.yaml | 6 ++++-- config/crd/bases/formance.com_gatewayhttpapis.yaml | 6 ++++-- .../02-Custom Resource Definitions.md | 2 +- ...stomresourcedefinition_gatewaygrpcapis.formance.com.yaml | 6 ++++-- ...stomresourcedefinition_gatewayhttpapis.formance.com.yaml | 6 ++++-- 6 files changed, 19 insertions(+), 9 deletions(-) diff --git a/api/formance.com/v1beta1/gatewaybackend_types.go b/api/formance.com/v1beta1/gatewaybackend_types.go index ac56fb124..c270817e2 100644 --- a/api/formance.com/v1beta1/gatewaybackend_types.go +++ b/api/formance.com/v1beta1/gatewaybackend_types.go @@ -21,6 +21,8 @@ const GatewayBackendTLSSecretLabel = "formance.com/gateway-backend-tls" // GatewayBackendTLS configures TLS when Gateway connects to a backend. type GatewayBackendTLS struct { // SecretName contains the CA used to verify the backend certificate. + // The Secret must carry the `formance.com/gateway-backend-tls: "true"` + // label so that certificate rotations trigger a Gateway rollout. // +kubebuilder:validation:MinLength=1 SecretName string `json:"secretName"` // CASecretKey is the key containing the CA certificate. diff --git a/config/crd/bases/formance.com_gatewaygrpcapis.yaml b/config/crd/bases/formance.com_gatewaygrpcapis.yaml index bd28b5384..5f02d2c5c 100644 --- a/config/crd/bases/formance.com_gatewaygrpcapis.yaml +++ b/config/crd/bases/formance.com_gatewaygrpcapis.yaml @@ -68,8 +68,10 @@ spec: description: CASecretKey is the key containing the CA certificate. type: string secretName: - description: SecretName contains the CA used to verify the - backend certificate. + description: |- + SecretName contains the CA used to verify the backend certificate. + The Secret must carry the `formance.com/gateway-backend-tls: "true"` + label so that certificate rotations trigger a Gateway rollout. minLength: 1 type: string serverName: diff --git a/config/crd/bases/formance.com_gatewayhttpapis.yaml b/config/crd/bases/formance.com_gatewayhttpapis.yaml index e6a2d49c1..c4f7a5963 100644 --- a/config/crd/bases/formance.com_gatewayhttpapis.yaml +++ b/config/crd/bases/formance.com_gatewayhttpapis.yaml @@ -82,8 +82,10 @@ spec: certificate. type: string secretName: - description: SecretName contains the CA used to verify - the backend certificate. + description: |- + SecretName contains the CA used to verify the backend certificate. + The Secret must carry the `formance.com/gateway-backend-tls: "true"` + label so that certificate rotations trigger a Gateway rollout. minLength: 1 type: string serverName: diff --git a/docs/09-Configuration reference/02-Custom Resource Definitions.md b/docs/09-Configuration reference/02-Custom Resource Definitions.md index 8fdb089c2..8429bdf00 100644 --- a/docs/09-Configuration reference/02-Custom Resource Definitions.md +++ b/docs/09-Configuration reference/02-Custom Resource Definitions.md @@ -2352,7 +2352,7 @@ GatewayBackendTLS configures TLS when Gateway connects to a backend. | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `secretName` _string_ | SecretName contains the CA used to verify the backend certificate. | | MinLength: 1
| +| `secretName` _string_ | SecretName contains the CA used to verify the backend certificate.
The Secret must carry the `formance.com/gateway-backend-tls: "true"`
label so that certificate rotations trigger a Gateway rollout. | | MinLength: 1
| | `caSecretKey` _string_ | CASecretKey is the key containing the CA certificate. | ca.crt | | | `serverName` _string_ | ServerName is used for backend certificate verification. | | MinLength: 1
| diff --git a/helm/crds/templates/crds/apiextensions.k8s.io_v1_customresourcedefinition_gatewaygrpcapis.formance.com.yaml b/helm/crds/templates/crds/apiextensions.k8s.io_v1_customresourcedefinition_gatewaygrpcapis.formance.com.yaml index dceaebccf..3ed754f29 100644 --- a/helm/crds/templates/crds/apiextensions.k8s.io_v1_customresourcedefinition_gatewaygrpcapis.formance.com.yaml +++ b/helm/crds/templates/crds/apiextensions.k8s.io_v1_customresourcedefinition_gatewaygrpcapis.formance.com.yaml @@ -71,8 +71,10 @@ spec: description: CASecretKey is the key containing the CA certificate. type: string secretName: - description: SecretName contains the CA used to verify the - backend certificate. + description: |- + SecretName contains the CA used to verify the backend certificate. + The Secret must carry the `formance.com/gateway-backend-tls: "true"` + label so that certificate rotations trigger a Gateway rollout. minLength: 1 type: string serverName: diff --git a/helm/crds/templates/crds/apiextensions.k8s.io_v1_customresourcedefinition_gatewayhttpapis.formance.com.yaml b/helm/crds/templates/crds/apiextensions.k8s.io_v1_customresourcedefinition_gatewayhttpapis.formance.com.yaml index 062619ccf..fff2d921e 100644 --- a/helm/crds/templates/crds/apiextensions.k8s.io_v1_customresourcedefinition_gatewayhttpapis.formance.com.yaml +++ b/helm/crds/templates/crds/apiextensions.k8s.io_v1_customresourcedefinition_gatewayhttpapis.formance.com.yaml @@ -85,8 +85,10 @@ spec: certificate. type: string secretName: - description: SecretName contains the CA used to verify - the backend certificate. + description: |- + SecretName contains the CA used to verify the backend certificate. + The Secret must carry the `formance.com/gateway-backend-tls: "true"` + label so that certificate rotations trigger a Gateway rollout. minLength: 1 type: string serverName: From 34dfe7a356b4c6440f4ca68fe851727803c8d5bf Mon Sep 17 00:00:00 2001 From: Maxence Maireaux Date: Wed, 22 Jul 2026 14:08:54 +0200 Subject: [PATCH 26/30] fix(ledger): guard v3 startup against migration jobs --- .../settings.catalog.json | 10 ++-- internal/resources/ledgers/v3.go | 20 ++++++- internal/resources/ledgers/v3_test.go | 54 +++++++++++++++++++ internal/resources/ledgers/v3_tls.go | 2 +- internal/tests/ledger_v3_controller_test.go | 6 +-- 5 files changed, 81 insertions(+), 11 deletions(-) diff --git a/docs/09-Configuration reference/settings.catalog.json b/docs/09-Configuration reference/settings.catalog.json index 0bc89d032..93a04bed5 100644 --- a/docs/09-Configuration reference/settings.catalog.json +++ b/docs/09-Configuration reference/settings.catalog.json @@ -201,21 +201,21 @@ "key": "deployments.ledger.containers.ledger.resource-requirements.claims", "valueType": "string[]", "sources": [ - "internal/resources/ledgers/v3.go:346" + "internal/resources/ledgers/v3.go:361" ] }, { "key": "deployments.ledger.containers.ledger.resource-requirements.limits", "valueType": "map[string]string", "sources": [ - "internal/resources/ledgers/v3.go:346" + "internal/resources/ledgers/v3.go:361" ] }, { "key": "deployments.ledger.containers.ledger.resource-requirements.requests", "valueType": "map[string]string", "sources": [ - "internal/resources/ledgers/v3.go:346" + "internal/resources/ledgers/v3.go:361" ] }, { @@ -223,14 +223,14 @@ "valueType": "int32", "default": "3", "sources": [ - "internal/resources/ledgers/v3.go:332" + "internal/resources/ledgers/v3.go:347" ] }, { "key": "deployments.ledger.topology-spread-constraints", "valueType": "bool", "sources": [ - "internal/resources/ledgers/v3.go:364" + "internal/resources/ledgers/v3.go:379" ] }, { diff --git a/internal/resources/ledgers/v3.go b/internal/resources/ledgers/v3.go index 86f117678..ef157cd69 100644 --- a/internal/resources/ledgers/v3.go +++ b/internal/resources/ledgers/v3.go @@ -8,6 +8,7 @@ import ( "golang.org/x/mod/semver" appsv1 "k8s.io/api/apps/v1" authorizationv1 "k8s.io/api/authorization/v1" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -197,7 +198,7 @@ func reconcileV3(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, clearLegacyLedgerConditions(ledger) - hasLegacyResources, err := legacyLedgerResourcesExist(ctx, stack) + hasLegacyResources, err := legacyLedgerResourcesExist(ctx, stack, ledger) if err != nil { return err } @@ -276,7 +277,7 @@ func setLedgerV3Condition(ledger *v1beta1.Ledger, status metav1.ConditionStatus, }, v1beta1.ConditionTypeMatch(ledgerV3ClusterReadyCondition)) } -func legacyLedgerResourcesExist(ctx core.Context, stack *v1beta1.Stack) (bool, error) { +func legacyLedgerResourcesExist(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger) (bool, error) { for _, name := range []string{"ledger", "ledger-worker", "ledger-write", "ledger-read", "ledger-gateway"} { deployment := &appsv1.Deployment{} err := ctx.GetClient().Get(ctx, types.NamespacedName{Namespace: stack.Name, Name: name}, deployment) @@ -296,6 +297,21 @@ func legacyLedgerResourcesExist(ctx core.Context, stack *v1beta1.Stack) (bool, e if !apierrors.IsNotFound(err) { return false, err } + + jobs := &batchv1.JobList{} + if err := ctx.GetClient().List(ctx, jobs, client.InNamespace(stack.Name)); err != nil { + return false, err + } + databaseName := core.GetObjectName(stack.Name, "ledger") + for _, job := range jobs.Items { + for _, owner := range job.OwnerReferences { + if owner.APIVersion == v1beta1.GroupVersion.String() && + ((owner.Kind == "Ledger" && owner.Name == ledger.Name) || + (owner.Kind == "Database" && owner.Name == databaseName)) { + return true, nil + } + } + } return false, nil } diff --git a/internal/resources/ledgers/v3_test.go b/internal/resources/ledgers/v3_test.go index c420d6286..409cd17a1 100644 --- a/internal/resources/ledgers/v3_test.go +++ b/internal/resources/ledgers/v3_test.go @@ -5,7 +5,9 @@ import ( "errors" "testing" + appsv1 "k8s.io/api/apps/v1" authorizationv1 "k8s.io/api/authorization/v1" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -14,6 +16,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" "github.com/formancehq/operator/v3/api/formance.com/v1beta1" "github.com/formancehq/operator/v3/internal/core" @@ -225,6 +228,57 @@ func TestLedgerV3MissingWatchPermissionDisablesCapabilityWithoutFailing(t *testi } } +func TestLegacyLedgerResourcesExistIncludesMigrationJobs(t *testing.T) { + t.Parallel() + + scheme := runtime.NewScheme() + if err := appsv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + if err := batchv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + if err := v1beta1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + + stack := &v1beta1.Stack{ObjectMeta: metav1.ObjectMeta{Name: "stack0"}} + ledger := &v1beta1.Ledger{ObjectMeta: metav1.ObjectMeta{Name: "ledger0"}} + tests := []struct { + name string + owner metav1.OwnerReference + }{ + { + name: "Ledger migration Job", + owner: metav1.OwnerReference{APIVersion: v1beta1.GroupVersion.String(), Kind: "Ledger", Name: ledger.Name}, + }, + { + name: "Database migration Job", + owner: metav1.OwnerReference{APIVersion: v1beta1.GroupVersion.String(), Kind: "Database", Name: core.GetObjectName(stack.Name, "ledger")}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + job := &batchv1.Job{ObjectMeta: metav1.ObjectMeta{ + Namespace: stack.Name, + Name: "migration", + OwnerReferences: []metav1.OwnerReference{tt.owner}, + }} + kubernetesClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(job).Build() + ctx := ledgerV3DiscoveryContext{Context: context.Background(), client: kubernetesClient} + + exists, err := legacyLedgerResourcesExist(ctx, stack, ledger) + if err != nil { + t.Fatalf("legacyLedgerResourcesExist() returned error: %v", err) + } + if !exists { + t.Fatal("legacyLedgerResourcesExist() ignored a legacy migration Job") + } + }) + } +} + func TestIsLedgerV3(t *testing.T) { t.Parallel() diff --git a/internal/resources/ledgers/v3_tls.go b/internal/resources/ledgers/v3_tls.go index 20dc42ef6..588bfaf02 100644 --- a/internal/resources/ledgers/v3_tls.go +++ b/internal/resources/ledgers/v3_tls.go @@ -118,7 +118,7 @@ func setLedgerV3TLSResourceMetadata(resource *unstructured.Unstructured, stackNa } func ledgerV3IssuerName(stackName string) string { - return stackName + "-selfsigned" + return stackName + "-ledger-v3-selfsigned" } func ledgerV3TLSName(stackName string) string { diff --git a/internal/tests/ledger_v3_controller_test.go b/internal/tests/ledger_v3_controller_test.go index 3dfb2c599..5bdd94992 100644 --- a/internal/tests/ledger_v3_controller_test.go +++ b/internal/tests/ledger_v3_controller_test.go @@ -231,7 +231,7 @@ var _ = Describe("Ledger v3 controller", func() { tlsName := stack.Name + "-ledger-v3-tls" Eventually(func(g Gomega) { - g.Expect(Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name + "-selfsigned"}, issuer)).To(Succeed()) + g.Expect(Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name + "-ledger-v3-selfsigned"}, issuer)).To(Succeed()) g.Expect(issuer).To(BeControlledBy(ledger)) selfSigned, found, err := unstructured.NestedMap(issuer.Object, "spec", "selfSigned") g.Expect(err).NotTo(HaveOccurred()) @@ -888,7 +888,7 @@ var _ = Describe("Ledger v3 controller", func() { issuer := newLedgerV3Issuer() Eventually(func(g Gomega) { g.Expect(Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name + "-ledger-v3-tls"}, certificate)).To(Succeed()) - g.Expect(Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name + "-selfsigned"}, issuer)).To(Succeed()) + g.Expect(Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name + "-ledger-v3-selfsigned"}, issuer)).To(Succeed()) }).Should(Succeed()) Expect(Delete(previewSettings)).To(Succeed()) @@ -900,7 +900,7 @@ var _ = Describe("Ledger v3 controller", func() { return Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name + "-ledger-v3-tls"}, certificate) }).Should(BeNotFound()) Eventually(func() error { - return Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name + "-selfsigned"}, issuer) + return Get(types.NamespacedName{Namespace: stack.Name, Name: stack.Name + "-ledger-v3-selfsigned"}, issuer) }).Should(BeNotFound()) Eventually(func() error { From c202902df5310b742fc377a96e393e321504eb65 Mon Sep 17 00:00:00 2001 From: Maxence Maireaux Date: Wed, 22 Jul 2026 14:13:03 +0200 Subject: [PATCH 27/30] chore(docs): refresh settings catalog --- docs/09-Configuration reference/settings.catalog.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/09-Configuration reference/settings.catalog.json b/docs/09-Configuration reference/settings.catalog.json index 93a04bed5..6215ee286 100644 --- a/docs/09-Configuration reference/settings.catalog.json +++ b/docs/09-Configuration reference/settings.catalog.json @@ -201,21 +201,21 @@ "key": "deployments.ledger.containers.ledger.resource-requirements.claims", "valueType": "string[]", "sources": [ - "internal/resources/ledgers/v3.go:361" + "internal/resources/ledgers/v3.go:362" ] }, { "key": "deployments.ledger.containers.ledger.resource-requirements.limits", "valueType": "map[string]string", "sources": [ - "internal/resources/ledgers/v3.go:361" + "internal/resources/ledgers/v3.go:362" ] }, { "key": "deployments.ledger.containers.ledger.resource-requirements.requests", "valueType": "map[string]string", "sources": [ - "internal/resources/ledgers/v3.go:361" + "internal/resources/ledgers/v3.go:362" ] }, { @@ -223,14 +223,14 @@ "valueType": "int32", "default": "3", "sources": [ - "internal/resources/ledgers/v3.go:347" + "internal/resources/ledgers/v3.go:348" ] }, { "key": "deployments.ledger.topology-spread-constraints", "valueType": "bool", "sources": [ - "internal/resources/ledgers/v3.go:379" + "internal/resources/ledgers/v3.go:380" ] }, { From f76c581f055fdc7499d178d062c2a4e2f2369ccd Mon Sep 17 00:00:00 2001 From: Maxence Maireaux Date: Wed, 22 Jul 2026 14:21:19 +0200 Subject: [PATCH 28/30] fix(ledger): withdraw routes when v3 TLS is unready --- .../settings.catalog.json | 10 +++++----- internal/resources/ledgers/v3.go | 3 +++ internal/resources/ledgers/v3_preview.go | 7 +++++++ internal/tests/ledger_v3_controller_test.go | 20 +++++++++++++++++++ 4 files changed, 35 insertions(+), 5 deletions(-) diff --git a/docs/09-Configuration reference/settings.catalog.json b/docs/09-Configuration reference/settings.catalog.json index 6215ee286..acbaeed88 100644 --- a/docs/09-Configuration reference/settings.catalog.json +++ b/docs/09-Configuration reference/settings.catalog.json @@ -201,21 +201,21 @@ "key": "deployments.ledger.containers.ledger.resource-requirements.claims", "valueType": "string[]", "sources": [ - "internal/resources/ledgers/v3.go:362" + "internal/resources/ledgers/v3.go:365" ] }, { "key": "deployments.ledger.containers.ledger.resource-requirements.limits", "valueType": "map[string]string", "sources": [ - "internal/resources/ledgers/v3.go:362" + "internal/resources/ledgers/v3.go:365" ] }, { "key": "deployments.ledger.containers.ledger.resource-requirements.requests", "valueType": "map[string]string", "sources": [ - "internal/resources/ledgers/v3.go:362" + "internal/resources/ledgers/v3.go:365" ] }, { @@ -223,14 +223,14 @@ "valueType": "int32", "default": "3", "sources": [ - "internal/resources/ledgers/v3.go:348" + "internal/resources/ledgers/v3.go:351" ] }, { "key": "deployments.ledger.topology-spread-constraints", "valueType": "bool", "sources": [ - "internal/resources/ledgers/v3.go:380" + "internal/resources/ledgers/v3.go:383" ] }, { diff --git a/internal/resources/ledgers/v3.go b/internal/resources/ledgers/v3.go index ef157cd69..4ba227d9b 100644 --- a/internal/resources/ledgers/v3.go +++ b/internal/resources/ledgers/v3.go @@ -222,6 +222,9 @@ func reconcileV3(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, if err := core.DeleteIfExists[*v1beta1.GatewayGRPCAPI](ctx, core.GetResourceName(core.GetObjectName(stack.Name, "ledger"))); err != nil { return err } + if err := core.DeleteIfExists[*v1beta1.GatewayHTTPAPI](ctx, core.GetResourceName(core.GetObjectName(stack.Name, "ledger"))); err != nil { + return err + } setLedgerV3Condition(ledger, metav1.ConditionFalse, "TLSCertificatePending", tlsMessage) return core.NewPendingError().WithMessage("Ledger v3 TLS is not ready: %s", tlsMessage) } diff --git a/internal/resources/ledgers/v3_preview.go b/internal/resources/ledgers/v3_preview.go index 0df08df0d..125773156 100644 --- a/internal/resources/ledgers/v3_preview.go +++ b/internal/resources/ledgers/v3_preview.go @@ -71,6 +71,13 @@ func reconcileV3Preview(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1. if err := core.DeleteIfExists[*v1beta1.GatewayGRPCAPI](ctx, core.GetResourceName(core.GetObjectName(stack.Name, "ledger"))); err != nil { return err } + if err := gatewayhttpapis.Create(ctx, ledger, + gatewayhttpapis.WithHealthCheckEndpoint("_healthcheck"), + gatewayhttpapis.WithRules(gatewayhttpapis.RuleSecured()), + ); err != nil { + setLedgerV3PreviewCondition(ledger, metav1.ConditionFalse, "GatewayReconcileFailed", err.Error()) + return err + } setLedgerV3PreviewCondition(ledger, metav1.ConditionFalse, "TLSCertificatePending", tlsMessage) return core.NewPendingError().WithMessage("Ledger v3 preview TLS is not ready: %s", tlsMessage) } diff --git a/internal/tests/ledger_v3_controller_test.go b/internal/tests/ledger_v3_controller_test.go index 5bdd94992..3f0436891 100644 --- a/internal/tests/ledger_v3_controller_test.go +++ b/internal/tests/ledger_v3_controller_test.go @@ -383,6 +383,9 @@ var _ = Describe("Ledger v3 controller", func() { Eventually(func() error { return Get(core.GetResourceName(core.GetObjectName(stack.Name, "ledger")), &v1beta1.GatewayGRPCAPI{}) }).Should(BeNotFound()) + Eventually(func() error { + return Get(core.GetResourceName(core.GetObjectName(stack.Name, "ledger")), &v1beta1.GatewayHTTPAPI{}) + }).Should(BeNotFound()) }) It("configures authentication and monitoring from the existing stack settings", func() { @@ -856,6 +859,23 @@ var _ = Describe("Ledger v3 controller", func() { ServerName: "ledger-" + stack.Name + "." + stack.Name + ".svc.cluster.local", }, })) + + certificate.Object["status"] = map[string]any{ + "conditions": []any{ + map[string]any{"type": "Ready", "status": "False", "reason": "SecretMissing"}, + }, + } + Expect(TestContext().GetClient().Status().Update(TestContext(), certificate)).To(Succeed()) + Eventually(func(g Gomega) []v1beta1.GatewayHTTPAPIRule { + g.Expect(Get(core.GetResourceName(core.GetObjectName(stack.Name, "ledger")), httpAPI)).To(Succeed()) + return httpAPI.Spec.Rules + }).Should(SatisfyAll( + HaveLen(1), + ContainElement(SatisfyAll(HaveField("Path", ""), HaveField("BackendRef", BeNil()))), + )) + Eventually(func() error { + return Get(core.GetResourceName(core.GetObjectName(stack.Name, "ledger")), &v1beta1.GatewayGRPCAPI{}) + }).Should(BeNotFound()) }) It("does not allow the preview Cluster to bypass the migration guard", func() { From 8a314033c88abf0adbd8808f4ac1088d4061906d Mon Sep 17 00:00:00 2001 From: Maxence Maireaux Date: Wed, 22 Jul 2026 14:36:22 +0200 Subject: [PATCH 29/30] test(networkpolicy): accept dynamic ledger v3 ports --- .../02-stack-lifecycle/asserts/networkpolicies.yaml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/tests/e2e/chainsaw/02-stack-lifecycle/asserts/networkpolicies.yaml b/tests/e2e/chainsaw/02-stack-lifecycle/asserts/networkpolicies.yaml index 57c399a00..3a079b543 100644 --- a/tests/e2e/chainsaw/02-stack-lifecycle/asserts/networkpolicies.yaml +++ b/tests/e2e/chainsaw/02-stack-lifecycle/asserts/networkpolicies.yaml @@ -65,11 +65,6 @@ spec: matchLabels: app.kubernetes.io/instance: chainsaw-networkpolicies app.kubernetes.io/name: ledger - ports: - - protocol: TCP - port: 7777 - - protocol: TCP - port: 8888 policyTypes: - Ingress --- @@ -87,11 +82,6 @@ spec: - podSelector: matchLabels: formance.com/ledger-v3-preview: "true" - ports: - - protocol: TCP - port: 7777 - - protocol: TCP - port: 8888 policyTypes: - Ingress --- From 9abe4294db38ff72768a04c9ba4367e1aa5280f0 Mon Sep 17 00:00:00 2001 From: Maxence Maireaux Date: Wed, 22 Jul 2026 14:47:48 +0200 Subject: [PATCH 30/30] fix(gateway): reject HTTP backend service collisions --- api/formance.com/v1beta1/gatewaybackend_types.go | 1 + api/formance.com/v1beta1/gatewayhttpapi_types.go | 3 +++ .../crd/bases/formance.com_gatewaygrpcapis.yaml | 1 + .../crd/bases/formance.com_gatewayhttpapis.yaml | 8 ++++++++ .../02-Custom Resource Definitions.md | 6 +++--- ...cedefinition_gatewaygrpcapis.formance.com.yaml | 1 + ...cedefinition_gatewayhttpapis.formance.com.yaml | 8 ++++++++ internal/tests/gatewayhttpapi_controller_test.go | 15 +++++++++++++++ 8 files changed, 40 insertions(+), 3 deletions(-) diff --git a/api/formance.com/v1beta1/gatewaybackend_types.go b/api/formance.com/v1beta1/gatewaybackend_types.go index c270817e2..be9323813 100644 --- a/api/formance.com/v1beta1/gatewaybackend_types.go +++ b/api/formance.com/v1beta1/gatewaybackend_types.go @@ -39,6 +39,7 @@ type GatewayBackendTLS struct { type GatewayBackendRef struct { // Name is the backend Service name in the Stack namespace. // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=253 Name string `json:"name"` // Port is the backend Service port. // +kubebuilder:validation:Minimum=1 diff --git a/api/formance.com/v1beta1/gatewayhttpapi_types.go b/api/formance.com/v1beta1/gatewayhttpapi_types.go index 1f9a44851..0a7892581 100644 --- a/api/formance.com/v1beta1/gatewayhttpapi_types.go +++ b/api/formance.com/v1beta1/gatewayhttpapi_types.go @@ -32,11 +32,14 @@ type GatewayHTTPAPIRule struct { BackendRef *GatewayBackendRef `json:"backendRef,omitempty"` } +// +kubebuilder:validation:XValidation:rule="self.rules.all(rule, !has(rule.backendRef) || rule.backendRef.name != self.name)",message="backendRef.name must differ from spec.name because the latter is managed by the GatewayHTTPAPI controller" type GatewayHTTPAPISpec struct { StackDependency `json:",inline"` // Name indicates prefix api + // +kubebuilder:validation:MaxLength=253 Name string `json:"name"` // Rules + // +kubebuilder:validation:MaxItems=100 Rules []GatewayHTTPAPIRule `json:"rules"` // Health check endpoint HealthCheckEndpoint string `json:"healthCheckEndpoint,omitempty"` diff --git a/config/crd/bases/formance.com_gatewaygrpcapis.yaml b/config/crd/bases/formance.com_gatewaygrpcapis.yaml index 5f02d2c5c..80364812c 100644 --- a/config/crd/bases/formance.com_gatewaygrpcapis.yaml +++ b/config/crd/bases/formance.com_gatewaygrpcapis.yaml @@ -52,6 +52,7 @@ spec: properties: name: description: Name is the backend Service name in the Stack namespace. + maxLength: 253 minLength: 1 type: string port: diff --git a/config/crd/bases/formance.com_gatewayhttpapis.yaml b/config/crd/bases/formance.com_gatewayhttpapis.yaml index c4f7a5963..e1da43301 100644 --- a/config/crd/bases/formance.com_gatewayhttpapis.yaml +++ b/config/crd/bases/formance.com_gatewayhttpapis.yaml @@ -52,6 +52,7 @@ spec: type: string name: description: Name indicates prefix api + maxLength: 253 type: string rules: description: Rules @@ -64,6 +65,7 @@ spec: name: description: Name is the backend Service name in the Stack namespace. + maxLength: 253 minLength: 1 type: string port: @@ -113,6 +115,7 @@ spec: required: - path type: object + maxItems: 100 type: array stack: description: Stack indicates the stack on which the module is installed @@ -121,6 +124,11 @@ spec: - name - rules type: object + x-kubernetes-validations: + - message: backendRef.name must differ from spec.name because the latter + is managed by the GatewayHTTPAPI controller + rule: self.rules.all(rule, !has(rule.backendRef) || rule.backendRef.name + != self.name) status: properties: conditions: diff --git a/docs/09-Configuration reference/02-Custom Resource Definitions.md b/docs/09-Configuration reference/02-Custom Resource Definitions.md index 8429bdf00..1bf7cb4e4 100644 --- a/docs/09-Configuration reference/02-Custom Resource Definitions.md +++ b/docs/09-Configuration reference/02-Custom Resource Definitions.md @@ -2326,7 +2326,7 @@ When omitted, Gateway keeps using the module's historical Service. | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `name` _string_ | Name is the backend Service name in the Stack namespace. | | MinLength: 1
| +| `name` _string_ | Name is the backend Service name in the Stack namespace. | | MaxLength: 253
MinLength: 1
| | `port` _integer_ | Port is the backend Service port. | | Maximum: 65535
Minimum: 1
| | `tls` _[GatewayBackendTLS](#gatewaybackendtls)_ | TLS enables a verified TLS connection to the backend. | | | @@ -2439,8 +2439,8 @@ GatewayHTTPAPI is the Schema for the HTTPAPIs API | Field | Description | Default | Validation | | --- | --- | --- | --- | | `stack` _string_ | Stack indicates the stack on which the module is installed | | | -| `name` _string_ | Name indicates prefix api | | | -| `rules` _[GatewayHTTPAPIRule](#gatewayhttpapirule) array_ | Rules | | | +| `name` _string_ | Name indicates prefix api | | MaxLength: 253
| +| `rules` _[GatewayHTTPAPIRule](#gatewayhttpapirule) array_ | Rules | | MaxItems: 100
| | `healthCheckEndpoint` _string_ | Health check endpoint | | | ###### GatewayHTTPAPIRule diff --git a/helm/crds/templates/crds/apiextensions.k8s.io_v1_customresourcedefinition_gatewaygrpcapis.formance.com.yaml b/helm/crds/templates/crds/apiextensions.k8s.io_v1_customresourcedefinition_gatewaygrpcapis.formance.com.yaml index 3ed754f29..a0fe54d22 100644 --- a/helm/crds/templates/crds/apiextensions.k8s.io_v1_customresourcedefinition_gatewaygrpcapis.formance.com.yaml +++ b/helm/crds/templates/crds/apiextensions.k8s.io_v1_customresourcedefinition_gatewaygrpcapis.formance.com.yaml @@ -55,6 +55,7 @@ spec: properties: name: description: Name is the backend Service name in the Stack namespace. + maxLength: 253 minLength: 1 type: string port: diff --git a/helm/crds/templates/crds/apiextensions.k8s.io_v1_customresourcedefinition_gatewayhttpapis.formance.com.yaml b/helm/crds/templates/crds/apiextensions.k8s.io_v1_customresourcedefinition_gatewayhttpapis.formance.com.yaml index fff2d921e..502476b78 100644 --- a/helm/crds/templates/crds/apiextensions.k8s.io_v1_customresourcedefinition_gatewayhttpapis.formance.com.yaml +++ b/helm/crds/templates/crds/apiextensions.k8s.io_v1_customresourcedefinition_gatewayhttpapis.formance.com.yaml @@ -55,6 +55,7 @@ spec: type: string name: description: Name indicates prefix api + maxLength: 253 type: string rules: description: Rules @@ -67,6 +68,7 @@ spec: name: description: Name is the backend Service name in the Stack namespace. + maxLength: 253 minLength: 1 type: string port: @@ -116,6 +118,7 @@ spec: required: - path type: object + maxItems: 100 type: array stack: description: Stack indicates the stack on which the module is installed @@ -124,6 +127,11 @@ spec: - name - rules type: object + x-kubernetes-validations: + - message: backendRef.name must differ from spec.name because the latter + is managed by the GatewayHTTPAPI controller + rule: self.rules.all(rule, !has(rule.backendRef) || rule.backendRef.name + != self.name) status: properties: conditions: diff --git a/internal/tests/gatewayhttpapi_controller_test.go b/internal/tests/gatewayhttpapi_controller_test.go index b1ebaae53..d3937fb75 100644 --- a/internal/tests/gatewayhttpapi_controller_test.go +++ b/internal/tests/gatewayhttpapi_controller_test.go @@ -5,6 +5,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" v1beta1 "github.com/formancehq/operator/v3/api/formance.com/v1beta1" "github.com/formancehq/operator/v3/internal/resources/gatewayhttpapis" @@ -13,6 +14,20 @@ import ( ) var _ = Describe("GatewayHTTPAPI", func() { + It("rejects a backendRef that collides with the managed Service name", func() { + httpAPI := &v1beta1.GatewayHTTPAPI{ + ObjectMeta: RandObjectMeta(), + Spec: v1beta1.GatewayHTTPAPISpec{ + StackDependency: v1beta1.StackDependency{Stack: "stack0"}, + Name: "ledger", + Rules: []v1beta1.GatewayHTTPAPIRule{{ + BackendRef: &v1beta1.GatewayBackendRef{Name: "ledger", Port: 8081}, + }}, + }, + } + Expect(apierrors.IsInvalid(Create(httpAPI))).To(BeTrue()) + }) + Context("When creating an GatewayHTTPAPI", func() { var ( stack *v1beta1.Stack