From fd9129eefb27c0621fa4b323913fa886b3f27f2c Mon Sep 17 00:00:00 2001 From: Jan Wozniak Date: Thu, 4 Jun 2026 08:50:26 +0000 Subject: [PATCH 1/4] kedify-agent: multitenant keda config ConfigMap-based self-registration: enable multitenancy on the default KEDA installation and grant the agent RBAC to sync tenant certs and the shared multitenancy config. Ported onto the hookless CR install (#462): the multitenancy toggle now lives in files/crs/default-kedifyconfig.yaml. Squashes the two original PR #424 commits (the second reverted the verbose per-tenant CR templating of the first). Signed-off-by: Jan Wozniak --- .../files/crds/kedify-configuration.yaml | 118 +++++++++++++++++- .../files/crs/default-kedifyconfig.yaml | 3 + kedify-agent/templates/agent-rbac.yaml | 31 +++++ kedify-agent/values.yaml | 5 + 4 files changed, 156 insertions(+), 1 deletion(-) diff --git a/kedify-agent/files/crds/kedify-configuration.yaml b/kedify-agent/files/crds/kedify-configuration.yaml index 10a41c186..9e53fb7b3 100644 --- a/kedify-agent/files/crds/kedify-configuration.yaml +++ b/kedify-agent/files/crds/kedify-configuration.yaml @@ -99,7 +99,9 @@ spec: Dashboard type: string kedaInstallations: - description: KEDAInstallations defines the KEDA installations options + description: |- + KEDAInstallations defines the KEDA installations options. + Names must be unique across installations. items: description: KEDAInstallation defines the desired state of a KEDA installation @@ -184,11 +186,24 @@ spec: - Disabled - Helm type: string + multitenancy: + description: |- + Multitenancy defines if KEDA should be installed in multitenant mode, where metrics API is shared between multiple installations and how to route HPA requests to the right KEDA operator. + Multitenancy is only supported with Helm management mode. + properties: + multitenancyConfigRef: + description: |- + ConfigRef is the reference to the secret for the multitenancy connections, if not provided defaults to "kedify-multitenancy-config" secret in the namespace of the default KEDA installation. + This is where the keda adapter gets all TLS certs to each tenant keda-operator and the routing configuration for the HPA requests. + type: string + type: object name: description: Name of the KEDA installation + minLength: 1 type: string namespace: description: Namespace where KEDA should be installed + minLength: 1 type: string otelAddonHelm: description: OTELAddonHelm defines the otel-addon Helm values @@ -298,11 +313,23 @@ spec: type: integer type: object type: object + watchNamespace: + description: |- + WatchNamespace is the namespace that this KEDA operator watches for ScaledObjects/ScaledJobs. + In per-namespace mode this equals Namespace and can be omitted. + In single-namespace mode (multiple operators in the same namespace), set this to the target namespace each operator watches. + type: string required: - agentAutoUpdate - mode + - name + - namespace type: object + maxItems: 32 type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map type: object status: description: KedifyConfigurationStatus defines the observed state of KedifyConfiguration @@ -363,6 +390,46 @@ spec: type: string observedHelmValues: type: string + tenants: + additionalProperties: + description: TenantStatus defines the observed state of a + non-default multitenant KEDA installation + properties: + configSynced: + description: KedifyConfigurationResourceStatus defines + the observed state of a KedifyConfiguration owned resources + properties: + phase: + type: string + reason: + type: string + type: object + operator: + description: KedifyConfigurationResourceStatus defines + the observed state of a KedifyConfiguration owned resources + properties: + phase: + type: string + reason: + type: string + type: object + tlsCertSecret: + description: KedifyConfigurationResourceStatus defines + the observed state of a KedifyConfiguration owned resources + properties: + phase: + type: string + reason: + type: string + type: object + required: + - configSynced + - operator + - tlsCertSecret + type: object + description: Tenants contains per-tenant status for non-default + multitenant KEDA installations + type: object version: type: string type: object @@ -373,6 +440,55 @@ spec: description: ConfigSHA256 calculated sha over all relevant field used for internal purposes type: string + discoveredTenants: + description: DiscoveredTenants contains the list of tenants discovered + via registration ConfigMaps + items: + description: DiscoveredTenant represents a tenant discovered via + a registration ConfigMap, stored in KedifyConfiguration status. + properties: + address: + description: Address is the gRPC address of the keda-operator + type: string + configSynced: + description: ConfigSynced indicates whether the tenant config + has been synced to the multitenancy config secret + type: boolean + isDefault: + description: IsDefault indicates this is the default tenant + type: boolean + message: + description: Message contains additional information about the + tenant status + type: string + name: + description: Name is the globally unique tenant identifier in + "namespace/release" format + type: string + namespace: + description: Namespace where the KEDA operator runs + type: string + operatorReady: + description: OperatorReady indicates whether the keda-operator + deployment is ready + type: boolean + tlsCertReady: + description: TLSCertReady indicates whether the TLS cert secret + is found and populated + type: boolean + watchNamespace: + description: WatchNamespace is the namespace KEDA watches for + ScaledObjects/ScaledJobs + type: string + required: + - address + - configSynced + - name + - namespace + - operatorReady + - tlsCertReady + type: object + type: array multiClusterStatus: description: MultiClusterStatus contains status information about multi-cluster deployments diff --git a/kedify-agent/files/crs/default-kedifyconfig.yaml b/kedify-agent/files/crs/default-kedifyconfig.yaml index a7a5d5cfc..069015d84 100644 --- a/kedify-agent/files/crs/default-kedifyconfig.yaml +++ b/kedify-agent/files/crs/default-kedifyconfig.yaml @@ -10,4 +10,7 @@ spec: {{- end }} kedaInstallations: - namespace: {{ .Release.Namespace }} + {{- if .Values.agent.features.multitenantKEDAEnabled }} + multitenancy: {} + {{- end }} {{ toYaml .Values.agent.kedifyConfiguration | indent 4 }} diff --git a/kedify-agent/templates/agent-rbac.yaml b/kedify-agent/templates/agent-rbac.yaml index c200a29e1..1f7a8718b 100644 --- a/kedify-agent/templates/agent-rbac.yaml +++ b/kedify-agent/templates/agent-rbac.yaml @@ -236,6 +236,37 @@ rules: - get - list - watch +{{- if .Values.agent.features.multitenantKEDAEnabled }} +# for tenant configuration and cert management in fleet installation +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch +# tenant TLS secrets are only read (synced into the multitenancy config secret) +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + resourceNames: + - kedaorg-certs +# the multitenancy config secret is owned and written by the agent +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - patch + - update + resourceNames: + - kedify-multitenancy-config +{{- end }} {{- if .Values.agent.rbac.readNodes }} # for metrics and metered billing - apiGroups: diff --git a/kedify-agent/values.yaml b/kedify-agent/values.yaml index 1f499a382..395a39089 100644 --- a/kedify-agent/values.yaml +++ b/kedify-agent/values.yaml @@ -130,6 +130,7 @@ agent: distributedScaledJobsEnabled: false scalingPoliciesEnabled: true recommendationsForLabeledNamespaces: true + multitenantKEDAEnabled: false autowire: # period for EndpointSlices and Services in autowiring mode for redundant synchronization # standard reconciliation still applies and changes in EndpointSlices for autowired Services are picked up automatically too without waiting for this period @@ -244,6 +245,10 @@ agent: affinity: {} # additional cli arguments passed to the agent in form of "key: value" => "--key=value" extraArgs: {} + # configuration for KEDA multitenancy, must have features.multitenantKEDAEnabled set to true to take effect + # tenants are discovered dynamically via ConfigMaps with label kedify.io/tenant-registration=true + multitenant: {} + # when set to: 'helm' (default), the installation will rely on helm hooks for CRD and CR installation and cleanup # 'jobs' - the helm hooks will be disabled for CRD & CR installation and k8s jobs will be used instead # 'argo' - everything will be installed as regular manifests and argocd annotations will be used From 43c9de8a3b56d601072fc26daaf0a696818c9a22 Mon Sep 17 00:00:00 2001 From: Jan Wozniak Date: Thu, 4 Jun 2026 13:10:47 +0000 Subject: [PATCH 2/4] kedify-agent: set name on default kedaInstallation; drop unused multitenant value - The updated KedifyConfiguration CRD requires a non-empty name on each kedaInstallations entry, but the default CR only set namespace, so CR creation/upgrade would fail validation. Set name to the release namespace. - Remove agent.multitenant (was an empty value wired to nothing); multitenancy is driven by features.multitenantKEDAEnabled and dynamic tenant registration. --- kedify-agent/files/crs/default-kedifyconfig.yaml | 3 ++- kedify-agent/values.yaml | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/kedify-agent/files/crs/default-kedifyconfig.yaml b/kedify-agent/files/crs/default-kedifyconfig.yaml index 069015d84..9594c1b96 100644 --- a/kedify-agent/files/crs/default-kedifyconfig.yaml +++ b/kedify-agent/files/crs/default-kedifyconfig.yaml @@ -9,7 +9,8 @@ spec: cloudAccountID: "{{ .Values.agent.cloudAccountID }}" {{- end }} kedaInstallations: - - namespace: {{ .Release.Namespace }} + - name: {{ .Release.Namespace }} + namespace: {{ .Release.Namespace }} {{- if .Values.agent.features.multitenantKEDAEnabled }} multitenancy: {} {{- end }} diff --git a/kedify-agent/values.yaml b/kedify-agent/values.yaml index 395a39089..5b0d4104e 100644 --- a/kedify-agent/values.yaml +++ b/kedify-agent/values.yaml @@ -245,9 +245,9 @@ agent: affinity: {} # additional cli arguments passed to the agent in form of "key: value" => "--key=value" extraArgs: {} - # configuration for KEDA multitenancy, must have features.multitenantKEDAEnabled set to true to take effect - # tenants are discovered dynamically via ConfigMaps with label kedify.io/tenant-registration=true - multitenant: {} + # KEDA multitenancy is enabled with features.multitenantKEDAEnabled=true. Tenants are + # discovered dynamically by the agent from ConfigMaps labeled + # kedify.io/tenant-registration=true, so there is no static tenant list to configure here. # when set to: 'helm' (default), the installation will rely on helm hooks for CRD and CR installation and cleanup # 'jobs' - the helm hooks will be disabled for CRD & CR installation and k8s jobs will be used instead From 749aaec5a7ca2f91f1261c9f81fed572e5140c4f Mon Sep 17 00:00:00 2001 From: kedifybot Date: Fri, 12 Jun 2026 13:11:14 +0000 Subject: [PATCH 3/4] kedify-agent: drop redundant cluster-wide get on tenant cert secrets The multitenant ClusterRole granted the agent cluster-wide `get` on Secrets named `kedaorg-certs`. That is redundant and hardcodes the secret name: the keda chart already grants the agent per-tenant access via namespaced, resourceName- scoped Roles created in each tenant namespace (kedify-agent-secret-rbac.yaml), bound to the agent ServiceAccount and keyed on each tenant's own configurable certificates.secretName. The cluster-wide rule only ever matched the default name and broadened secret read unnecessarily. Keep the cluster-wide ConfigMap discovery (RBAC can't label-filter list/watch) and the resourceName-scoped get/patch/update on the kedify-multitenancy-config secret the agent owns. Co-Authored-By: Claude Opus 4.8 (1M context) --- kedify-agent/templates/agent-rbac.yaml | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/kedify-agent/templates/agent-rbac.yaml b/kedify-agent/templates/agent-rbac.yaml index 1f7a8718b..787cdbff6 100644 --- a/kedify-agent/templates/agent-rbac.yaml +++ b/kedify-agent/templates/agent-rbac.yaml @@ -237,7 +237,10 @@ rules: - list - watch {{- if .Values.agent.features.multitenantKEDAEnabled }} -# for tenant configuration and cert management in fleet installation +# Discover tenant registration ConfigMaps cluster-wide. RBAC cannot filter +# list/watch by label and tenants live in arbitrary namespaces, so this has to +# be cluster-wide; only ConfigMaps labelled kedify.io/tenant-registration=true +# are acted upon. - apiGroups: - "" resources: @@ -246,16 +249,12 @@ rules: - get - list - watch -# tenant TLS secrets are only read (synced into the multitenancy config secret) -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - resourceNames: - - kedaorg-certs -# the multitenancy config secret is owned and written by the agent +# Per-tenant TLS cert secrets are read through the namespaced, resourceName-scoped +# Roles the keda chart creates in each tenant namespace +# (kedify-agent-secret-rbac.yaml). Those bind this agent ServiceAccount and use +# each tenant's own (configurable) certificates.secretName, so no cluster-wide +# secret read is granted here. +# The multitenancy config secret is owned and written by the agent. - apiGroups: - "" resources: From 236630ec4857c921d10ca769ac1536478b55f600 Mon Sep 17 00:00:00 2001 From: Jan Wozniak Date: Fri, 12 Jun 2026 15:31:31 +0200 Subject: [PATCH 4/4] Apply suggestion from @wozniakjan Signed-off-by: Jan Wozniak --- kedify-agent/values.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/kedify-agent/values.yaml b/kedify-agent/values.yaml index 5b0d4104e..9c4d990f6 100644 --- a/kedify-agent/values.yaml +++ b/kedify-agent/values.yaml @@ -245,9 +245,6 @@ agent: affinity: {} # additional cli arguments passed to the agent in form of "key: value" => "--key=value" extraArgs: {} - # KEDA multitenancy is enabled with features.multitenantKEDAEnabled=true. Tenants are - # discovered dynamically by the agent from ConfigMaps labeled - # kedify.io/tenant-registration=true, so there is no static tenant list to configure here. # when set to: 'helm' (default), the installation will rely on helm hooks for CRD and CR installation and cleanup # 'jobs' - the helm hooks will be disabled for CRD & CR installation and k8s jobs will be used instead