kedify-agent: multitenant keda config

kedify-agent/files/crds/kedify-configuration.yaml

@@ -99,7 +99,9 @@ spec:
                   Dashboard
                 type: string
               kedaInstallations:
-                description: KEDAInstallations defines the KEDA installations options
+                description: |-
+                  KEDAInstallations defines the KEDA installations options.
+                  Names must be unique across installations.
                 items:
                   description: KEDAInstallation defines the desired state of a KEDA
                     installation
@@ -184,11 +186,24 @@ spec:
                       - Disabled
                       - Helm
                       type: string
+                    multitenancy:
+                      description: |-
+                        Multitenancy defines if KEDA should be installed in multitenant mode, where metrics API is shared between multiple installations and how to route HPA requests to the right KEDA operator.
+                        Multitenancy is only supported with Helm management mode.
+                      properties:
+                        multitenancyConfigRef:
+                          description: |-
+                            ConfigRef is the reference to the secret for the multitenancy connections, if not provided defaults to "kedify-multitenancy-config" secret in the namespace of the default KEDA installation.
+                            This is where the keda adapter gets all TLS certs to each tenant keda-operator and the routing configuration for the HPA requests.
+                          type: string
+                      type: object
                     name:
                       description: Name of the KEDA installation
+                      minLength: 1
                       type: string
                     namespace:
                       description: Namespace where KEDA should be installed
+                      minLength: 1
                       type: string
                     otelAddonHelm:
                       description: OTELAddonHelm defines the otel-addon Helm values
@@ -298,11 +313,23 @@ spec:
                               type: integer
                           type: object
                       type: object
+                    watchNamespace:
+                      description: |-
+                        WatchNamespace is the namespace that this KEDA operator watches for ScaledObjects/ScaledJobs.
+                        In per-namespace mode this equals Namespace and can be omitted.
+                        In single-namespace mode (multiple operators in the same namespace), set this to the target namespace each operator watches.
+                      type: string
                   required:
                   - agentAutoUpdate
                   - mode
+                  - name
+                  - namespace
                   type: object
+                maxItems: 32
                 type: array
+                x-kubernetes-list-map-keys:
+                - name
+                x-kubernetes-list-type: map
             type: object
           status:
             description: KedifyConfigurationStatus defines the observed state of KedifyConfiguration
@@ -363,6 +390,46 @@ spec:
                       type: string
                     observedHelmValues:
                       type: string
+                    tenants:
+                      additionalProperties:
+                        description: TenantStatus defines the observed state of a
+                          non-default multitenant KEDA installation
+                        properties:
+                          configSynced:
+                            description: KedifyConfigurationResourceStatus defines
+                              the observed state of a KedifyConfiguration owned resources
+                            properties:
+                              phase:
+                                type: string
+                              reason:
+                                type: string
+                            type: object
+                          operator:
+                            description: KedifyConfigurationResourceStatus defines
+                              the observed state of a KedifyConfiguration owned resources
+                            properties:
+                              phase:
+                                type: string
+                              reason:
+                                type: string
+                            type: object
+                          tlsCertSecret:
+                            description: KedifyConfigurationResourceStatus defines
+                              the observed state of a KedifyConfiguration owned resources
+                            properties:
+                              phase:
+                                type: string
+                              reason:
+                                type: string
+                            type: object
+                        required:
+                        - configSynced
+                        - operator
+                        - tlsCertSecret
+                        type: object
+                      description: Tenants contains per-tenant status for non-default
+                        multitenant KEDA installations
+                      type: object
                     version:
                       type: string
                   type: object
@@ -373,6 +440,55 @@ spec:
                 description: ConfigSHA256 calculated sha over all relevant field used
                   for internal purposes
                 type: string
+              discoveredTenants:
+                description: DiscoveredTenants contains the list of tenants discovered
+                  via registration ConfigMaps
+                items:
+                  description: DiscoveredTenant represents a tenant discovered via
+                    a registration ConfigMap, stored in KedifyConfiguration status.
+                  properties:
+                    address:
+                      description: Address is the gRPC address of the keda-operator
+                      type: string
+                    configSynced:
+                      description: ConfigSynced indicates whether the tenant config
+                        has been synced to the multitenancy config secret
+                      type: boolean
+                    isDefault:
+                      description: IsDefault indicates this is the default tenant
+                      type: boolean
+                    message:
+                      description: Message contains additional information about the
+                        tenant status
+                      type: string
+                    name:
+                      description: Name is the globally unique tenant identifier in
+                        "namespace/release" format
+                      type: string
+                    namespace:
+                      description: Namespace where the KEDA operator runs
+                      type: string
+                    operatorReady:
+                      description: OperatorReady indicates whether the keda-operator
+                        deployment is ready
+                      type: boolean
+                    tlsCertReady:
+                      description: TLSCertReady indicates whether the TLS cert secret
+                        is found and populated
+                      type: boolean
+                    watchNamespace:
+                      description: WatchNamespace is the namespace KEDA watches for
+                        ScaledObjects/ScaledJobs
+                      type: string
+                  required:
+                  - address
+                  - configSynced
+                  - name
+                  - namespace
+                  - operatorReady
+                  - tlsCertReady
+                  type: object
+                type: array
               multiClusterStatus:
                 description: MultiClusterStatus contains status information about
                   multi-cluster deployments

kedify-agent/files/crs/default-kedifyconfig.yaml

@@ -9,5 +9,9 @@ spec:
   cloudAccountID: "{{ .Values.agent.cloudAccountID }}"
   {{- end }}
   kedaInstallations:
-  - namespace: {{ .Release.Namespace }}
+  - name: {{ .Release.Namespace }}
+    namespace: {{ .Release.Namespace }}
+    {{- if .Values.agent.features.multitenantKEDAEnabled }}
+    multitenancy: {}
+    {{- end }}
 {{ toYaml .Values.agent.kedifyConfiguration | indent 4 }}

kedify-agent/templates/agent-rbac.yaml

@@ -236,6 +236,36 @@ rules:
   - get
   - list
   - watch
+{{- if .Values.agent.features.multitenantKEDAEnabled }}
+# Discover tenant registration ConfigMaps cluster-wide. RBAC cannot filter
+# list/watch by label and tenants live in arbitrary namespaces, so this has to
+# be cluster-wide; only ConfigMaps labelled kedify.io/tenant-registration=true
+# are acted upon.
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - get
+  - list
+  - watch
+# Per-tenant TLS cert secrets are read through the namespaced, resourceName-scoped
+# Roles the keda chart creates in each tenant namespace
+# (kedify-agent-secret-rbac.yaml). Those bind this agent ServiceAccount and use
+# each tenant's own (configurable) certificates.secretName, so no cluster-wide
+# secret read is granted here.
+# The multitenancy config secret is owned and written by the agent.
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
+  - patch
+  - update
+  resourceNames:
+  - kedify-multitenancy-config
+{{- end }}
 {{- if .Values.agent.rbac.readNodes }}
 # for metrics and metered billing
 - apiGroups:

kedify-agent/values.yaml

@@ -130,6 +130,7 @@ agent:
     distributedScaledJobsEnabled: false
     scalingPoliciesEnabled: true
     recommendationsForLabeledNamespaces: true
+    multitenantKEDAEnabled: false
   autowire:
     # period for EndpointSlices and Services in autowiring mode for redundant synchronization
     # standard reconciliation still applies and changes in EndpointSlices for autowired Services are picked up automatically too without waiting for this period
@@ -244,6 +245,10 @@ agent:
   affinity: {}
   # additional cli arguments passed to the agent in form of "key: value" => "--key=value"
   extraArgs: {}
+  # KEDA multitenancy is enabled with features.multitenantKEDAEnabled=true. Tenants are
+  # discovered dynamically by the agent from ConfigMaps labeled
+  # kedify.io/tenant-registration=true, so there is no static tenant list to configure here.
+
 # when set to: 'helm' (default), the installation will rely on helm hooks for CRD and CR installation and cleanup
 #              'jobs' - the helm hooks will be disabled for CRD & CR installation and k8s jobs will be used instead
 #              'argo' - everything will be installed as regular manifests and argocd annotations will be used