keda: multitenant setup

keda/templates/kedify-agent-secret-rbac.yaml

@@ -0,0 +1,43 @@
+{{- if and .Values.kedify.multitenant .Values.kedify.multitenant.mode }}
+# Grant kedify-agent read access to this tenant's TLS certificate secret.
+# The agent needs this to sync tenant certificates into the shared multitenancy config.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  {{- with .Values.additionalAnnotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  labels:
+    app.kubernetes.io/name: {{ .Release.Name }}-kedify-agent-secret-reader
+    {{- include "keda.labels" . | indent 4 }}
+  name: {{ .Release.Name }}-kedify-agent-secret-reader
+  namespace: {{ .Release.Namespace }}
+rules:
+- apiGroups: [""]
+  resources: ["secrets"]
+  verbs: ["get"]
+  resourceNames:
+  - {{ .Values.certificates.secretName | quote }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  {{- with .Values.additionalAnnotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  labels:
+    app.kubernetes.io/name: {{ .Release.Name }}-kedify-agent-secret-reader
+    {{- include "keda.labels" . | indent 4 }}
+  name: {{ .Release.Name }}-kedify-agent-secret-reader
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ .Release.Name }}-kedify-agent-secret-reader
+subjects:
+- kind: ServiceAccount
+  name: {{ .Values.kedify.multitenant.agentServiceAccount }}
+  namespace: {{ .Values.kedify.multitenant.agentNamespace }}
+{{- end }}

keda/templates/kedify-tenant-registration-configmap.yaml

@@ -0,0 +1,25 @@
+{{- if and .Values.kedify.multitenant .Values.kedify.multitenant.mode }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Release.Name }}-kedify-tenant-registration
+  namespace: {{ .Release.Namespace }}
+  labels:
+    kedify.io/tenant-registration: "true"
+    {{- include "keda.labels" . | nindent 4 }}
+  {{- with .Values.additionalAnnotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+data:
+  name: {{ printf "%s/%s" .Release.Namespace .Release.Name | quote }}
+  namespace: {{ .Release.Namespace }}
+  watchNamespace: {{ .Values.watchNamespace | default .Release.Namespace | quote }}
+  address: {{ .Values.kedify.multitenant.address | default (printf "%s.%s.svc.%s:9666" .Values.operator.name .Release.Namespace .Values.clusterDomain) | quote }}
+  {{- if .Values.kedify.multitenant.authority }}
+  authority: {{ .Values.kedify.multitenant.authority | quote }}
+  {{- end }}
+  tlsSecretRef: {{ .Values.certificates.secretName | quote }}
+  isDefaultTenant: {{ eq .Values.kedify.multitenant.mode "default" | quote }}
+  operatorDeploymentName: {{ .Values.operator.name | quote }}
+{{- end }}

keda/templates/manager/deployment.yaml

@@ -81,6 +81,10 @@ spec:
           - "--leader-elect"
           {{- if .Values.operator.leaderElectionID }}
           - "--leader-election-id={{ .Values.operator.leaderElectionID }}"
+          {{- else if eq .Values.kedify.multitenant.mode "tenant" }}
+          {{- /* tenant operators may share a namespace with the default install or other tenants;
+                 derive a unique leader-election lock from the (unique) operator name */}}
+          - "--leader-election-id={{ .Values.operator.name }}.keda.sh"
           {{- end }}
           - "--disable-compression={{ .Values.operator.disableCompression}}"
           - "--zap-log-level={{ .Values.logging.operator.level }}"
@@ -174,6 +178,10 @@ spec:
             - name: KEDA_HTTP_DISABLE_KEEP_ALIVE
               value: "true"
             {{- end }}
+            {{- if eq .Values.kedify.multitenant.mode "tenant" }}
+            - name: KEDIFY_MULTITENANT_MODE_DISABLE_CERT_INJECTION
+              value: "true"
+            {{- end }}
             {{- if .Values.permissions.operator.restrict.secret }}
             - name: KEDA_RESTRICT_SECRET_ACCESS
               value: {{ .Values.permissions.operator.restrict.secret | quote }}

keda/templates/metrics-server/deployment.yaml

@@ -99,6 +99,10 @@ spec:
           env:
             - name: WATCH_NAMESPACE
               value: {{ .Values.watchNamespace | quote }}
+            {{- if eq .Values.kedify.multitenant.mode "default" }}
+            - name: KEDIFY_MULTITENANT_MODE
+              value: "true"
+            {{- end }}
             - name: POD_NAMESPACE
               valueFrom:
                 fieldRef:
@@ -162,6 +166,13 @@ spec:
           - name: grpc-certs
             mountPath: /grpccerts
           {{- end }}
+          {{- if eq .Values.kedify.multitenant.mode "default" }}
+          - name: kedify-multitenancy-config
+            mountPath: /kedify-multitenancy-config
+            readOnly: true
+          - name: kedify-tenant-certs
+            mountPath: /kedify-tenant-certs
+          {{- end }}
           {{- if .Values.hashiCorpVaultTLS }}
           - name: hashicorp-vault-certs
             mountPath: /hashicorp-vaultcerts
@@ -185,6 +196,14 @@ spec:
         secret:
           secretName: {{ .Values.grpcTLSCertsSecret }}
       {{- end }}
+      {{- if eq .Values.kedify.multitenant.mode "default" }}
+      - name: kedify-multitenancy-config
+        secret:
+          defaultMode: 420
+          secretName: {{ .Values.kedify.multitenant.configSecretName }}
+      - name: kedify-tenant-certs
+        emptyDir: {}
+      {{- end }}
       {{- if .Values.hashiCorpVaultTLS }}
       - name: hashicorp-vault-certs
         secret:

keda/templates/metrics-server/kedify-multitenancy-config.yaml

@@ -0,0 +1,15 @@
+{{- if eq .Values.kedify.multitenant.mode "default" }}
+# secret for kedify multitenancy configuration, dynamically populated by kedify-agent at runtime
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ .Values.kedify.multitenant.configSecretName }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "keda.labels" . | nindent 4 }}
+  {{- with .Values.additionalAnnotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+type: Opaque
+{{- end }}

keda/templates/metrics-server/kedify-role.yaml

@@ -0,0 +1,43 @@
+{{- if and .Values.metricsServer.enabled .Values.rbac.create (eq .Values.kedify.multitenant.mode "default") }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  {{- with .Values.additionalAnnotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  labels:
+    app.kubernetes.io/name: {{ .Values.operator.name }}-metrics-server-kedify
+    {{- include "keda.labels" . | indent 4 }}
+  name: {{ .Values.operator.name }}-metrics-server-kedify
+  namespace: {{ .Release.Namespace }}
+rules:
+- apiGroups:
+  - events.k8s.io
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  {{- with .Values.additionalAnnotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  labels:
+    app.kubernetes.io/name: {{ .Values.operator.name }}-metrics-server-kedify
+    {{- include "keda.labels" . | indent 4 }}
+  name: {{ .Values.operator.name }}-metrics-server-kedify
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ .Values.operator.name }}-metrics-server-kedify
+subjects:
+- kind: ServiceAccount
+  name: {{ (.Values.serviceAccount.metricServer).name | default .Values.serviceAccount.name }}
+  namespace: {{ .Release.Namespace }}
+{{- end -}}

keda/templates/webhooks/clusterrole.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.rbac.create }}
+{{- if and .Values.rbac.create .Values.webhooks.enabled }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:

keda/templates/webhooks/clusterrolebindings.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.rbac.create }}
+{{- if and .Values.rbac.create .Values.webhooks.enabled }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:

keda/templates/webhooks/serviceaccount.yaml

@@ -1,4 +1,4 @@
-{{- if (.Values.serviceAccount.webhooks).create | default .Values.serviceAccount.create -}}
+{{- if and .Values.webhooks.enabled ((.Values.serviceAccount.webhooks).create | default .Values.serviceAccount.create) -}}
 apiVersion: v1
 kind: ServiceAccount
 metadata:

keda/values.schema.json

@@ -23,6 +23,9 @@
                 "watchNamespace": {
                     "$ref": "#/definitions/WatchNamespace"
                 },
+                "kedify": {
+                    "$ref": "#/definitions/Kedify"
+                },
                 "imagePullSecrets": {
                     "$ref": "#/definitions/ImagePullSecrets"
                 },
@@ -142,6 +145,7 @@
                 "clusterDomain",
                 "crds",
                 "watchNamespace",
+                "kedify",
                 "imagePullSecrets",
                 "networkPolicy",
                 "operator",
@@ -275,6 +279,60 @@
                 }
             ]
         },
+        "Kedify": {
+            "type": "object",
+            "properties": {
+                "multitenant": {
+                    "$ref": "#/definitions/KedifyMultitenant"
+                }
+            },
+            "required": [
+                "multitenant"
+            ]
+        },
+        "KedifyMultitenant": {
+            "type": "object",
+            "properties": {
+                "mode": {
+                    "type": "string",
+                    "description": "Multitenant mode: \"\" (disabled), \"default\" or \"tenant\"",
+                    "enum": [
+                        "",
+                        "default",
+                        "tenant"
+                    ],
+                    "default": ""
+                },
+                "configSecretName": {
+                    "type": "string",
+                    "default": "kedify-multitenancy-config"
+                },
+                "address": {
+                    "type": "string",
+                    "default": ""
+                },
+                "authority": {
+                    "type": "string",
+                    "default": ""
+                },
+                "agentNamespace": {
+                    "type": "string",
+                    "default": "keda"
+                },
+                "agentServiceAccount": {
+                    "type": "string",
+                    "default": "kedify-agent"
+                }
+            },
+            "required": [
+                "mode",
+                "configSecretName",
+                "address",
+                "authority",
+                "agentNamespace",
+                "agentServiceAccount"
+            ]
+        },
         "ImagePullSecrets": {
             "type": "array",
             "items": {

keda/values.yaml

@@ -62,6 +62,29 @@ crds:
 # -- Defines Kubernetes namespaces to watch to scale their workloads. Default watches all namespaces
 watchNamespace: ""
 
+kedify:
+  # -- For keda installed in multitenant environment
+  # Each tenant keda should also configure watchNamespace to ensure desired multitenancy sharding behavior
+  multitenant:
+    # -- Multitenant mode: "" (disabled), "default" or "tenant"
+    # "": multitenancy disabled, standard KEDA deployment
+    # "default": deploys keda-operator + keda-operator-metrics-apiserver (and optionally keda-admission-webhooks); the metrics apiserver is configured to route HPA metric requests to the tenant keda-operators
+    # "tenant": deploys only keda-operator, should be used with metricsServer.enabled=false, crds.install=false, webhooks.enabled=false. Give each tenant a unique operator.name (and serviceAccount.operator.name, certificates.secretName) so its cluster RBAC does not collide with the default install or other tenants
+    mode: ""
+
+    # -- Name of the Secret to store the connection information for tenants to configure default keda-operator-metrics-apiserver to connect to the correct keda-operator in the tenant namespace
+    configSecretName: kedify-multitenancy-config
+
+    # -- Override gRPC address for the tenant's keda-operator (default: <operator.name>.<namespace>.svc.<clusterDomain>:9666)
+    address: ""
+    # -- SNI authority override for gRPC connection to the tenant's keda-operator
+    authority: ""
+
+    # -- Namespace where kedify-agent runs (used to grant it read access to this tenant's TLS secret)
+    agentNamespace: "keda"
+    # -- ServiceAccount name of the kedify-agent
+    agentServiceAccount: "kedify-agent"
+
 # -- Name of secret to use to pull images to use to pull Docker images
 imagePullSecrets: []