[4.22] OCPBUGS-98604: Add NodeSelectorAdjuster admission plugin for HCP clusters

openshift-kube-apiserver/admission/admissionenablement/register.go

@@ -22,6 +22,7 @@ import (
 	ingressadmission "k8s.io/kubernetes/openshift-kube-apiserver/admission/route"
 	"k8s.io/kubernetes/openshift-kube-apiserver/admission/route/hostassignment"
 	projectnodeenv "k8s.io/kubernetes/openshift-kube-apiserver/admission/scheduler/nodeenv"
+	"k8s.io/kubernetes/openshift-kube-apiserver/admission/scheduler/nodeselectoradjuster"
 	schedulerpodnodeconstraints "k8s.io/kubernetes/openshift-kube-apiserver/admission/scheduler/podnodeconstraints"
 	"k8s.io/kubernetes/openshift-kube-apiserver/admission/storage/csiinlinevolumesecurity"
 	"k8s.io/kubernetes/openshift-kube-apiserver/admission/storage/performantsecuritypolicy"
@@ -46,6 +47,9 @@ func RegisterOpenshiftKubeAdmissionPlugins(plugins *admission.Plugins) {
 	restrictedendpoints.RegisterRestrictedEndpoints(plugins)
 	csiinlinevolumesecurity.Register(plugins)
 	performantsecuritypolicy.Register(plugins)
+	if !nodeselectoradjuster.IsStandalone() {
+		nodeselectoradjuster.Register(plugins)
+	}
 }
 
 var (
@@ -62,25 +66,31 @@ var (
 	)
 
 	// openshiftAdmissionPluginsForKubeBeforeMutating are the admission plugins to add after kube admission, before mutating webhooks
-	openshiftAdmissionPluginsForKubeBeforeMutating = []string{
-		"autoscaling.openshift.io/ClusterResourceOverride",
-		managementcpusoverride.PluginName, // "autoscaling.openshift.io/ManagementCPUsOverride"
-		"authorization.openshift.io/RestrictSubjectBindings",
-		"autoscaling.openshift.io/RunOnceDuration",
-		"scheduling.openshift.io/PodNodeConstraints",
-		"scheduling.openshift.io/OriginPodNodeEnvironment",
-		"network.openshift.io/ExternalIPRanger",
-		"network.openshift.io/RestrictedEndpointsAdmission",
-		imagepolicyapiv1.PluginName, // "image.openshift.io/ImagePolicy"
-		"security.openshift.io/SecurityContextConstraint",
-		"security.openshift.io/SCCExecRestrictions",
-		"route.openshift.io/IngressAdmission",
-		hostassignment.PluginName,           // "route.openshift.io/RouteHostAssignment"
-		csiinlinevolumesecurity.PluginName,  // "storage.openshift.io/CSIInlineVolumeSecurity"
-		managednode.PluginName,              // "autoscaling.openshift.io/ManagedNode"
-		mixedcpus.PluginName,                // "autoscaling.openshift.io/MixedCPUs"
-		performantsecuritypolicy.PluginName, // "storage.openshift.io/PerformantSecurityPolicy"
-	}
+	openshiftAdmissionPluginsForKubeBeforeMutating = func() []string {
+		plugins := []string{
+			"autoscaling.openshift.io/ClusterResourceOverride",
+			managementcpusoverride.PluginName, // "autoscaling.openshift.io/ManagementCPUsOverride"
+			"authorization.openshift.io/RestrictSubjectBindings",
+			"autoscaling.openshift.io/RunOnceDuration",
+			"scheduling.openshift.io/PodNodeConstraints",
+			"scheduling.openshift.io/OriginPodNodeEnvironment",
+			"network.openshift.io/ExternalIPRanger",
+			"network.openshift.io/RestrictedEndpointsAdmission",
+			imagepolicyapiv1.PluginName, // "image.openshift.io/ImagePolicy"
+			"security.openshift.io/SecurityContextConstraint",
+			"security.openshift.io/SCCExecRestrictions",
+			"route.openshift.io/IngressAdmission",
+			hostassignment.PluginName,           // "route.openshift.io/RouteHostAssignment"
+			csiinlinevolumesecurity.PluginName,  // "storage.openshift.io/CSIInlineVolumeSecurity"
+			managednode.PluginName,              // "autoscaling.openshift.io/ManagedNode"
+			mixedcpus.PluginName,                // "autoscaling.openshift.io/MixedCPUs"
+			performantsecuritypolicy.PluginName, // "storage.openshift.io/PerformantSecurityPolicy"
+		}
+		if !nodeselectoradjuster.IsStandalone() {
+			plugins = append(plugins, nodeselectoradjuster.PluginName) // "scheduling.openshift.io/NodeSelectorAdjuster"
+		}
+		return plugins
+	}()
 
 	// openshiftAdmissionPluginsForKubeAfterResourceQuota are the plugins to add after ResourceQuota plugin
 	openshiftAdmissionPluginsForKubeAfterResourceQuota = []string{

openshift-kube-apiserver/admission/scheduler/nodeselectoradjuster/admission.go

@@ -0,0 +1,108 @@
+package nodeselectoradjuster
+
+// The NodeSelectorAdjuster admission plugin removes the
+// node-role.kubernetes.io/master node selector from qualifying pods. It only
+// activates on HCP (hosted control plane) clusters, detected by
+// POD_NAMESPACE != openshift-kube-apiserver. On standalone OpenShift clusters
+// the plugin does not register itself and takes no action, leaving the master
+// node selector in place so that qualifying pods run on master nodes as
+// intended.
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apiserver/pkg/admission"
+	coreapi "k8s.io/kubernetes/pkg/apis/core"
+)
+
+const (
+	// PluginName is the name used to identify this plugin in the admission chain.
+	PluginName = "scheduling.openshift.io/NodeSelectorAdjuster"
+
+	// masterRoleKey is the node role label used as a node selector key in
+	// 4.x VPA operator manifests.
+	masterRoleKey = "node-role.kubernetes.io/master"
+
+	// vpaOperatorLabelKey / vpaOperatorLabelValue identify the VPA operator pod.
+	vpaOperatorLabelKey   = "k8s-app"
+	vpaOperatorLabelValue = "vertical-pod-autoscaler-operator"
+	// vpaOperatorNamespace is the namespace the VPA operator is expected to run in.
+	vpaOperatorNamespace = "openshift-vertical-pod-autoscaler"
+
+	// standaloneEnvVar is the environment variable checked at start-up.
+	// It is injected by the downward API and reflects the namespace the
+	// kube-apiserver pod runs in.
+	standaloneEnvVar = "POD_NAMESPACE"
+	// standaloneEnvValue is the namespace used by the kube-apiserver on a
+	// standalone OpenShift cluster.
+	standaloneEnvValue = "openshift-kube-apiserver"
+)
+
+// IsStandalone reports whether the current process is running inside a standalone
+// OpenShift cluster. It is checked once at start-up to decide whether the plugin
+// should register itself.
+func IsStandalone() bool {
+	return os.Getenv(standaloneEnvVar) == standaloneEnvValue
+}
+
+// Register adds the plugin to the admission plugin registry. It must only be
+// called when IsStandalone() returns false (i.e. on HCP clusters).
+func Register(plugins *admission.Plugins) {
+	plugins.Register(PluginName, func(_ io.Reader) (admission.Interface, error) {
+		return &nodeSelectorAdjuster{
+			Handler: admission.NewHandler(admission.Create),
+		}, nil
+	})
+}
+
+// nodeSelectorAdjuster implements admission.MutationInterface.
+type nodeSelectorAdjuster struct {
+	*admission.Handler
+}
+
+var _ admission.MutationInterface = &nodeSelectorAdjuster{}
+
+// Admit examines newly-created Pod objects and, for qualifying pods, removes the
+// master node selector so that they can run on worker nodes in HCP clusters.
+func (p *nodeSelectorAdjuster) Admit(_ context.Context, attr admission.Attributes, _ admission.ObjectInterfaces) error {
+	if attr.GetResource().GroupResource() != corev1.Resource("pods") || attr.GetSubresource() != "" {
+		return nil
+	}
+
+	pod, ok := attr.GetObject().(*coreapi.Pod)
+	if !ok {
+		return admission.NewForbidden(attr, fmt.Errorf("unexpected object type: %T", attr.GetObject()))
+	}
+
+	if !requiresNodeSelectorAdjustment(pod) {
+		return nil
+	}
+
+	delete(pod.Spec.NodeSelector, masterRoleKey)
+	return nil
+}
+
+// ValidateInitialization satisfies admission.InitializationValidator. The plugin
+// has no external dependencies to validate.
+func (p *nodeSelectorAdjuster) ValidateInitialization() error {
+	return nil
+}
+
+// requiresNodeSelectorAdjustment returns true when the pod carries a label that
+// opts it in to node placement and lives in a namespace where that
+// label is expected. Control-plane-adjacent Day 2 operators can be added here.
+func requiresNodeSelectorAdjustment(pod *coreapi.Pod) bool {
+	// For VPA, we only want to update if the sole node selector is the
+	// master role key, which is the default from the 4.x VPA operator manifests.
+	if pod.Labels[vpaOperatorLabelKey] == vpaOperatorLabelValue &&
+		pod.Namespace == vpaOperatorNamespace && len(pod.Spec.NodeSelector) == 1 {
+		if _, found := pod.Spec.NodeSelector[masterRoleKey]; found {
+			return true
+		}
+	}
+	return false
+}