diff --git a/api/hypershift/v1beta1/azure.go b/api/hypershift/v1beta1/azure.go index d6258444e842..dafc4aa67154 100644 --- a/api/hypershift/v1beta1/azure.go +++ b/api/hypershift/v1beta1/azure.go @@ -469,6 +469,16 @@ type AzurePlatformSpec struct { // +kubebuilder:validation:MaxLength=255 TenantID string `json:"tenantID"` + // containerRegistry configures how worker nodes authenticate to Azure Container Registry (ACR). + // When set, the managed identity is attached to worker virtual machines and its resource ID is + // written into the worker cloud provider config so kubelet's ACR credential provider can + // authenticate without image pull secrets. + // Changing this value will trigger a rollout for all existing NodePools in the cluster. + // + // +rollout + // +optional + ContainerRegistry AzureContainerRegistryConfig `json:"containerRegistry,omitzero"` + // topology specifies the network topology of the API server endpoint for the hosted cluster. // - Public: The API server is accessible only via a public endpoint. // - PublicAndPrivate: The API server is accessible via both public and private endpoints. @@ -546,6 +556,67 @@ type AzureResourceManagedIdentities struct { // +kubebuilder:validation:Pattern=`^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$` type AzureClientID string +// AzureManagedIdentityResourceID is an ARM resource ID for a user-assigned managed identity +// in the format /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{name}. +// +// +kubebuilder:validation:XValidation:rule="self.lowerAscii().matches('^/subscriptions/[^/]+/resourcegroups/[^/]+/providers/microsoft\\\\.managedidentity/userassignedidentities/[^/]+$')",message="must be a user-assigned managed identity ARM resource ID in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}" +// +kubebuilder:validation:MinLength=131 +// +kubebuilder:validation:MaxLength=345 +type AzureManagedIdentityResourceID string + +// UserAssignedManagedIdentity identifies a user-assigned managed identity by its ARM resource ID. +type UserAssignedManagedIdentity struct { + // resourceID is the ARM resource ID of the user-assigned managed identity + // in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}. + // The identity must have the AcrPull role on the target Azure Container Registry. + // It does not need to be in the same subscription or resource group as the HostedCluster, + // but it must be in the same Azure AD tenant. + // + // +required + ResourceID AzureManagedIdentityResourceID `json:"resourceID,omitempty"` +} + +// AzureContainerRegistryConfig configures Azure Container Registry integration for a hosted cluster. +type AzureContainerRegistryConfig struct { + // credentials configures authentication for worker nodes pulling images from ACR + // using a user-assigned managed identity. + // The identity does not need to be in the same subscription or resource group as the + // HostedCluster, but it must be in the same Azure AD tenant. The management cluster's + // CAPZ identity must have Microsoft.ManagedIdentity/userAssignedIdentities/*/assign/action + // on the identity's scope to attach it to worker virtual machines at creation time. + // + // +required + Credentials AzureContainerRegistryCredentialConfig `json:"credentials,omitzero"` +} + +// AzureContainerRegistryCredentialType identifies the type of credential used for ACR image pulls. +// +// +kubebuilder:validation:Enum=ManagedIdentity +type AzureContainerRegistryCredentialType string + +const ( + // AzureContainerRegistryCredentialManagedIdentity uses a user-assigned managed identity for ACR authentication. + AzureContainerRegistryCredentialManagedIdentity AzureContainerRegistryCredentialType = "ManagedIdentity" +) + +// AzureContainerRegistryCredentialConfig configures authentication credentials for Azure Container Registry. +// +// +kubebuilder:validation:XValidation:rule="self.type == 'ManagedIdentity' ? has(self.managedIdentity) : !has(self.managedIdentity)",message="managedIdentity is required when type is ManagedIdentity, and forbidden otherwise" +// +union +type AzureContainerRegistryCredentialConfig struct { + // type specifies the credential type used for ACR image pulls. + // + // +required + // +unionDiscriminator + Type AzureContainerRegistryCredentialType `json:"type,omitempty"` + + // managedIdentity identifies the user-assigned managed identity used for ACR image pulls. + // + // +optional + // +unionMember + ManagedIdentity UserAssignedManagedIdentity `json:"managedIdentity,omitzero"` +} + // AzureWorkloadIdentities is a struct that contains the client IDs of all the managed identities in self-managed Azure // needing to authenticate with Azure's API. type AzureWorkloadIdentities struct { diff --git a/api/hypershift/v1beta1/zz_generated.deepcopy.go b/api/hypershift/v1beta1/zz_generated.deepcopy.go index 3c1718831a7c..2c767b50dfc9 100644 --- a/api/hypershift/v1beta1/zz_generated.deepcopy.go +++ b/api/hypershift/v1beta1/zz_generated.deepcopy.go @@ -637,6 +637,38 @@ func (in *AzureAuthenticationConfiguration) DeepCopy() *AzureAuthenticationConfi return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureContainerRegistryConfig) DeepCopyInto(out *AzureContainerRegistryConfig) { + *out = *in + out.Credentials = in.Credentials +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureContainerRegistryConfig. +func (in *AzureContainerRegistryConfig) DeepCopy() *AzureContainerRegistryConfig { + if in == nil { + return nil + } + out := new(AzureContainerRegistryConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureContainerRegistryCredentialConfig) DeepCopyInto(out *AzureContainerRegistryCredentialConfig) { + *out = *in + out.ManagedIdentity = in.ManagedIdentity +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureContainerRegistryCredentialConfig. +func (in *AzureContainerRegistryCredentialConfig) DeepCopy() *AzureContainerRegistryCredentialConfig { + if in == nil { + return nil + } + out := new(AzureContainerRegistryCredentialConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AzureKMSKey) DeepCopyInto(out *AzureKMSKey) { *out = *in @@ -735,6 +767,7 @@ func (in *AzureNodePoolPlatform) DeepCopy() *AzureNodePoolPlatform { func (in *AzurePlatformSpec) DeepCopyInto(out *AzurePlatformSpec) { *out = *in in.AzureAuthenticationConfig.DeepCopyInto(&out.AzureAuthenticationConfig) + out.ContainerRegistry = in.ContainerRegistry in.Private.DeepCopyInto(&out.Private) } @@ -4709,6 +4742,21 @@ func (in *UnmanagedEtcdSpec) DeepCopy() *UnmanagedEtcdSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *UserAssignedManagedIdentity) DeepCopyInto(out *UserAssignedManagedIdentity) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UserAssignedManagedIdentity. +func (in *UserAssignedManagedIdentity) DeepCopy() *UserAssignedManagedIdentity { + if in == nil { + return nil + } + out := new(UserAssignedManagedIdentity) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *UserManagedDiagnostics) DeepCopyInto(out *UserManagedDiagnostics) { *out = *in diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/AAA_ungated.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/AAA_ungated.yaml index 6a43dd36a5b8..3a8e96f75719 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/AAA_ungated.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/AAA_ungated.yaml @@ -4982,6 +4982,61 @@ spec: - AzureGermanCloud - AzureStackCloud type: string + containerRegistry: + description: |- + containerRegistry configures how worker nodes authenticate to Azure Container Registry (ACR). + When set, the managed identity is attached to worker virtual machines and its resource ID is + written into the worker cloud provider config so kubelet's ACR credential provider can + authenticate without image pull secrets. + Changing this value will trigger a rollout for all existing NodePools in the cluster. + properties: + credentials: + description: |- + credentials configures authentication for worker nodes pulling images from ACR + using a user-assigned managed identity. + The identity does not need to be in the same subscription or resource group as the + HostedCluster, but it must be in the same Azure AD tenant. The management cluster's + CAPZ identity must have Microsoft.ManagedIdentity/userAssignedIdentities/*/assign/action + on the identity's scope to attach it to worker virtual machines at creation time. + properties: + managedIdentity: + description: managedIdentity identifies the user-assigned + managed identity used for ACR image pulls. + properties: + resourceID: + description: |- + resourceID is the ARM resource ID of the user-assigned managed identity + in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}. + The identity must have the AcrPull role on the target Azure Container Registry. + It does not need to be in the same subscription or resource group as the HostedCluster, + but it must be in the same Azure AD tenant. + maxLength: 345 + minLength: 131 + type: string + x-kubernetes-validations: + - message: must be a user-assigned managed identity + ARM resource ID in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName} + rule: self.lowerAscii().matches('^/subscriptions/[^/]+/resourcegroups/[^/]+/providers/microsoft\\.managedidentity/userassignedidentities/[^/]+$') + required: + - resourceID + type: object + type: + description: type specifies the credential type used + for ACR image pulls. + enum: + - ManagedIdentity + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: managedIdentity is required when type is ManagedIdentity, + and forbidden otherwise + rule: 'self.type == ''ManagedIdentity'' ? has(self.managedIdentity) + : !has(self.managedIdentity)' + required: + - credentials + type: object location: description: |- location is the Azure region in where all the cloud infrastructure resources will be created. diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/AutoNodeKarpenter.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/AutoNodeKarpenter.yaml index dba29960a1b8..da3eb4810c69 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/AutoNodeKarpenter.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/AutoNodeKarpenter.yaml @@ -5109,6 +5109,61 @@ spec: - AzureGermanCloud - AzureStackCloud type: string + containerRegistry: + description: |- + containerRegistry configures how worker nodes authenticate to Azure Container Registry (ACR). + When set, the managed identity is attached to worker virtual machines and its resource ID is + written into the worker cloud provider config so kubelet's ACR credential provider can + authenticate without image pull secrets. + Changing this value will trigger a rollout for all existing NodePools in the cluster. + properties: + credentials: + description: |- + credentials configures authentication for worker nodes pulling images from ACR + using a user-assigned managed identity. + The identity does not need to be in the same subscription or resource group as the + HostedCluster, but it must be in the same Azure AD tenant. The management cluster's + CAPZ identity must have Microsoft.ManagedIdentity/userAssignedIdentities/*/assign/action + on the identity's scope to attach it to worker virtual machines at creation time. + properties: + managedIdentity: + description: managedIdentity identifies the user-assigned + managed identity used for ACR image pulls. + properties: + resourceID: + description: |- + resourceID is the ARM resource ID of the user-assigned managed identity + in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}. + The identity must have the AcrPull role on the target Azure Container Registry. + It does not need to be in the same subscription or resource group as the HostedCluster, + but it must be in the same Azure AD tenant. + maxLength: 345 + minLength: 131 + type: string + x-kubernetes-validations: + - message: must be a user-assigned managed identity + ARM resource ID in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName} + rule: self.lowerAscii().matches('^/subscriptions/[^/]+/resourcegroups/[^/]+/providers/microsoft\\.managedidentity/userassignedidentities/[^/]+$') + required: + - resourceID + type: object + type: + description: type specifies the credential type used + for ACR image pulls. + enum: + - ManagedIdentity + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: managedIdentity is required when type is ManagedIdentity, + and forbidden otherwise + rule: 'self.type == ''ManagedIdentity'' ? has(self.managedIdentity) + : !has(self.managedIdentity)' + required: + - credentials + type: object location: description: |- location is the Azure region in where all the cloud infrastructure resources will be created. diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ClusterUpdateAcceptRisks.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ClusterUpdateAcceptRisks.yaml index ae33d6d3e07d..d747a7c40d7b 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ClusterUpdateAcceptRisks.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ClusterUpdateAcceptRisks.yaml @@ -4973,6 +4973,61 @@ spec: - AzureGermanCloud - AzureStackCloud type: string + containerRegistry: + description: |- + containerRegistry configures how worker nodes authenticate to Azure Container Registry (ACR). + When set, the managed identity is attached to worker virtual machines and its resource ID is + written into the worker cloud provider config so kubelet's ACR credential provider can + authenticate without image pull secrets. + Changing this value will trigger a rollout for all existing NodePools in the cluster. + properties: + credentials: + description: |- + credentials configures authentication for worker nodes pulling images from ACR + using a user-assigned managed identity. + The identity does not need to be in the same subscription or resource group as the + HostedCluster, but it must be in the same Azure AD tenant. The management cluster's + CAPZ identity must have Microsoft.ManagedIdentity/userAssignedIdentities/*/assign/action + on the identity's scope to attach it to worker virtual machines at creation time. + properties: + managedIdentity: + description: managedIdentity identifies the user-assigned + managed identity used for ACR image pulls. + properties: + resourceID: + description: |- + resourceID is the ARM resource ID of the user-assigned managed identity + in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}. + The identity must have the AcrPull role on the target Azure Container Registry. + It does not need to be in the same subscription or resource group as the HostedCluster, + but it must be in the same Azure AD tenant. + maxLength: 345 + minLength: 131 + type: string + x-kubernetes-validations: + - message: must be a user-assigned managed identity + ARM resource ID in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName} + rule: self.lowerAscii().matches('^/subscriptions/[^/]+/resourcegroups/[^/]+/providers/microsoft\\.managedidentity/userassignedidentities/[^/]+$') + required: + - resourceID + type: object + type: + description: type specifies the credential type used + for ACR image pulls. + enum: + - ManagedIdentity + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: managedIdentity is required when type is ManagedIdentity, + and forbidden otherwise + rule: 'self.type == ''ManagedIdentity'' ? has(self.managedIdentity) + : !has(self.managedIdentity)' + required: + - credentials + type: object location: description: |- location is the Azure region in where all the cloud infrastructure resources will be created. diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ClusterVersionOperatorConfiguration.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ClusterVersionOperatorConfiguration.yaml index c20cf7ab2b3e..ae5d5938e6cf 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ClusterVersionOperatorConfiguration.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ClusterVersionOperatorConfiguration.yaml @@ -4993,6 +4993,61 @@ spec: - AzureGermanCloud - AzureStackCloud type: string + containerRegistry: + description: |- + containerRegistry configures how worker nodes authenticate to Azure Container Registry (ACR). + When set, the managed identity is attached to worker virtual machines and its resource ID is + written into the worker cloud provider config so kubelet's ACR credential provider can + authenticate without image pull secrets. + Changing this value will trigger a rollout for all existing NodePools in the cluster. + properties: + credentials: + description: |- + credentials configures authentication for worker nodes pulling images from ACR + using a user-assigned managed identity. + The identity does not need to be in the same subscription or resource group as the + HostedCluster, but it must be in the same Azure AD tenant. The management cluster's + CAPZ identity must have Microsoft.ManagedIdentity/userAssignedIdentities/*/assign/action + on the identity's scope to attach it to worker virtual machines at creation time. + properties: + managedIdentity: + description: managedIdentity identifies the user-assigned + managed identity used for ACR image pulls. + properties: + resourceID: + description: |- + resourceID is the ARM resource ID of the user-assigned managed identity + in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}. + The identity must have the AcrPull role on the target Azure Container Registry. + It does not need to be in the same subscription or resource group as the HostedCluster, + but it must be in the same Azure AD tenant. + maxLength: 345 + minLength: 131 + type: string + x-kubernetes-validations: + - message: must be a user-assigned managed identity + ARM resource ID in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName} + rule: self.lowerAscii().matches('^/subscriptions/[^/]+/resourcegroups/[^/]+/providers/microsoft\\.managedidentity/userassignedidentities/[^/]+$') + required: + - resourceID + type: object + type: + description: type specifies the credential type used + for ACR image pulls. + enum: + - ManagedIdentity + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: managedIdentity is required when type is ManagedIdentity, + and forbidden otherwise + rule: 'self.type == ''ManagedIdentity'' ? has(self.managedIdentity) + : !has(self.managedIdentity)' + required: + - credentials + type: object location: description: |- location is the Azure region in where all the cloud infrastructure resources will be created. diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ExternalOIDC.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ExternalOIDC.yaml index 7e35088d6ff8..fc956cd9940b 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ExternalOIDC.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ExternalOIDC.yaml @@ -5306,6 +5306,61 @@ spec: - AzureGermanCloud - AzureStackCloud type: string + containerRegistry: + description: |- + containerRegistry configures how worker nodes authenticate to Azure Container Registry (ACR). + When set, the managed identity is attached to worker virtual machines and its resource ID is + written into the worker cloud provider config so kubelet's ACR credential provider can + authenticate without image pull secrets. + Changing this value will trigger a rollout for all existing NodePools in the cluster. + properties: + credentials: + description: |- + credentials configures authentication for worker nodes pulling images from ACR + using a user-assigned managed identity. + The identity does not need to be in the same subscription or resource group as the + HostedCluster, but it must be in the same Azure AD tenant. The management cluster's + CAPZ identity must have Microsoft.ManagedIdentity/userAssignedIdentities/*/assign/action + on the identity's scope to attach it to worker virtual machines at creation time. + properties: + managedIdentity: + description: managedIdentity identifies the user-assigned + managed identity used for ACR image pulls. + properties: + resourceID: + description: |- + resourceID is the ARM resource ID of the user-assigned managed identity + in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}. + The identity must have the AcrPull role on the target Azure Container Registry. + It does not need to be in the same subscription or resource group as the HostedCluster, + but it must be in the same Azure AD tenant. + maxLength: 345 + minLength: 131 + type: string + x-kubernetes-validations: + - message: must be a user-assigned managed identity + ARM resource ID in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName} + rule: self.lowerAscii().matches('^/subscriptions/[^/]+/resourcegroups/[^/]+/providers/microsoft\\.managedidentity/userassignedidentities/[^/]+$') + required: + - resourceID + type: object + type: + description: type specifies the credential type used + for ACR image pulls. + enum: + - ManagedIdentity + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: managedIdentity is required when type is ManagedIdentity, + and forbidden otherwise + rule: 'self.type == ''ManagedIdentity'' ? has(self.managedIdentity) + : !has(self.managedIdentity)' + required: + - credentials + type: object location: description: |- location is the Azure region in where all the cloud infrastructure resources will be created. diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ExternalOIDCWithUIDAndExtraClaimMappings.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ExternalOIDCWithUIDAndExtraClaimMappings.yaml index 9693ed970d74..5c50b3b4be20 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ExternalOIDCWithUIDAndExtraClaimMappings.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ExternalOIDCWithUIDAndExtraClaimMappings.yaml @@ -5446,6 +5446,61 @@ spec: - AzureGermanCloud - AzureStackCloud type: string + containerRegistry: + description: |- + containerRegistry configures how worker nodes authenticate to Azure Container Registry (ACR). + When set, the managed identity is attached to worker virtual machines and its resource ID is + written into the worker cloud provider config so kubelet's ACR credential provider can + authenticate without image pull secrets. + Changing this value will trigger a rollout for all existing NodePools in the cluster. + properties: + credentials: + description: |- + credentials configures authentication for worker nodes pulling images from ACR + using a user-assigned managed identity. + The identity does not need to be in the same subscription or resource group as the + HostedCluster, but it must be in the same Azure AD tenant. The management cluster's + CAPZ identity must have Microsoft.ManagedIdentity/userAssignedIdentities/*/assign/action + on the identity's scope to attach it to worker virtual machines at creation time. + properties: + managedIdentity: + description: managedIdentity identifies the user-assigned + managed identity used for ACR image pulls. + properties: + resourceID: + description: |- + resourceID is the ARM resource ID of the user-assigned managed identity + in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}. + The identity must have the AcrPull role on the target Azure Container Registry. + It does not need to be in the same subscription or resource group as the HostedCluster, + but it must be in the same Azure AD tenant. + maxLength: 345 + minLength: 131 + type: string + x-kubernetes-validations: + - message: must be a user-assigned managed identity + ARM resource ID in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName} + rule: self.lowerAscii().matches('^/subscriptions/[^/]+/resourcegroups/[^/]+/providers/microsoft\\.managedidentity/userassignedidentities/[^/]+$') + required: + - resourceID + type: object + type: + description: type specifies the credential type used + for ACR image pulls. + enum: + - ManagedIdentity + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: managedIdentity is required when type is ManagedIdentity, + and forbidden otherwise + rule: 'self.type == ''ManagedIdentity'' ? has(self.managedIdentity) + : !has(self.managedIdentity)' + required: + - credentials + type: object location: description: |- location is the Azure region in where all the cloud infrastructure resources will be created. diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ExternalOIDCWithUpstreamParity.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ExternalOIDCWithUpstreamParity.yaml index ad2f5eb3c432..a498f636735b 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ExternalOIDCWithUpstreamParity.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ExternalOIDCWithUpstreamParity.yaml @@ -5427,6 +5427,61 @@ spec: - AzureGermanCloud - AzureStackCloud type: string + containerRegistry: + description: |- + containerRegistry configures how worker nodes authenticate to Azure Container Registry (ACR). + When set, the managed identity is attached to worker virtual machines and its resource ID is + written into the worker cloud provider config so kubelet's ACR credential provider can + authenticate without image pull secrets. + Changing this value will trigger a rollout for all existing NodePools in the cluster. + properties: + credentials: + description: |- + credentials configures authentication for worker nodes pulling images from ACR + using a user-assigned managed identity. + The identity does not need to be in the same subscription or resource group as the + HostedCluster, but it must be in the same Azure AD tenant. The management cluster's + CAPZ identity must have Microsoft.ManagedIdentity/userAssignedIdentities/*/assign/action + on the identity's scope to attach it to worker virtual machines at creation time. + properties: + managedIdentity: + description: managedIdentity identifies the user-assigned + managed identity used for ACR image pulls. + properties: + resourceID: + description: |- + resourceID is the ARM resource ID of the user-assigned managed identity + in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}. + The identity must have the AcrPull role on the target Azure Container Registry. + It does not need to be in the same subscription or resource group as the HostedCluster, + but it must be in the same Azure AD tenant. + maxLength: 345 + minLength: 131 + type: string + x-kubernetes-validations: + - message: must be a user-assigned managed identity + ARM resource ID in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName} + rule: self.lowerAscii().matches('^/subscriptions/[^/]+/resourcegroups/[^/]+/providers/microsoft\\.managedidentity/userassignedidentities/[^/]+$') + required: + - resourceID + type: object + type: + description: type specifies the credential type used + for ACR image pulls. + enum: + - ManagedIdentity + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: managedIdentity is required when type is ManagedIdentity, + and forbidden otherwise + rule: 'self.type == ''ManagedIdentity'' ? has(self.managedIdentity) + : !has(self.managedIdentity)' + required: + - credentials + type: object location: description: |- location is the Azure region in where all the cloud infrastructure resources will be created. diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/GCPPlatform.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/GCPPlatform.yaml index e925ed7ee0a9..c1aa77bbf9c8 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/GCPPlatform.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/GCPPlatform.yaml @@ -4973,6 +4973,61 @@ spec: - AzureGermanCloud - AzureStackCloud type: string + containerRegistry: + description: |- + containerRegistry configures how worker nodes authenticate to Azure Container Registry (ACR). + When set, the managed identity is attached to worker virtual machines and its resource ID is + written into the worker cloud provider config so kubelet's ACR credential provider can + authenticate without image pull secrets. + Changing this value will trigger a rollout for all existing NodePools in the cluster. + properties: + credentials: + description: |- + credentials configures authentication for worker nodes pulling images from ACR + using a user-assigned managed identity. + The identity does not need to be in the same subscription or resource group as the + HostedCluster, but it must be in the same Azure AD tenant. The management cluster's + CAPZ identity must have Microsoft.ManagedIdentity/userAssignedIdentities/*/assign/action + on the identity's scope to attach it to worker virtual machines at creation time. + properties: + managedIdentity: + description: managedIdentity identifies the user-assigned + managed identity used for ACR image pulls. + properties: + resourceID: + description: |- + resourceID is the ARM resource ID of the user-assigned managed identity + in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}. + The identity must have the AcrPull role on the target Azure Container Registry. + It does not need to be in the same subscription or resource group as the HostedCluster, + but it must be in the same Azure AD tenant. + maxLength: 345 + minLength: 131 + type: string + x-kubernetes-validations: + - message: must be a user-assigned managed identity + ARM resource ID in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName} + rule: self.lowerAscii().matches('^/subscriptions/[^/]+/resourcegroups/[^/]+/providers/microsoft\\.managedidentity/userassignedidentities/[^/]+$') + required: + - resourceID + type: object + type: + description: type specifies the credential type used + for ACR image pulls. + enum: + - ManagedIdentity + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: managedIdentity is required when type is ManagedIdentity, + and forbidden otherwise + rule: 'self.type == ''ManagedIdentity'' ? has(self.managedIdentity) + : !has(self.managedIdentity)' + required: + - credentials + type: object location: description: |- location is the Azure region in where all the cloud infrastructure resources will be created. diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/HCPEtcdBackup.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/HCPEtcdBackup.yaml index 8c514b6f9470..3524c1d59c2e 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/HCPEtcdBackup.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/HCPEtcdBackup.yaml @@ -5038,6 +5038,61 @@ spec: - AzureGermanCloud - AzureStackCloud type: string + containerRegistry: + description: |- + containerRegistry configures how worker nodes authenticate to Azure Container Registry (ACR). + When set, the managed identity is attached to worker virtual machines and its resource ID is + written into the worker cloud provider config so kubelet's ACR credential provider can + authenticate without image pull secrets. + Changing this value will trigger a rollout for all existing NodePools in the cluster. + properties: + credentials: + description: |- + credentials configures authentication for worker nodes pulling images from ACR + using a user-assigned managed identity. + The identity does not need to be in the same subscription or resource group as the + HostedCluster, but it must be in the same Azure AD tenant. The management cluster's + CAPZ identity must have Microsoft.ManagedIdentity/userAssignedIdentities/*/assign/action + on the identity's scope to attach it to worker virtual machines at creation time. + properties: + managedIdentity: + description: managedIdentity identifies the user-assigned + managed identity used for ACR image pulls. + properties: + resourceID: + description: |- + resourceID is the ARM resource ID of the user-assigned managed identity + in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}. + The identity must have the AcrPull role on the target Azure Container Registry. + It does not need to be in the same subscription or resource group as the HostedCluster, + but it must be in the same Azure AD tenant. + maxLength: 345 + minLength: 131 + type: string + x-kubernetes-validations: + - message: must be a user-assigned managed identity + ARM resource ID in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName} + rule: self.lowerAscii().matches('^/subscriptions/[^/]+/resourcegroups/[^/]+/providers/microsoft\\.managedidentity/userassignedidentities/[^/]+$') + required: + - resourceID + type: object + type: + description: type specifies the credential type used + for ACR image pulls. + enum: + - ManagedIdentity + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: managedIdentity is required when type is ManagedIdentity, + and forbidden otherwise + rule: 'self.type == ''ManagedIdentity'' ? has(self.managedIdentity) + : !has(self.managedIdentity)' + required: + - credentials + type: object location: description: |- location is the Azure region in where all the cloud infrastructure resources will be created. diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/HyperShiftOnlyDynamicResourceAllocation.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/HyperShiftOnlyDynamicResourceAllocation.yaml index 0d88e73b6802..1583d8314322 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/HyperShiftOnlyDynamicResourceAllocation.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/HyperShiftOnlyDynamicResourceAllocation.yaml @@ -4995,6 +4995,61 @@ spec: - AzureGermanCloud - AzureStackCloud type: string + containerRegistry: + description: |- + containerRegistry configures how worker nodes authenticate to Azure Container Registry (ACR). + When set, the managed identity is attached to worker virtual machines and its resource ID is + written into the worker cloud provider config so kubelet's ACR credential provider can + authenticate without image pull secrets. + Changing this value will trigger a rollout for all existing NodePools in the cluster. + properties: + credentials: + description: |- + credentials configures authentication for worker nodes pulling images from ACR + using a user-assigned managed identity. + The identity does not need to be in the same subscription or resource group as the + HostedCluster, but it must be in the same Azure AD tenant. The management cluster's + CAPZ identity must have Microsoft.ManagedIdentity/userAssignedIdentities/*/assign/action + on the identity's scope to attach it to worker virtual machines at creation time. + properties: + managedIdentity: + description: managedIdentity identifies the user-assigned + managed identity used for ACR image pulls. + properties: + resourceID: + description: |- + resourceID is the ARM resource ID of the user-assigned managed identity + in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}. + The identity must have the AcrPull role on the target Azure Container Registry. + It does not need to be in the same subscription or resource group as the HostedCluster, + but it must be in the same Azure AD tenant. + maxLength: 345 + minLength: 131 + type: string + x-kubernetes-validations: + - message: must be a user-assigned managed identity + ARM resource ID in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName} + rule: self.lowerAscii().matches('^/subscriptions/[^/]+/resourcegroups/[^/]+/providers/microsoft\\.managedidentity/userassignedidentities/[^/]+$') + required: + - resourceID + type: object + type: + description: type specifies the credential type used + for ACR image pulls. + enum: + - ManagedIdentity + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: managedIdentity is required when type is ManagedIdentity, + and forbidden otherwise + rule: 'self.type == ''ManagedIdentity'' ? has(self.managedIdentity) + : !has(self.managedIdentity)' + required: + - credentials + type: object location: description: |- location is the Azure region in where all the cloud infrastructure resources will be created. diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ImageStreamImportMode.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ImageStreamImportMode.yaml index 03bf685635d1..b690aa40f34d 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ImageStreamImportMode.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ImageStreamImportMode.yaml @@ -4991,6 +4991,61 @@ spec: - AzureGermanCloud - AzureStackCloud type: string + containerRegistry: + description: |- + containerRegistry configures how worker nodes authenticate to Azure Container Registry (ACR). + When set, the managed identity is attached to worker virtual machines and its resource ID is + written into the worker cloud provider config so kubelet's ACR credential provider can + authenticate without image pull secrets. + Changing this value will trigger a rollout for all existing NodePools in the cluster. + properties: + credentials: + description: |- + credentials configures authentication for worker nodes pulling images from ACR + using a user-assigned managed identity. + The identity does not need to be in the same subscription or resource group as the + HostedCluster, but it must be in the same Azure AD tenant. The management cluster's + CAPZ identity must have Microsoft.ManagedIdentity/userAssignedIdentities/*/assign/action + on the identity's scope to attach it to worker virtual machines at creation time. + properties: + managedIdentity: + description: managedIdentity identifies the user-assigned + managed identity used for ACR image pulls. + properties: + resourceID: + description: |- + resourceID is the ARM resource ID of the user-assigned managed identity + in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}. + The identity must have the AcrPull role on the target Azure Container Registry. + It does not need to be in the same subscription or resource group as the HostedCluster, + but it must be in the same Azure AD tenant. + maxLength: 345 + minLength: 131 + type: string + x-kubernetes-validations: + - message: must be a user-assigned managed identity + ARM resource ID in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName} + rule: self.lowerAscii().matches('^/subscriptions/[^/]+/resourcegroups/[^/]+/providers/microsoft\\.managedidentity/userassignedidentities/[^/]+$') + required: + - resourceID + type: object + type: + description: type specifies the credential type used + for ACR image pulls. + enum: + - ManagedIdentity + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: managedIdentity is required when type is ManagedIdentity, + and forbidden otherwise + rule: 'self.type == ''ManagedIdentity'' ? has(self.managedIdentity) + : !has(self.managedIdentity)' + required: + - credentials + type: object location: description: |- location is the Azure region in where all the cloud infrastructure resources will be created. diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/KMSEncryptionProvider.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/KMSEncryptionProvider.yaml index cc37823e4884..d3d9b2cd0d51 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/KMSEncryptionProvider.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/KMSEncryptionProvider.yaml @@ -5049,6 +5049,61 @@ spec: - AzureGermanCloud - AzureStackCloud type: string + containerRegistry: + description: |- + containerRegistry configures how worker nodes authenticate to Azure Container Registry (ACR). + When set, the managed identity is attached to worker virtual machines and its resource ID is + written into the worker cloud provider config so kubelet's ACR credential provider can + authenticate without image pull secrets. + Changing this value will trigger a rollout for all existing NodePools in the cluster. + properties: + credentials: + description: |- + credentials configures authentication for worker nodes pulling images from ACR + using a user-assigned managed identity. + The identity does not need to be in the same subscription or resource group as the + HostedCluster, but it must be in the same Azure AD tenant. The management cluster's + CAPZ identity must have Microsoft.ManagedIdentity/userAssignedIdentities/*/assign/action + on the identity's scope to attach it to worker virtual machines at creation time. + properties: + managedIdentity: + description: managedIdentity identifies the user-assigned + managed identity used for ACR image pulls. + properties: + resourceID: + description: |- + resourceID is the ARM resource ID of the user-assigned managed identity + in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}. + The identity must have the AcrPull role on the target Azure Container Registry. + It does not need to be in the same subscription or resource group as the HostedCluster, + but it must be in the same Azure AD tenant. + maxLength: 345 + minLength: 131 + type: string + x-kubernetes-validations: + - message: must be a user-assigned managed identity + ARM resource ID in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName} + rule: self.lowerAscii().matches('^/subscriptions/[^/]+/resourcegroups/[^/]+/providers/microsoft\\.managedidentity/userassignedidentities/[^/]+$') + required: + - resourceID + type: object + type: + description: type specifies the credential type used + for ACR image pulls. + enum: + - ManagedIdentity + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: managedIdentity is required when type is ManagedIdentity, + and forbidden otherwise + rule: 'self.type == ''ManagedIdentity'' ? has(self.managedIdentity) + : !has(self.managedIdentity)' + required: + - credentials + type: object location: description: |- location is the Azure region in where all the cloud infrastructure resources will be created. diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/OpenStack.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/OpenStack.yaml index 078863df10ed..5fbaad0d86da 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/OpenStack.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/OpenStack.yaml @@ -4973,6 +4973,61 @@ spec: - AzureGermanCloud - AzureStackCloud type: string + containerRegistry: + description: |- + containerRegistry configures how worker nodes authenticate to Azure Container Registry (ACR). + When set, the managed identity is attached to worker virtual machines and its resource ID is + written into the worker cloud provider config so kubelet's ACR credential provider can + authenticate without image pull secrets. + Changing this value will trigger a rollout for all existing NodePools in the cluster. + properties: + credentials: + description: |- + credentials configures authentication for worker nodes pulling images from ACR + using a user-assigned managed identity. + The identity does not need to be in the same subscription or resource group as the + HostedCluster, but it must be in the same Azure AD tenant. The management cluster's + CAPZ identity must have Microsoft.ManagedIdentity/userAssignedIdentities/*/assign/action + on the identity's scope to attach it to worker virtual machines at creation time. + properties: + managedIdentity: + description: managedIdentity identifies the user-assigned + managed identity used for ACR image pulls. + properties: + resourceID: + description: |- + resourceID is the ARM resource ID of the user-assigned managed identity + in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}. + The identity must have the AcrPull role on the target Azure Container Registry. + It does not need to be in the same subscription or resource group as the HostedCluster, + but it must be in the same Azure AD tenant. + maxLength: 345 + minLength: 131 + type: string + x-kubernetes-validations: + - message: must be a user-assigned managed identity + ARM resource ID in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName} + rule: self.lowerAscii().matches('^/subscriptions/[^/]+/resourcegroups/[^/]+/providers/microsoft\\.managedidentity/userassignedidentities/[^/]+$') + required: + - resourceID + type: object + type: + description: type specifies the credential type used + for ACR image pulls. + enum: + - ManagedIdentity + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: managedIdentity is required when type is ManagedIdentity, + and forbidden otherwise + rule: 'self.type == ''ManagedIdentity'' ? has(self.managedIdentity) + : !has(self.managedIdentity)' + required: + - credentials + type: object location: description: |- location is the Azure region in where all the cloud infrastructure resources will be created. diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/AAA_ungated.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/AAA_ungated.yaml index a761964daf44..547a157e8595 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/AAA_ungated.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/AAA_ungated.yaml @@ -4860,6 +4860,61 @@ spec: - AzureGermanCloud - AzureStackCloud type: string + containerRegistry: + description: |- + containerRegistry configures how worker nodes authenticate to Azure Container Registry (ACR). + When set, the managed identity is attached to worker virtual machines and its resource ID is + written into the worker cloud provider config so kubelet's ACR credential provider can + authenticate without image pull secrets. + Changing this value will trigger a rollout for all existing NodePools in the cluster. + properties: + credentials: + description: |- + credentials configures authentication for worker nodes pulling images from ACR + using a user-assigned managed identity. + The identity does not need to be in the same subscription or resource group as the + HostedCluster, but it must be in the same Azure AD tenant. The management cluster's + CAPZ identity must have Microsoft.ManagedIdentity/userAssignedIdentities/*/assign/action + on the identity's scope to attach it to worker virtual machines at creation time. + properties: + managedIdentity: + description: managedIdentity identifies the user-assigned + managed identity used for ACR image pulls. + properties: + resourceID: + description: |- + resourceID is the ARM resource ID of the user-assigned managed identity + in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}. + The identity must have the AcrPull role on the target Azure Container Registry. + It does not need to be in the same subscription or resource group as the HostedCluster, + but it must be in the same Azure AD tenant. + maxLength: 345 + minLength: 131 + type: string + x-kubernetes-validations: + - message: must be a user-assigned managed identity + ARM resource ID in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName} + rule: self.lowerAscii().matches('^/subscriptions/[^/]+/resourcegroups/[^/]+/providers/microsoft\\.managedidentity/userassignedidentities/[^/]+$') + required: + - resourceID + type: object + type: + description: type specifies the credential type used + for ACR image pulls. + enum: + - ManagedIdentity + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: managedIdentity is required when type is ManagedIdentity, + and forbidden otherwise + rule: 'self.type == ''ManagedIdentity'' ? has(self.managedIdentity) + : !has(self.managedIdentity)' + required: + - credentials + type: object location: description: |- location is the Azure region in where all the cloud infrastructure resources will be created. diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/AutoNodeKarpenter.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/AutoNodeKarpenter.yaml index eaec5504e031..f69a31f26b4a 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/AutoNodeKarpenter.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/AutoNodeKarpenter.yaml @@ -4989,6 +4989,61 @@ spec: - AzureGermanCloud - AzureStackCloud type: string + containerRegistry: + description: |- + containerRegistry configures how worker nodes authenticate to Azure Container Registry (ACR). + When set, the managed identity is attached to worker virtual machines and its resource ID is + written into the worker cloud provider config so kubelet's ACR credential provider can + authenticate without image pull secrets. + Changing this value will trigger a rollout for all existing NodePools in the cluster. + properties: + credentials: + description: |- + credentials configures authentication for worker nodes pulling images from ACR + using a user-assigned managed identity. + The identity does not need to be in the same subscription or resource group as the + HostedCluster, but it must be in the same Azure AD tenant. The management cluster's + CAPZ identity must have Microsoft.ManagedIdentity/userAssignedIdentities/*/assign/action + on the identity's scope to attach it to worker virtual machines at creation time. + properties: + managedIdentity: + description: managedIdentity identifies the user-assigned + managed identity used for ACR image pulls. + properties: + resourceID: + description: |- + resourceID is the ARM resource ID of the user-assigned managed identity + in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}. + The identity must have the AcrPull role on the target Azure Container Registry. + It does not need to be in the same subscription or resource group as the HostedCluster, + but it must be in the same Azure AD tenant. + maxLength: 345 + minLength: 131 + type: string + x-kubernetes-validations: + - message: must be a user-assigned managed identity + ARM resource ID in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName} + rule: self.lowerAscii().matches('^/subscriptions/[^/]+/resourcegroups/[^/]+/providers/microsoft\\.managedidentity/userassignedidentities/[^/]+$') + required: + - resourceID + type: object + type: + description: type specifies the credential type used + for ACR image pulls. + enum: + - ManagedIdentity + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: managedIdentity is required when type is ManagedIdentity, + and forbidden otherwise + rule: 'self.type == ''ManagedIdentity'' ? has(self.managedIdentity) + : !has(self.managedIdentity)' + required: + - credentials + type: object location: description: |- location is the Azure region in where all the cloud infrastructure resources will be created. diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ClusterUpdateAcceptRisks.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ClusterUpdateAcceptRisks.yaml index 29801153bfd3..a2ab38c1a131 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ClusterUpdateAcceptRisks.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ClusterUpdateAcceptRisks.yaml @@ -4851,6 +4851,61 @@ spec: - AzureGermanCloud - AzureStackCloud type: string + containerRegistry: + description: |- + containerRegistry configures how worker nodes authenticate to Azure Container Registry (ACR). + When set, the managed identity is attached to worker virtual machines and its resource ID is + written into the worker cloud provider config so kubelet's ACR credential provider can + authenticate without image pull secrets. + Changing this value will trigger a rollout for all existing NodePools in the cluster. + properties: + credentials: + description: |- + credentials configures authentication for worker nodes pulling images from ACR + using a user-assigned managed identity. + The identity does not need to be in the same subscription or resource group as the + HostedCluster, but it must be in the same Azure AD tenant. The management cluster's + CAPZ identity must have Microsoft.ManagedIdentity/userAssignedIdentities/*/assign/action + on the identity's scope to attach it to worker virtual machines at creation time. + properties: + managedIdentity: + description: managedIdentity identifies the user-assigned + managed identity used for ACR image pulls. + properties: + resourceID: + description: |- + resourceID is the ARM resource ID of the user-assigned managed identity + in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}. + The identity must have the AcrPull role on the target Azure Container Registry. + It does not need to be in the same subscription or resource group as the HostedCluster, + but it must be in the same Azure AD tenant. + maxLength: 345 + minLength: 131 + type: string + x-kubernetes-validations: + - message: must be a user-assigned managed identity + ARM resource ID in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName} + rule: self.lowerAscii().matches('^/subscriptions/[^/]+/resourcegroups/[^/]+/providers/microsoft\\.managedidentity/userassignedidentities/[^/]+$') + required: + - resourceID + type: object + type: + description: type specifies the credential type used + for ACR image pulls. + enum: + - ManagedIdentity + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: managedIdentity is required when type is ManagedIdentity, + and forbidden otherwise + rule: 'self.type == ''ManagedIdentity'' ? has(self.managedIdentity) + : !has(self.managedIdentity)' + required: + - credentials + type: object location: description: |- location is the Azure region in where all the cloud infrastructure resources will be created. diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ClusterVersionOperatorConfiguration.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ClusterVersionOperatorConfiguration.yaml index babd4ac02c0b..91bfdff386b2 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ClusterVersionOperatorConfiguration.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ClusterVersionOperatorConfiguration.yaml @@ -4871,6 +4871,61 @@ spec: - AzureGermanCloud - AzureStackCloud type: string + containerRegistry: + description: |- + containerRegistry configures how worker nodes authenticate to Azure Container Registry (ACR). + When set, the managed identity is attached to worker virtual machines and its resource ID is + written into the worker cloud provider config so kubelet's ACR credential provider can + authenticate without image pull secrets. + Changing this value will trigger a rollout for all existing NodePools in the cluster. + properties: + credentials: + description: |- + credentials configures authentication for worker nodes pulling images from ACR + using a user-assigned managed identity. + The identity does not need to be in the same subscription or resource group as the + HostedCluster, but it must be in the same Azure AD tenant. The management cluster's + CAPZ identity must have Microsoft.ManagedIdentity/userAssignedIdentities/*/assign/action + on the identity's scope to attach it to worker virtual machines at creation time. + properties: + managedIdentity: + description: managedIdentity identifies the user-assigned + managed identity used for ACR image pulls. + properties: + resourceID: + description: |- + resourceID is the ARM resource ID of the user-assigned managed identity + in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}. + The identity must have the AcrPull role on the target Azure Container Registry. + It does not need to be in the same subscription or resource group as the HostedCluster, + but it must be in the same Azure AD tenant. + maxLength: 345 + minLength: 131 + type: string + x-kubernetes-validations: + - message: must be a user-assigned managed identity + ARM resource ID in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName} + rule: self.lowerAscii().matches('^/subscriptions/[^/]+/resourcegroups/[^/]+/providers/microsoft\\.managedidentity/userassignedidentities/[^/]+$') + required: + - resourceID + type: object + type: + description: type specifies the credential type used + for ACR image pulls. + enum: + - ManagedIdentity + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: managedIdentity is required when type is ManagedIdentity, + and forbidden otherwise + rule: 'self.type == ''ManagedIdentity'' ? has(self.managedIdentity) + : !has(self.managedIdentity)' + required: + - credentials + type: object location: description: |- location is the Azure region in where all the cloud infrastructure resources will be created. diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ExternalOIDC.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ExternalOIDC.yaml index c01534ee08f4..792be100d6c6 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ExternalOIDC.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ExternalOIDC.yaml @@ -5184,6 +5184,61 @@ spec: - AzureGermanCloud - AzureStackCloud type: string + containerRegistry: + description: |- + containerRegistry configures how worker nodes authenticate to Azure Container Registry (ACR). + When set, the managed identity is attached to worker virtual machines and its resource ID is + written into the worker cloud provider config so kubelet's ACR credential provider can + authenticate without image pull secrets. + Changing this value will trigger a rollout for all existing NodePools in the cluster. + properties: + credentials: + description: |- + credentials configures authentication for worker nodes pulling images from ACR + using a user-assigned managed identity. + The identity does not need to be in the same subscription or resource group as the + HostedCluster, but it must be in the same Azure AD tenant. The management cluster's + CAPZ identity must have Microsoft.ManagedIdentity/userAssignedIdentities/*/assign/action + on the identity's scope to attach it to worker virtual machines at creation time. + properties: + managedIdentity: + description: managedIdentity identifies the user-assigned + managed identity used for ACR image pulls. + properties: + resourceID: + description: |- + resourceID is the ARM resource ID of the user-assigned managed identity + in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}. + The identity must have the AcrPull role on the target Azure Container Registry. + It does not need to be in the same subscription or resource group as the HostedCluster, + but it must be in the same Azure AD tenant. + maxLength: 345 + minLength: 131 + type: string + x-kubernetes-validations: + - message: must be a user-assigned managed identity + ARM resource ID in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName} + rule: self.lowerAscii().matches('^/subscriptions/[^/]+/resourcegroups/[^/]+/providers/microsoft\\.managedidentity/userassignedidentities/[^/]+$') + required: + - resourceID + type: object + type: + description: type specifies the credential type used + for ACR image pulls. + enum: + - ManagedIdentity + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: managedIdentity is required when type is ManagedIdentity, + and forbidden otherwise + rule: 'self.type == ''ManagedIdentity'' ? has(self.managedIdentity) + : !has(self.managedIdentity)' + required: + - credentials + type: object location: description: |- location is the Azure region in where all the cloud infrastructure resources will be created. diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ExternalOIDCWithUIDAndExtraClaimMappings.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ExternalOIDCWithUIDAndExtraClaimMappings.yaml index e5deab3ce474..c37a1b5b4e3f 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ExternalOIDCWithUIDAndExtraClaimMappings.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ExternalOIDCWithUIDAndExtraClaimMappings.yaml @@ -5324,6 +5324,61 @@ spec: - AzureGermanCloud - AzureStackCloud type: string + containerRegistry: + description: |- + containerRegistry configures how worker nodes authenticate to Azure Container Registry (ACR). + When set, the managed identity is attached to worker virtual machines and its resource ID is + written into the worker cloud provider config so kubelet's ACR credential provider can + authenticate without image pull secrets. + Changing this value will trigger a rollout for all existing NodePools in the cluster. + properties: + credentials: + description: |- + credentials configures authentication for worker nodes pulling images from ACR + using a user-assigned managed identity. + The identity does not need to be in the same subscription or resource group as the + HostedCluster, but it must be in the same Azure AD tenant. The management cluster's + CAPZ identity must have Microsoft.ManagedIdentity/userAssignedIdentities/*/assign/action + on the identity's scope to attach it to worker virtual machines at creation time. + properties: + managedIdentity: + description: managedIdentity identifies the user-assigned + managed identity used for ACR image pulls. + properties: + resourceID: + description: |- + resourceID is the ARM resource ID of the user-assigned managed identity + in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}. + The identity must have the AcrPull role on the target Azure Container Registry. + It does not need to be in the same subscription or resource group as the HostedCluster, + but it must be in the same Azure AD tenant. + maxLength: 345 + minLength: 131 + type: string + x-kubernetes-validations: + - message: must be a user-assigned managed identity + ARM resource ID in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName} + rule: self.lowerAscii().matches('^/subscriptions/[^/]+/resourcegroups/[^/]+/providers/microsoft\\.managedidentity/userassignedidentities/[^/]+$') + required: + - resourceID + type: object + type: + description: type specifies the credential type used + for ACR image pulls. + enum: + - ManagedIdentity + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: managedIdentity is required when type is ManagedIdentity, + and forbidden otherwise + rule: 'self.type == ''ManagedIdentity'' ? has(self.managedIdentity) + : !has(self.managedIdentity)' + required: + - credentials + type: object location: description: |- location is the Azure region in where all the cloud infrastructure resources will be created. diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ExternalOIDCWithUpstreamParity.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ExternalOIDCWithUpstreamParity.yaml index 28f733262692..d5ea61348ca6 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ExternalOIDCWithUpstreamParity.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ExternalOIDCWithUpstreamParity.yaml @@ -5305,6 +5305,61 @@ spec: - AzureGermanCloud - AzureStackCloud type: string + containerRegistry: + description: |- + containerRegistry configures how worker nodes authenticate to Azure Container Registry (ACR). + When set, the managed identity is attached to worker virtual machines and its resource ID is + written into the worker cloud provider config so kubelet's ACR credential provider can + authenticate without image pull secrets. + Changing this value will trigger a rollout for all existing NodePools in the cluster. + properties: + credentials: + description: |- + credentials configures authentication for worker nodes pulling images from ACR + using a user-assigned managed identity. + The identity does not need to be in the same subscription or resource group as the + HostedCluster, but it must be in the same Azure AD tenant. The management cluster's + CAPZ identity must have Microsoft.ManagedIdentity/userAssignedIdentities/*/assign/action + on the identity's scope to attach it to worker virtual machines at creation time. + properties: + managedIdentity: + description: managedIdentity identifies the user-assigned + managed identity used for ACR image pulls. + properties: + resourceID: + description: |- + resourceID is the ARM resource ID of the user-assigned managed identity + in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}. + The identity must have the AcrPull role on the target Azure Container Registry. + It does not need to be in the same subscription or resource group as the HostedCluster, + but it must be in the same Azure AD tenant. + maxLength: 345 + minLength: 131 + type: string + x-kubernetes-validations: + - message: must be a user-assigned managed identity + ARM resource ID in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName} + rule: self.lowerAscii().matches('^/subscriptions/[^/]+/resourcegroups/[^/]+/providers/microsoft\\.managedidentity/userassignedidentities/[^/]+$') + required: + - resourceID + type: object + type: + description: type specifies the credential type used + for ACR image pulls. + enum: + - ManagedIdentity + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: managedIdentity is required when type is ManagedIdentity, + and forbidden otherwise + rule: 'self.type == ''ManagedIdentity'' ? has(self.managedIdentity) + : !has(self.managedIdentity)' + required: + - credentials + type: object location: description: |- location is the Azure region in where all the cloud infrastructure resources will be created. diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/GCPPlatform.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/GCPPlatform.yaml index c91707459fdc..c1f2db6c3050 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/GCPPlatform.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/GCPPlatform.yaml @@ -4851,6 +4851,61 @@ spec: - AzureGermanCloud - AzureStackCloud type: string + containerRegistry: + description: |- + containerRegistry configures how worker nodes authenticate to Azure Container Registry (ACR). + When set, the managed identity is attached to worker virtual machines and its resource ID is + written into the worker cloud provider config so kubelet's ACR credential provider can + authenticate without image pull secrets. + Changing this value will trigger a rollout for all existing NodePools in the cluster. + properties: + credentials: + description: |- + credentials configures authentication for worker nodes pulling images from ACR + using a user-assigned managed identity. + The identity does not need to be in the same subscription or resource group as the + HostedCluster, but it must be in the same Azure AD tenant. The management cluster's + CAPZ identity must have Microsoft.ManagedIdentity/userAssignedIdentities/*/assign/action + on the identity's scope to attach it to worker virtual machines at creation time. + properties: + managedIdentity: + description: managedIdentity identifies the user-assigned + managed identity used for ACR image pulls. + properties: + resourceID: + description: |- + resourceID is the ARM resource ID of the user-assigned managed identity + in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}. + The identity must have the AcrPull role on the target Azure Container Registry. + It does not need to be in the same subscription or resource group as the HostedCluster, + but it must be in the same Azure AD tenant. + maxLength: 345 + minLength: 131 + type: string + x-kubernetes-validations: + - message: must be a user-assigned managed identity + ARM resource ID in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName} + rule: self.lowerAscii().matches('^/subscriptions/[^/]+/resourcegroups/[^/]+/providers/microsoft\\.managedidentity/userassignedidentities/[^/]+$') + required: + - resourceID + type: object + type: + description: type specifies the credential type used + for ACR image pulls. + enum: + - ManagedIdentity + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: managedIdentity is required when type is ManagedIdentity, + and forbidden otherwise + rule: 'self.type == ''ManagedIdentity'' ? has(self.managedIdentity) + : !has(self.managedIdentity)' + required: + - credentials + type: object location: description: |- location is the Azure region in where all the cloud infrastructure resources will be created. diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/HCPEtcdBackup.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/HCPEtcdBackup.yaml index 8edd8df4ca2a..6d3189021df0 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/HCPEtcdBackup.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/HCPEtcdBackup.yaml @@ -4916,6 +4916,61 @@ spec: - AzureGermanCloud - AzureStackCloud type: string + containerRegistry: + description: |- + containerRegistry configures how worker nodes authenticate to Azure Container Registry (ACR). + When set, the managed identity is attached to worker virtual machines and its resource ID is + written into the worker cloud provider config so kubelet's ACR credential provider can + authenticate without image pull secrets. + Changing this value will trigger a rollout for all existing NodePools in the cluster. + properties: + credentials: + description: |- + credentials configures authentication for worker nodes pulling images from ACR + using a user-assigned managed identity. + The identity does not need to be in the same subscription or resource group as the + HostedCluster, but it must be in the same Azure AD tenant. The management cluster's + CAPZ identity must have Microsoft.ManagedIdentity/userAssignedIdentities/*/assign/action + on the identity's scope to attach it to worker virtual machines at creation time. + properties: + managedIdentity: + description: managedIdentity identifies the user-assigned + managed identity used for ACR image pulls. + properties: + resourceID: + description: |- + resourceID is the ARM resource ID of the user-assigned managed identity + in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}. + The identity must have the AcrPull role on the target Azure Container Registry. + It does not need to be in the same subscription or resource group as the HostedCluster, + but it must be in the same Azure AD tenant. + maxLength: 345 + minLength: 131 + type: string + x-kubernetes-validations: + - message: must be a user-assigned managed identity + ARM resource ID in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName} + rule: self.lowerAscii().matches('^/subscriptions/[^/]+/resourcegroups/[^/]+/providers/microsoft\\.managedidentity/userassignedidentities/[^/]+$') + required: + - resourceID + type: object + type: + description: type specifies the credential type used + for ACR image pulls. + enum: + - ManagedIdentity + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: managedIdentity is required when type is ManagedIdentity, + and forbidden otherwise + rule: 'self.type == ''ManagedIdentity'' ? has(self.managedIdentity) + : !has(self.managedIdentity)' + required: + - credentials + type: object location: description: |- location is the Azure region in where all the cloud infrastructure resources will be created. diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/HyperShiftOnlyDynamicResourceAllocation.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/HyperShiftOnlyDynamicResourceAllocation.yaml index c86527fffe00..c3382fe56b33 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/HyperShiftOnlyDynamicResourceAllocation.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/HyperShiftOnlyDynamicResourceAllocation.yaml @@ -4873,6 +4873,61 @@ spec: - AzureGermanCloud - AzureStackCloud type: string + containerRegistry: + description: |- + containerRegistry configures how worker nodes authenticate to Azure Container Registry (ACR). + When set, the managed identity is attached to worker virtual machines and its resource ID is + written into the worker cloud provider config so kubelet's ACR credential provider can + authenticate without image pull secrets. + Changing this value will trigger a rollout for all existing NodePools in the cluster. + properties: + credentials: + description: |- + credentials configures authentication for worker nodes pulling images from ACR + using a user-assigned managed identity. + The identity does not need to be in the same subscription or resource group as the + HostedCluster, but it must be in the same Azure AD tenant. The management cluster's + CAPZ identity must have Microsoft.ManagedIdentity/userAssignedIdentities/*/assign/action + on the identity's scope to attach it to worker virtual machines at creation time. + properties: + managedIdentity: + description: managedIdentity identifies the user-assigned + managed identity used for ACR image pulls. + properties: + resourceID: + description: |- + resourceID is the ARM resource ID of the user-assigned managed identity + in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}. + The identity must have the AcrPull role on the target Azure Container Registry. + It does not need to be in the same subscription or resource group as the HostedCluster, + but it must be in the same Azure AD tenant. + maxLength: 345 + minLength: 131 + type: string + x-kubernetes-validations: + - message: must be a user-assigned managed identity + ARM resource ID in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName} + rule: self.lowerAscii().matches('^/subscriptions/[^/]+/resourcegroups/[^/]+/providers/microsoft\\.managedidentity/userassignedidentities/[^/]+$') + required: + - resourceID + type: object + type: + description: type specifies the credential type used + for ACR image pulls. + enum: + - ManagedIdentity + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: managedIdentity is required when type is ManagedIdentity, + and forbidden otherwise + rule: 'self.type == ''ManagedIdentity'' ? has(self.managedIdentity) + : !has(self.managedIdentity)' + required: + - credentials + type: object location: description: |- location is the Azure region in where all the cloud infrastructure resources will be created. diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ImageStreamImportMode.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ImageStreamImportMode.yaml index ce0bd35a480b..705b5e49fe8a 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ImageStreamImportMode.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ImageStreamImportMode.yaml @@ -4869,6 +4869,61 @@ spec: - AzureGermanCloud - AzureStackCloud type: string + containerRegistry: + description: |- + containerRegistry configures how worker nodes authenticate to Azure Container Registry (ACR). + When set, the managed identity is attached to worker virtual machines and its resource ID is + written into the worker cloud provider config so kubelet's ACR credential provider can + authenticate without image pull secrets. + Changing this value will trigger a rollout for all existing NodePools in the cluster. + properties: + credentials: + description: |- + credentials configures authentication for worker nodes pulling images from ACR + using a user-assigned managed identity. + The identity does not need to be in the same subscription or resource group as the + HostedCluster, but it must be in the same Azure AD tenant. The management cluster's + CAPZ identity must have Microsoft.ManagedIdentity/userAssignedIdentities/*/assign/action + on the identity's scope to attach it to worker virtual machines at creation time. + properties: + managedIdentity: + description: managedIdentity identifies the user-assigned + managed identity used for ACR image pulls. + properties: + resourceID: + description: |- + resourceID is the ARM resource ID of the user-assigned managed identity + in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}. + The identity must have the AcrPull role on the target Azure Container Registry. + It does not need to be in the same subscription or resource group as the HostedCluster, + but it must be in the same Azure AD tenant. + maxLength: 345 + minLength: 131 + type: string + x-kubernetes-validations: + - message: must be a user-assigned managed identity + ARM resource ID in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName} + rule: self.lowerAscii().matches('^/subscriptions/[^/]+/resourcegroups/[^/]+/providers/microsoft\\.managedidentity/userassignedidentities/[^/]+$') + required: + - resourceID + type: object + type: + description: type specifies the credential type used + for ACR image pulls. + enum: + - ManagedIdentity + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: managedIdentity is required when type is ManagedIdentity, + and forbidden otherwise + rule: 'self.type == ''ManagedIdentity'' ? has(self.managedIdentity) + : !has(self.managedIdentity)' + required: + - credentials + type: object location: description: |- location is the Azure region in where all the cloud infrastructure resources will be created. diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/KMSEncryptionProvider.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/KMSEncryptionProvider.yaml index 5b233d53ff9f..251226f9b48a 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/KMSEncryptionProvider.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/KMSEncryptionProvider.yaml @@ -4927,6 +4927,61 @@ spec: - AzureGermanCloud - AzureStackCloud type: string + containerRegistry: + description: |- + containerRegistry configures how worker nodes authenticate to Azure Container Registry (ACR). + When set, the managed identity is attached to worker virtual machines and its resource ID is + written into the worker cloud provider config so kubelet's ACR credential provider can + authenticate without image pull secrets. + Changing this value will trigger a rollout for all existing NodePools in the cluster. + properties: + credentials: + description: |- + credentials configures authentication for worker nodes pulling images from ACR + using a user-assigned managed identity. + The identity does not need to be in the same subscription or resource group as the + HostedCluster, but it must be in the same Azure AD tenant. The management cluster's + CAPZ identity must have Microsoft.ManagedIdentity/userAssignedIdentities/*/assign/action + on the identity's scope to attach it to worker virtual machines at creation time. + properties: + managedIdentity: + description: managedIdentity identifies the user-assigned + managed identity used for ACR image pulls. + properties: + resourceID: + description: |- + resourceID is the ARM resource ID of the user-assigned managed identity + in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}. + The identity must have the AcrPull role on the target Azure Container Registry. + It does not need to be in the same subscription or resource group as the HostedCluster, + but it must be in the same Azure AD tenant. + maxLength: 345 + minLength: 131 + type: string + x-kubernetes-validations: + - message: must be a user-assigned managed identity + ARM resource ID in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName} + rule: self.lowerAscii().matches('^/subscriptions/[^/]+/resourcegroups/[^/]+/providers/microsoft\\.managedidentity/userassignedidentities/[^/]+$') + required: + - resourceID + type: object + type: + description: type specifies the credential type used + for ACR image pulls. + enum: + - ManagedIdentity + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: managedIdentity is required when type is ManagedIdentity, + and forbidden otherwise + rule: 'self.type == ''ManagedIdentity'' ? has(self.managedIdentity) + : !has(self.managedIdentity)' + required: + - credentials + type: object location: description: |- location is the Azure region in where all the cloud infrastructure resources will be created. diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/OpenStack.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/OpenStack.yaml index da3ccbc41d00..88f1f4a99c38 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/OpenStack.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/OpenStack.yaml @@ -4851,6 +4851,61 @@ spec: - AzureGermanCloud - AzureStackCloud type: string + containerRegistry: + description: |- + containerRegistry configures how worker nodes authenticate to Azure Container Registry (ACR). + When set, the managed identity is attached to worker virtual machines and its resource ID is + written into the worker cloud provider config so kubelet's ACR credential provider can + authenticate without image pull secrets. + Changing this value will trigger a rollout for all existing NodePools in the cluster. + properties: + credentials: + description: |- + credentials configures authentication for worker nodes pulling images from ACR + using a user-assigned managed identity. + The identity does not need to be in the same subscription or resource group as the + HostedCluster, but it must be in the same Azure AD tenant. The management cluster's + CAPZ identity must have Microsoft.ManagedIdentity/userAssignedIdentities/*/assign/action + on the identity's scope to attach it to worker virtual machines at creation time. + properties: + managedIdentity: + description: managedIdentity identifies the user-assigned + managed identity used for ACR image pulls. + properties: + resourceID: + description: |- + resourceID is the ARM resource ID of the user-assigned managed identity + in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}. + The identity must have the AcrPull role on the target Azure Container Registry. + It does not need to be in the same subscription or resource group as the HostedCluster, + but it must be in the same Azure AD tenant. + maxLength: 345 + minLength: 131 + type: string + x-kubernetes-validations: + - message: must be a user-assigned managed identity + ARM resource ID in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName} + rule: self.lowerAscii().matches('^/subscriptions/[^/]+/resourcegroups/[^/]+/providers/microsoft\\.managedidentity/userassignedidentities/[^/]+$') + required: + - resourceID + type: object + type: + description: type specifies the credential type used + for ACR image pulls. + enum: + - ManagedIdentity + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: managedIdentity is required when type is ManagedIdentity, + and forbidden otherwise + rule: 'self.type == ''ManagedIdentity'' ? has(self.managedIdentity) + : !has(self.managedIdentity)' + required: + - credentials + type: object location: description: |- location is the Azure region in where all the cloud infrastructure resources will be created. diff --git a/client/applyconfiguration/hypershift/v1beta1/azurecontainerregistryconfig.go b/client/applyconfiguration/hypershift/v1beta1/azurecontainerregistryconfig.go new file mode 100644 index 000000000000..9cdc555d424d --- /dev/null +++ b/client/applyconfiguration/hypershift/v1beta1/azurecontainerregistryconfig.go @@ -0,0 +1,38 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1beta1 + +// AzureContainerRegistryConfigApplyConfiguration represents a declarative configuration of the AzureContainerRegistryConfig type for use +// with apply. +type AzureContainerRegistryConfigApplyConfiguration struct { + Credentials *AzureContainerRegistryCredentialConfigApplyConfiguration `json:"credentials,omitempty"` +} + +// AzureContainerRegistryConfigApplyConfiguration constructs a declarative configuration of the AzureContainerRegistryConfig type for use with +// apply. +func AzureContainerRegistryConfig() *AzureContainerRegistryConfigApplyConfiguration { + return &AzureContainerRegistryConfigApplyConfiguration{} +} + +// WithCredentials sets the Credentials field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Credentials field is set to the value of the last call. +func (b *AzureContainerRegistryConfigApplyConfiguration) WithCredentials(value *AzureContainerRegistryCredentialConfigApplyConfiguration) *AzureContainerRegistryConfigApplyConfiguration { + b.Credentials = value + return b +} diff --git a/client/applyconfiguration/hypershift/v1beta1/azurecontainerregistrycredentialconfig.go b/client/applyconfiguration/hypershift/v1beta1/azurecontainerregistrycredentialconfig.go new file mode 100644 index 000000000000..4ad06bb3b4c1 --- /dev/null +++ b/client/applyconfiguration/hypershift/v1beta1/azurecontainerregistrycredentialconfig.go @@ -0,0 +1,51 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1beta1 + +import ( + hypershiftv1beta1 "github.com/openshift/hypershift/api/hypershift/v1beta1" +) + +// AzureContainerRegistryCredentialConfigApplyConfiguration represents a declarative configuration of the AzureContainerRegistryCredentialConfig type for use +// with apply. +type AzureContainerRegistryCredentialConfigApplyConfiguration struct { + Type *hypershiftv1beta1.AzureContainerRegistryCredentialType `json:"type,omitempty"` + ManagedIdentity *UserAssignedManagedIdentityApplyConfiguration `json:"managedIdentity,omitempty"` +} + +// AzureContainerRegistryCredentialConfigApplyConfiguration constructs a declarative configuration of the AzureContainerRegistryCredentialConfig type for use with +// apply. +func AzureContainerRegistryCredentialConfig() *AzureContainerRegistryCredentialConfigApplyConfiguration { + return &AzureContainerRegistryCredentialConfigApplyConfiguration{} +} + +// WithType sets the Type field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Type field is set to the value of the last call. +func (b *AzureContainerRegistryCredentialConfigApplyConfiguration) WithType(value hypershiftv1beta1.AzureContainerRegistryCredentialType) *AzureContainerRegistryCredentialConfigApplyConfiguration { + b.Type = &value + return b +} + +// WithManagedIdentity sets the ManagedIdentity field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the ManagedIdentity field is set to the value of the last call. +func (b *AzureContainerRegistryCredentialConfigApplyConfiguration) WithManagedIdentity(value *UserAssignedManagedIdentityApplyConfiguration) *AzureContainerRegistryCredentialConfigApplyConfiguration { + b.ManagedIdentity = value + return b +} diff --git a/client/applyconfiguration/hypershift/v1beta1/azureplatformspec.go b/client/applyconfiguration/hypershift/v1beta1/azureplatformspec.go index 5eb798c50dc8..2fb28573d2c2 100644 --- a/client/applyconfiguration/hypershift/v1beta1/azureplatformspec.go +++ b/client/applyconfiguration/hypershift/v1beta1/azureplatformspec.go @@ -33,6 +33,7 @@ type AzurePlatformSpecApplyConfiguration struct { SecurityGroupID *string `json:"securityGroupID,omitempty"` AzureAuthenticationConfig *AzureAuthenticationConfigurationApplyConfiguration `json:"azureAuthenticationConfig,omitempty"` TenantID *string `json:"tenantID,omitempty"` + ContainerRegistry *AzureContainerRegistryConfigApplyConfiguration `json:"containerRegistry,omitempty"` Topology *hypershiftv1beta1.AzureTopologyType `json:"topology,omitempty"` Private *AzurePrivateSpecApplyConfiguration `json:"private,omitempty"` } @@ -115,6 +116,14 @@ func (b *AzurePlatformSpecApplyConfiguration) WithTenantID(value string) *AzureP return b } +// WithContainerRegistry sets the ContainerRegistry field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the ContainerRegistry field is set to the value of the last call. +func (b *AzurePlatformSpecApplyConfiguration) WithContainerRegistry(value *AzureContainerRegistryConfigApplyConfiguration) *AzurePlatformSpecApplyConfiguration { + b.ContainerRegistry = value + return b +} + // WithTopology sets the Topology field in the declarative configuration to the given value // and returns the receiver, so that objects can be built by chaining "With" function invocations. // If called multiple times, the Topology field is set to the value of the last call. diff --git a/client/applyconfiguration/hypershift/v1beta1/userassignedmanagedidentity.go b/client/applyconfiguration/hypershift/v1beta1/userassignedmanagedidentity.go new file mode 100644 index 000000000000..cc7cd0c7438c --- /dev/null +++ b/client/applyconfiguration/hypershift/v1beta1/userassignedmanagedidentity.go @@ -0,0 +1,42 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1beta1 + +import ( + hypershiftv1beta1 "github.com/openshift/hypershift/api/hypershift/v1beta1" +) + +// UserAssignedManagedIdentityApplyConfiguration represents a declarative configuration of the UserAssignedManagedIdentity type for use +// with apply. +type UserAssignedManagedIdentityApplyConfiguration struct { + ResourceID *hypershiftv1beta1.AzureManagedIdentityResourceID `json:"resourceID,omitempty"` +} + +// UserAssignedManagedIdentityApplyConfiguration constructs a declarative configuration of the UserAssignedManagedIdentity type for use with +// apply. +func UserAssignedManagedIdentity() *UserAssignedManagedIdentityApplyConfiguration { + return &UserAssignedManagedIdentityApplyConfiguration{} +} + +// WithResourceID sets the ResourceID field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the ResourceID field is set to the value of the last call. +func (b *UserAssignedManagedIdentityApplyConfiguration) WithResourceID(value hypershiftv1beta1.AzureManagedIdentityResourceID) *UserAssignedManagedIdentityApplyConfiguration { + b.ResourceID = &value + return b +} diff --git a/client/applyconfiguration/utils.go b/client/applyconfiguration/utils.go index 169082f81c34..350a2d3e57bf 100644 --- a/client/applyconfiguration/utils.go +++ b/client/applyconfiguration/utils.go @@ -111,6 +111,10 @@ func ForKind(kind schema.GroupVersionKind) interface{} { return &hypershiftv1beta1.AWSSharedVPCRolesRefApplyConfiguration{} case v1beta1.SchemeGroupVersion.WithKind("AzureAuthenticationConfiguration"): return &hypershiftv1beta1.AzureAuthenticationConfigurationApplyConfiguration{} + case v1beta1.SchemeGroupVersion.WithKind("AzureContainerRegistryConfig"): + return &hypershiftv1beta1.AzureContainerRegistryConfigApplyConfiguration{} + case v1beta1.SchemeGroupVersion.WithKind("AzureContainerRegistryCredentialConfig"): + return &hypershiftv1beta1.AzureContainerRegistryCredentialConfigApplyConfiguration{} case v1beta1.SchemeGroupVersion.WithKind("AzureKMSKey"): return &hypershiftv1beta1.AzureKMSKeyApplyConfiguration{} case v1beta1.SchemeGroupVersion.WithKind("AzureKMSSpec"): @@ -417,6 +421,8 @@ func ForKind(kind schema.GroupVersionKind) interface{} { return &hypershiftv1beta1.TaintApplyConfiguration{} case v1beta1.SchemeGroupVersion.WithKind("UnmanagedEtcdSpec"): return &hypershiftv1beta1.UnmanagedEtcdSpecApplyConfiguration{} + case v1beta1.SchemeGroupVersion.WithKind("UserAssignedManagedIdentity"): + return &hypershiftv1beta1.UserAssignedManagedIdentityApplyConfiguration{} case v1beta1.SchemeGroupVersion.WithKind("UserManagedDiagnostics"): return &hypershiftv1beta1.UserManagedDiagnosticsApplyConfiguration{} case v1beta1.SchemeGroupVersion.WithKind("Volume"): diff --git a/cmd/install/assets/crds/hypershift-operator/tests/hostedclusters.hypershift.openshift.io/stable.hostedclusters.azure.testsuite.yaml b/cmd/install/assets/crds/hypershift-operator/tests/hostedclusters.hypershift.openshift.io/stable.hostedclusters.azure.testsuite.yaml index b47bb2199e4f..342d169f1d4c 100644 --- a/cmd/install/assets/crds/hypershift-operator/tests/hostedclusters.hypershift.openshift.io/stable.hostedclusters.azure.testsuite.yaml +++ b/cmd/install/assets/crds/hypershift-operator/tests/hostedclusters.hypershift.openshift.io/stable.hostedclusters.azure.testsuite.yaml @@ -1164,6 +1164,290 @@ tests: type: Route route: {} + + - name: When containerRegistry credentials has invalid resource ID format it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: Azure + azure: + location: eastus + resourceGroupName: test-rg + vnetID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/virtualNetworks/test-vnet" + subnetID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/virtualNetworks/test-vnet/subnets/test-subnet" + subscriptionID: "12345678-1234-5678-9012-123456789012" + securityGroupID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/networkSecurityGroups/test-nsg" + tenantID: "87654321-4321-8765-2109-876543210987" + containerRegistry: + credentials: + type: ManagedIdentity + managedIdentity: + resourceID: "not-a-valid-resource-id" + + azureAuthenticationConfig: + azureAuthenticationConfigType: ManagedIdentities + managedIdentities: + controlPlane: + managedIdentitiesKeyVault: + name: test-kv + tenantID: "87654321-4321-8765-2109-876543210987" + cloudProvider: + clientID: "12345678-1234-5678-9012-123456789012" + credentialsSecretName: cloud-provider-secret + objectEncoding: utf-8 + nodePoolManagement: + clientID: "12345678-1234-5678-9012-123456789012" + credentialsSecretName: node-pool-secret + objectEncoding: utf-8 + controlPlaneOperator: + clientID: "12345678-1234-5678-9012-123456789012" + credentialsSecretName: cpo-secret + objectEncoding: utf-8 + imageRegistry: + clientID: "12345678-1234-5678-9012-123456789012" + credentialsSecretName: image-registry-secret + objectEncoding: utf-8 + ingress: + clientID: "12345678-1234-5678-9012-123456789012" + credentialsSecretName: ingress-secret + objectEncoding: utf-8 + network: + clientID: "12345678-1234-5678-9012-123456789012" + credentialsSecretName: network-secret + objectEncoding: utf-8 + disk: + clientID: "12345678-1234-5678-9012-123456789012" + credentialsSecretName: disk-secret + objectEncoding: utf-8 + file: + clientID: "12345678-1234-5678-9012-123456789012" + credentialsSecretName: file-secret + objectEncoding: utf-8 + dataPlane: + imageRegistryMSIClientID: "12345678-1234-5678-9012-123456789012" + diskMSIClientID: "12345678-1234-5678-9012-123456789012" + fileMSIClientID: "12345678-1234-5678-9012-123456789012" + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "must be a user-assigned managed identity ARM resource ID in the format" + + - name: When containerRegistry credentials has wrong provider namespace it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: Azure + azure: + location: eastus + resourceGroupName: test-rg + vnetID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/virtualNetworks/test-vnet" + subnetID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/virtualNetworks/test-vnet/subnets/test-subnet" + subscriptionID: "12345678-1234-5678-9012-123456789012" + securityGroupID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/networkSecurityGroups/test-nsg" + tenantID: "87654321-4321-8765-2109-876543210987" + containerRegistry: + credentials: + type: ManagedIdentity + managedIdentity: + resourceID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Compute/virtualMachines/test-vm" + + azureAuthenticationConfig: + azureAuthenticationConfigType: ManagedIdentities + managedIdentities: + controlPlane: + managedIdentitiesKeyVault: + name: test-kv + tenantID: "87654321-4321-8765-2109-876543210987" + cloudProvider: + clientID: "12345678-1234-5678-9012-123456789012" + credentialsSecretName: cloud-provider-secret + objectEncoding: utf-8 + nodePoolManagement: + clientID: "12345678-1234-5678-9012-123456789012" + credentialsSecretName: node-pool-secret + objectEncoding: utf-8 + controlPlaneOperator: + clientID: "12345678-1234-5678-9012-123456789012" + credentialsSecretName: cpo-secret + objectEncoding: utf-8 + imageRegistry: + clientID: "12345678-1234-5678-9012-123456789012" + credentialsSecretName: image-registry-secret + objectEncoding: utf-8 + ingress: + clientID: "12345678-1234-5678-9012-123456789012" + credentialsSecretName: ingress-secret + objectEncoding: utf-8 + network: + clientID: "12345678-1234-5678-9012-123456789012" + credentialsSecretName: network-secret + objectEncoding: utf-8 + disk: + clientID: "12345678-1234-5678-9012-123456789012" + credentialsSecretName: disk-secret + objectEncoding: utf-8 + file: + clientID: "12345678-1234-5678-9012-123456789012" + credentialsSecretName: file-secret + objectEncoding: utf-8 + dataPlane: + imageRegistryMSIClientID: "12345678-1234-5678-9012-123456789012" + diskMSIClientID: "12345678-1234-5678-9012-123456789012" + fileMSIClientID: "12345678-1234-5678-9012-123456789012" + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "must be a user-assigned managed identity ARM resource ID in the format" + + - name: When containerRegistry credentials has trailing slash it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: Azure + azure: + location: eastus + resourceGroupName: test-rg + vnetID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/virtualNetworks/test-vnet" + subnetID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/virtualNetworks/test-vnet/subnets/test-subnet" + subscriptionID: "12345678-1234-5678-9012-123456789012" + securityGroupID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/networkSecurityGroups/test-nsg" + tenantID: "87654321-4321-8765-2109-876543210987" + containerRegistry: + credentials: + type: ManagedIdentity + managedIdentity: + resourceID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-identity/" + azureAuthenticationConfig: + azureAuthenticationConfigType: ManagedIdentities + managedIdentities: + controlPlane: + managedIdentitiesKeyVault: + name: test-kv + tenantID: "87654321-4321-8765-2109-876543210987" + cloudProvider: + clientID: "12345678-1234-5678-9012-123456789012" + credentialsSecretName: cloud-provider-secret + objectEncoding: utf-8 + nodePoolManagement: + clientID: "12345678-1234-5678-9012-123456789012" + credentialsSecretName: node-pool-secret + objectEncoding: utf-8 + controlPlaneOperator: + clientID: "12345678-1234-5678-9012-123456789012" + credentialsSecretName: cpo-secret + objectEncoding: utf-8 + imageRegistry: + clientID: "12345678-1234-5678-9012-123456789012" + credentialsSecretName: image-registry-secret + objectEncoding: utf-8 + ingress: + clientID: "12345678-1234-5678-9012-123456789012" + credentialsSecretName: ingress-secret + objectEncoding: utf-8 + network: + clientID: "12345678-1234-5678-9012-123456789012" + credentialsSecretName: network-secret + objectEncoding: utf-8 + disk: + clientID: "12345678-1234-5678-9012-123456789012" + credentialsSecretName: disk-secret + objectEncoding: utf-8 + file: + clientID: "12345678-1234-5678-9012-123456789012" + credentialsSecretName: file-secret + objectEncoding: utf-8 + dataPlane: + imageRegistryMSIClientID: "12345678-1234-5678-9012-123456789012" + diskMSIClientID: "12345678-1234-5678-9012-123456789012" + fileMSIClientID: "12345678-1234-5678-9012-123456789012" + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "must be a user-assigned managed identity ARM resource ID in the format" + + + onUpdate: # --- Azure Swift private networking immutability --- - name: When Private.Type changes from PrivateLink to Swift it should fail @@ -1834,3 +2118,181 @@ tests: type: Route route: {} expectedError: "transitions between Public and non-Public topology are not supported" + + - name: When containerRegistry credentials is removed after creation it should pass + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: Azure + azure: + location: eastus + resourceGroupName: test-rg + vnetID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/virtualNetworks/test-vnet" + subnetID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/virtualNetworks/test-vnet/subnets/test-subnet" + subscriptionID: "12345678-1234-5678-9012-123456789012" + securityGroupID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/networkSecurityGroups/test-nsg" + tenantID: "87654321-4321-8765-2109-876543210987" + containerRegistry: + credentials: + type: ManagedIdentity + managedIdentity: + resourceID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-identity" + azureAuthenticationConfig: + azureAuthenticationConfigType: ManagedIdentities + managedIdentities: + controlPlane: + managedIdentitiesKeyVault: + name: test-kv + tenantID: "87654321-4321-8765-2109-876543210987" + cloudProvider: + clientID: "12345678-1234-5678-9012-123456789012" + credentialsSecretName: cloud-provider-secret + objectEncoding: utf-8 + nodePoolManagement: + clientID: "12345678-1234-5678-9012-123456789012" + credentialsSecretName: node-pool-secret + objectEncoding: utf-8 + controlPlaneOperator: + clientID: "12345678-1234-5678-9012-123456789012" + credentialsSecretName: cpo-secret + objectEncoding: utf-8 + imageRegistry: + clientID: "12345678-1234-5678-9012-123456789012" + credentialsSecretName: image-registry-secret + objectEncoding: utf-8 + ingress: + clientID: "12345678-1234-5678-9012-123456789012" + credentialsSecretName: ingress-secret + objectEncoding: utf-8 + network: + clientID: "12345678-1234-5678-9012-123456789012" + credentialsSecretName: network-secret + objectEncoding: utf-8 + disk: + clientID: "12345678-1234-5678-9012-123456789012" + credentialsSecretName: disk-secret + objectEncoding: utf-8 + file: + clientID: "12345678-1234-5678-9012-123456789012" + credentialsSecretName: file-secret + objectEncoding: utf-8 + dataPlane: + imageRegistryMSIClientID: "12345678-1234-5678-9012-123456789012" + diskMSIClientID: "12345678-1234-5678-9012-123456789012" + fileMSIClientID: "12345678-1234-5678-9012-123456789012" + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + updated: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: Azure + azure: + location: eastus + resourceGroupName: test-rg + vnetID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/virtualNetworks/test-vnet" + subnetID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/virtualNetworks/test-vnet/subnets/test-subnet" + subscriptionID: "12345678-1234-5678-9012-123456789012" + securityGroupID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/networkSecurityGroups/test-nsg" + tenantID: "87654321-4321-8765-2109-876543210987" + azureAuthenticationConfig: + azureAuthenticationConfigType: ManagedIdentities + managedIdentities: + controlPlane: + managedIdentitiesKeyVault: + name: test-kv + tenantID: "87654321-4321-8765-2109-876543210987" + cloudProvider: + clientID: "12345678-1234-5678-9012-123456789012" + credentialsSecretName: cloud-provider-secret + objectEncoding: utf-8 + nodePoolManagement: + clientID: "12345678-1234-5678-9012-123456789012" + credentialsSecretName: node-pool-secret + objectEncoding: utf-8 + controlPlaneOperator: + clientID: "12345678-1234-5678-9012-123456789012" + credentialsSecretName: cpo-secret + objectEncoding: utf-8 + imageRegistry: + clientID: "12345678-1234-5678-9012-123456789012" + credentialsSecretName: image-registry-secret + objectEncoding: utf-8 + ingress: + clientID: "12345678-1234-5678-9012-123456789012" + credentialsSecretName: ingress-secret + objectEncoding: utf-8 + network: + clientID: "12345678-1234-5678-9012-123456789012" + credentialsSecretName: network-secret + objectEncoding: utf-8 + disk: + clientID: "12345678-1234-5678-9012-123456789012" + credentialsSecretName: disk-secret + objectEncoding: utf-8 + file: + clientID: "12345678-1234-5678-9012-123456789012" + credentialsSecretName: file-secret + objectEncoding: utf-8 + dataPlane: + imageRegistryMSIClientID: "12345678-1234-5678-9012-123456789012" + diskMSIClientID: "12345678-1234-5678-9012-123456789012" + fileMSIClientID: "12345678-1234-5678-9012-123456789012" + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "" diff --git a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedclusters-Hypershift-CustomNoUpgrade.crd.yaml b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedclusters-Hypershift-CustomNoUpgrade.crd.yaml index 8f4696f02dfa..1e298f49e127 100644 --- a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedclusters-Hypershift-CustomNoUpgrade.crd.yaml +++ b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedclusters-Hypershift-CustomNoUpgrade.crd.yaml @@ -5911,6 +5911,61 @@ spec: - AzureGermanCloud - AzureStackCloud type: string + containerRegistry: + description: |- + containerRegistry configures how worker nodes authenticate to Azure Container Registry (ACR). + When set, the managed identity is attached to worker virtual machines and its resource ID is + written into the worker cloud provider config so kubelet's ACR credential provider can + authenticate without image pull secrets. + Changing this value will trigger a rollout for all existing NodePools in the cluster. + properties: + credentials: + description: |- + credentials configures authentication for worker nodes pulling images from ACR + using a user-assigned managed identity. + The identity does not need to be in the same subscription or resource group as the + HostedCluster, but it must be in the same Azure AD tenant. The management cluster's + CAPZ identity must have Microsoft.ManagedIdentity/userAssignedIdentities/*/assign/action + on the identity's scope to attach it to worker virtual machines at creation time. + properties: + managedIdentity: + description: managedIdentity identifies the user-assigned + managed identity used for ACR image pulls. + properties: + resourceID: + description: |- + resourceID is the ARM resource ID of the user-assigned managed identity + in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}. + The identity must have the AcrPull role on the target Azure Container Registry. + It does not need to be in the same subscription or resource group as the HostedCluster, + but it must be in the same Azure AD tenant. + maxLength: 345 + minLength: 131 + type: string + x-kubernetes-validations: + - message: must be a user-assigned managed identity + ARM resource ID in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName} + rule: self.lowerAscii().matches('^/subscriptions/[^/]+/resourcegroups/[^/]+/providers/microsoft\\.managedidentity/userassignedidentities/[^/]+$') + required: + - resourceID + type: object + type: + description: type specifies the credential type used + for ACR image pulls. + enum: + - ManagedIdentity + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: managedIdentity is required when type is ManagedIdentity, + and forbidden otherwise + rule: 'self.type == ''ManagedIdentity'' ? has(self.managedIdentity) + : !has(self.managedIdentity)' + required: + - credentials + type: object location: description: |- location is the Azure region in where all the cloud infrastructure resources will be created. diff --git a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedclusters-Hypershift-Default.crd.yaml b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedclusters-Hypershift-Default.crd.yaml index 1e7000c992ea..119f40df6d9c 100644 --- a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedclusters-Hypershift-Default.crd.yaml +++ b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedclusters-Hypershift-Default.crd.yaml @@ -5475,6 +5475,61 @@ spec: - AzureGermanCloud - AzureStackCloud type: string + containerRegistry: + description: |- + containerRegistry configures how worker nodes authenticate to Azure Container Registry (ACR). + When set, the managed identity is attached to worker virtual machines and its resource ID is + written into the worker cloud provider config so kubelet's ACR credential provider can + authenticate without image pull secrets. + Changing this value will trigger a rollout for all existing NodePools in the cluster. + properties: + credentials: + description: |- + credentials configures authentication for worker nodes pulling images from ACR + using a user-assigned managed identity. + The identity does not need to be in the same subscription or resource group as the + HostedCluster, but it must be in the same Azure AD tenant. The management cluster's + CAPZ identity must have Microsoft.ManagedIdentity/userAssignedIdentities/*/assign/action + on the identity's scope to attach it to worker virtual machines at creation time. + properties: + managedIdentity: + description: managedIdentity identifies the user-assigned + managed identity used for ACR image pulls. + properties: + resourceID: + description: |- + resourceID is the ARM resource ID of the user-assigned managed identity + in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}. + The identity must have the AcrPull role on the target Azure Container Registry. + It does not need to be in the same subscription or resource group as the HostedCluster, + but it must be in the same Azure AD tenant. + maxLength: 345 + minLength: 131 + type: string + x-kubernetes-validations: + - message: must be a user-assigned managed identity + ARM resource ID in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName} + rule: self.lowerAscii().matches('^/subscriptions/[^/]+/resourcegroups/[^/]+/providers/microsoft\\.managedidentity/userassignedidentities/[^/]+$') + required: + - resourceID + type: object + type: + description: type specifies the credential type used + for ACR image pulls. + enum: + - ManagedIdentity + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: managedIdentity is required when type is ManagedIdentity, + and forbidden otherwise + rule: 'self.type == ''ManagedIdentity'' ? has(self.managedIdentity) + : !has(self.managedIdentity)' + required: + - credentials + type: object location: description: |- location is the Azure region in where all the cloud infrastructure resources will be created. diff --git a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedclusters-Hypershift-TechPreviewNoUpgrade.crd.yaml b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedclusters-Hypershift-TechPreviewNoUpgrade.crd.yaml index 449e067ff34f..2c7680dd9550 100644 --- a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedclusters-Hypershift-TechPreviewNoUpgrade.crd.yaml +++ b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedclusters-Hypershift-TechPreviewNoUpgrade.crd.yaml @@ -5822,6 +5822,61 @@ spec: - AzureGermanCloud - AzureStackCloud type: string + containerRegistry: + description: |- + containerRegistry configures how worker nodes authenticate to Azure Container Registry (ACR). + When set, the managed identity is attached to worker virtual machines and its resource ID is + written into the worker cloud provider config so kubelet's ACR credential provider can + authenticate without image pull secrets. + Changing this value will trigger a rollout for all existing NodePools in the cluster. + properties: + credentials: + description: |- + credentials configures authentication for worker nodes pulling images from ACR + using a user-assigned managed identity. + The identity does not need to be in the same subscription or resource group as the + HostedCluster, but it must be in the same Azure AD tenant. The management cluster's + CAPZ identity must have Microsoft.ManagedIdentity/userAssignedIdentities/*/assign/action + on the identity's scope to attach it to worker virtual machines at creation time. + properties: + managedIdentity: + description: managedIdentity identifies the user-assigned + managed identity used for ACR image pulls. + properties: + resourceID: + description: |- + resourceID is the ARM resource ID of the user-assigned managed identity + in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}. + The identity must have the AcrPull role on the target Azure Container Registry. + It does not need to be in the same subscription or resource group as the HostedCluster, + but it must be in the same Azure AD tenant. + maxLength: 345 + minLength: 131 + type: string + x-kubernetes-validations: + - message: must be a user-assigned managed identity + ARM resource ID in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName} + rule: self.lowerAscii().matches('^/subscriptions/[^/]+/resourcegroups/[^/]+/providers/microsoft\\.managedidentity/userassignedidentities/[^/]+$') + required: + - resourceID + type: object + type: + description: type specifies the credential type used + for ACR image pulls. + enum: + - ManagedIdentity + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: managedIdentity is required when type is ManagedIdentity, + and forbidden otherwise + rule: 'self.type == ''ManagedIdentity'' ? has(self.managedIdentity) + : !has(self.managedIdentity)' + required: + - credentials + type: object location: description: |- location is the Azure region in where all the cloud infrastructure resources will be created. diff --git a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-Hypershift-CustomNoUpgrade.crd.yaml b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-Hypershift-CustomNoUpgrade.crd.yaml index 1d428efff530..6f44a3d52388 100644 --- a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-Hypershift-CustomNoUpgrade.crd.yaml +++ b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-Hypershift-CustomNoUpgrade.crd.yaml @@ -5791,6 +5791,61 @@ spec: - AzureGermanCloud - AzureStackCloud type: string + containerRegistry: + description: |- + containerRegistry configures how worker nodes authenticate to Azure Container Registry (ACR). + When set, the managed identity is attached to worker virtual machines and its resource ID is + written into the worker cloud provider config so kubelet's ACR credential provider can + authenticate without image pull secrets. + Changing this value will trigger a rollout for all existing NodePools in the cluster. + properties: + credentials: + description: |- + credentials configures authentication for worker nodes pulling images from ACR + using a user-assigned managed identity. + The identity does not need to be in the same subscription or resource group as the + HostedCluster, but it must be in the same Azure AD tenant. The management cluster's + CAPZ identity must have Microsoft.ManagedIdentity/userAssignedIdentities/*/assign/action + on the identity's scope to attach it to worker virtual machines at creation time. + properties: + managedIdentity: + description: managedIdentity identifies the user-assigned + managed identity used for ACR image pulls. + properties: + resourceID: + description: |- + resourceID is the ARM resource ID of the user-assigned managed identity + in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}. + The identity must have the AcrPull role on the target Azure Container Registry. + It does not need to be in the same subscription or resource group as the HostedCluster, + but it must be in the same Azure AD tenant. + maxLength: 345 + minLength: 131 + type: string + x-kubernetes-validations: + - message: must be a user-assigned managed identity + ARM resource ID in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName} + rule: self.lowerAscii().matches('^/subscriptions/[^/]+/resourcegroups/[^/]+/providers/microsoft\\.managedidentity/userassignedidentities/[^/]+$') + required: + - resourceID + type: object + type: + description: type specifies the credential type used + for ACR image pulls. + enum: + - ManagedIdentity + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: managedIdentity is required when type is ManagedIdentity, + and forbidden otherwise + rule: 'self.type == ''ManagedIdentity'' ? has(self.managedIdentity) + : !has(self.managedIdentity)' + required: + - credentials + type: object location: description: |- location is the Azure region in where all the cloud infrastructure resources will be created. diff --git a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-Hypershift-Default.crd.yaml b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-Hypershift-Default.crd.yaml index 27ccd2cd2da2..68fc6a983002 100644 --- a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-Hypershift-Default.crd.yaml +++ b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-Hypershift-Default.crd.yaml @@ -5353,6 +5353,61 @@ spec: - AzureGermanCloud - AzureStackCloud type: string + containerRegistry: + description: |- + containerRegistry configures how worker nodes authenticate to Azure Container Registry (ACR). + When set, the managed identity is attached to worker virtual machines and its resource ID is + written into the worker cloud provider config so kubelet's ACR credential provider can + authenticate without image pull secrets. + Changing this value will trigger a rollout for all existing NodePools in the cluster. + properties: + credentials: + description: |- + credentials configures authentication for worker nodes pulling images from ACR + using a user-assigned managed identity. + The identity does not need to be in the same subscription or resource group as the + HostedCluster, but it must be in the same Azure AD tenant. The management cluster's + CAPZ identity must have Microsoft.ManagedIdentity/userAssignedIdentities/*/assign/action + on the identity's scope to attach it to worker virtual machines at creation time. + properties: + managedIdentity: + description: managedIdentity identifies the user-assigned + managed identity used for ACR image pulls. + properties: + resourceID: + description: |- + resourceID is the ARM resource ID of the user-assigned managed identity + in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}. + The identity must have the AcrPull role on the target Azure Container Registry. + It does not need to be in the same subscription or resource group as the HostedCluster, + but it must be in the same Azure AD tenant. + maxLength: 345 + minLength: 131 + type: string + x-kubernetes-validations: + - message: must be a user-assigned managed identity + ARM resource ID in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName} + rule: self.lowerAscii().matches('^/subscriptions/[^/]+/resourcegroups/[^/]+/providers/microsoft\\.managedidentity/userassignedidentities/[^/]+$') + required: + - resourceID + type: object + type: + description: type specifies the credential type used + for ACR image pulls. + enum: + - ManagedIdentity + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: managedIdentity is required when type is ManagedIdentity, + and forbidden otherwise + rule: 'self.type == ''ManagedIdentity'' ? has(self.managedIdentity) + : !has(self.managedIdentity)' + required: + - credentials + type: object location: description: |- location is the Azure region in where all the cloud infrastructure resources will be created. diff --git a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-Hypershift-TechPreviewNoUpgrade.crd.yaml b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-Hypershift-TechPreviewNoUpgrade.crd.yaml index 9b19e7197c89..9c4df30c5255 100644 --- a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-Hypershift-TechPreviewNoUpgrade.crd.yaml +++ b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-Hypershift-TechPreviewNoUpgrade.crd.yaml @@ -5702,6 +5702,61 @@ spec: - AzureGermanCloud - AzureStackCloud type: string + containerRegistry: + description: |- + containerRegistry configures how worker nodes authenticate to Azure Container Registry (ACR). + When set, the managed identity is attached to worker virtual machines and its resource ID is + written into the worker cloud provider config so kubelet's ACR credential provider can + authenticate without image pull secrets. + Changing this value will trigger a rollout for all existing NodePools in the cluster. + properties: + credentials: + description: |- + credentials configures authentication for worker nodes pulling images from ACR + using a user-assigned managed identity. + The identity does not need to be in the same subscription or resource group as the + HostedCluster, but it must be in the same Azure AD tenant. The management cluster's + CAPZ identity must have Microsoft.ManagedIdentity/userAssignedIdentities/*/assign/action + on the identity's scope to attach it to worker virtual machines at creation time. + properties: + managedIdentity: + description: managedIdentity identifies the user-assigned + managed identity used for ACR image pulls. + properties: + resourceID: + description: |- + resourceID is the ARM resource ID of the user-assigned managed identity + in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}. + The identity must have the AcrPull role on the target Azure Container Registry. + It does not need to be in the same subscription or resource group as the HostedCluster, + but it must be in the same Azure AD tenant. + maxLength: 345 + minLength: 131 + type: string + x-kubernetes-validations: + - message: must be a user-assigned managed identity + ARM resource ID in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName} + rule: self.lowerAscii().matches('^/subscriptions/[^/]+/resourcegroups/[^/]+/providers/microsoft\\.managedidentity/userassignedidentities/[^/]+$') + required: + - resourceID + type: object + type: + description: type specifies the credential type used + for ACR image pulls. + enum: + - ManagedIdentity + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: managedIdentity is required when type is ManagedIdentity, + and forbidden otherwise + rule: 'self.type == ''ManagedIdentity'' ? has(self.managedIdentity) + : !has(self.managedIdentity)' + required: + - credentials + type: object location: description: |- location is the Azure region in where all the cloud infrastructure resources will be created. diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/azure/config.go b/control-plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/azure/config.go index 623c0adac07e..9a7b8e103e6b 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/azure/config.go +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/azure/config.go @@ -28,6 +28,10 @@ func adaptConfig(cpContext component.WorkloadContext, cm *corev1.ConfigMap) erro return err } + if cpContext.HCP.Spec.Platform.Azure != nil && cpContext.HCP.Spec.Platform.Azure.ContainerRegistry.Credentials.ManagedIdentity.ResourceID != "" { + cfg.UserAssignedIdentityID = string(cpContext.HCP.Spec.Platform.Azure.ContainerRegistry.Credentials.ManagedIdentity.ResourceID) + } + serializedConfig, err := json.MarshalIndent(cfg, "", " ") if err != nil { return fmt.Errorf("failed to serialize cloudconfig: %w", err) @@ -199,4 +203,5 @@ type AzureConfig struct { ClusterServiceLoadBalancerHealthProbeMode string `json:"clusterServiceLoadBalancerHealthProbeMode"` ClusterServiceSharedLoadBalancerHealthProbePath string `json:"clusterServiceSharedLoadBalancerHealthProbePath,omitempty"` ClusterServiceSharedLoadBalancerHealthProbePort int32 `json:"clusterServiceSharedLoadBalancerHealthProbePort,omitempty"` + UserAssignedIdentityID string `json:"userAssignedIdentityID,omitempty"` } diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/azure/config_test.go b/control-plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/azure/config_test.go index 41632fc5d5bc..aa3488887e70 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/azure/config_test.go +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/azure/config_test.go @@ -4,6 +4,8 @@ import ( "strings" "testing" + . "github.com/onsi/gomega" + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" assets "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/v2/assets" "github.com/openshift/hypershift/control-plane-operator/hostedclusterconfigoperator/api" @@ -16,30 +18,60 @@ import ( ) func TestConfig(t *testing.T) { - hcp := newTestHCP(map[string]string{ - hyperv1.SharedLoadBalancerHealthProbePathAnnotation: "/healthz", - hyperv1.SharedLoadBalancerHealthProbePortAnnotation: "10256", + t.Run("When health probe annotations are set it should generate valid config", func(t *testing.T) { + g := NewGomegaWithT(t) + + hcp := newTestHCP(map[string]string{ + hyperv1.SharedLoadBalancerHealthProbePathAnnotation: "/healthz", + hyperv1.SharedLoadBalancerHealthProbePortAnnotation: "10256", + }) + hcp.Namespace = "HCP_NAMESPACE" + + cm := &corev1.ConfigMap{} + _, _, err := assets.LoadManifestInto(ComponentName, "config.yaml", cm) + g.Expect(err).ToNot(HaveOccurred()) + + cpContext := component.WorkloadContext{ + HCP: hcp, + } + err = adaptConfig(cpContext, cm) + g.Expect(err).ToNot(HaveOccurred()) + + yaml, err := util.SerializeResource(cm, api.Scheme) + g.Expect(err).ToNot(HaveOccurred()) + + testutil.CompareWithFixture(t, yaml) }) - hcp.Namespace = "HCP_NAMESPACE" - cm := &corev1.ConfigMap{} - _, _, err := assets.LoadManifestInto(ComponentName, "config.yaml", cm) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - cpContext := component.WorkloadContext{ - HCP: hcp, - } - err = adaptConfig(cpContext, cm) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } + t.Run("When containerRegistry is set it should include userAssignedIdentityID", func(t *testing.T) { + g := NewGomegaWithT(t) - yaml, err := util.SerializeResource(cm, api.Scheme) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - testutil.CompareWithFixture(t, yaml) + hcp := newTestHCP(nil) + hcp.Namespace = "HCP_NAMESPACE" + hcp.Spec.Platform.Azure.ContainerRegistry = hyperv1.AzureContainerRegistryConfig{ + Credentials: hyperv1.AzureContainerRegistryCredentialConfig{ + Type: hyperv1.AzureContainerRegistryCredentialManagedIdentity, + ManagedIdentity: hyperv1.UserAssignedManagedIdentity{ + ResourceID: "/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-mi", + }, + }, + } + + cm := &corev1.ConfigMap{} + _, _, err := assets.LoadManifestInto(ComponentName, "config.yaml", cm) + g.Expect(err).ToNot(HaveOccurred()) + + cpContext := component.WorkloadContext{ + HCP: hcp, + } + err = adaptConfig(cpContext, cm) + g.Expect(err).ToNot(HaveOccurred()) + + yaml, err := util.SerializeResource(cm, api.Scheme) + g.Expect(err).ToNot(HaveOccurred()) + + testutil.CompareWithFixture(t, yaml) + }) } // newTestHCP creates a HostedControlPlane with default Azure configuration for testing. @@ -77,6 +109,34 @@ func newTestHCP(annotations map[string]string) *hyperv1.HostedControlPlane { } } +func TestConfigSecretDoesNotContainAcrMI(t *testing.T) { + g := NewGomegaWithT(t) + + hcp := newTestHCP(nil) + hcp.Namespace = "HCP_NAMESPACE" + hcp.Spec.Platform.Azure.ContainerRegistry = hyperv1.AzureContainerRegistryConfig{ + Credentials: hyperv1.AzureContainerRegistryCredentialConfig{ + Type: hyperv1.AzureContainerRegistryCredentialManagedIdentity, + ManagedIdentity: hyperv1.UserAssignedManagedIdentity{ + ResourceID: "/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-mi", + }, + }, + } + + secret := &corev1.Secret{ + Data: map[string][]byte{}, + } + cpContext := component.WorkloadContext{ + HCP: hcp, + } + err := adaptConfigSecret(cpContext, secret) + g.Expect(err).ToNot(HaveOccurred()) + + cloudConf := string(secret.Data[ConfigKey]) + g.Expect(cloudConf).ToNot(ContainSubstring("userAssignedIdentityID"), + "CP secret should NOT contain userAssignedIdentityID") +} + func TestConfigErrorStates(t *testing.T) { tests := []struct { name string diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/azure/testdata/zz_fixture_TestConfig_When_containerRegistry_is_set_it_should_include_userAssignedIdentityID.yaml b/control-plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/azure/testdata/zz_fixture_TestConfig_When_containerRegistry_is_set_it_should_include_userAssignedIdentityID.yaml new file mode 100644 index 000000000000..2427db2e7f72 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/azure/testdata/zz_fixture_TestConfig_When_containerRegistry_is_set_it_should_include_userAssignedIdentityID.yaml @@ -0,0 +1,34 @@ +apiVersion: v1 +data: + cloud.conf: |- + { + "cloud": "AzurePublicCloud", + "tenantId": "my-tenant-id", + "useManagedIdentityExtension": false, + "useFederatedWorkloadIdentityExtension": true, + "subscriptionId": "my-subscription-id", + "aadClientId": "my-client-id", + "aadClientSecret": "", + "aadClientCertPath": "", + "aadFederatedTokenFile": "/var/run/secrets/openshift/serviceaccount/token", + "aadMSIDataPlaneIdentityPath": "", + "resourceGroup": "my-resource-group", + "location": "eastus", + "vnetName": "my-vnet", + "vnetResourceGroup": "my-vnet-rg", + "subnetName": "my-subnet", + "securityGroupName": "my-security-group", + "securityGroupResourceGroup": "my-sg-rg", + "routeTableName": "", + "cloudProviderBackoff": true, + "cloudProviderBackoffDuration": 6, + "useInstanceMetadata": true, + "loadBalancerSku": "standard", + "disableOutboundSNAT": true, + "loadBalancerName": "my-infra-ID", + "clusterServiceLoadBalancerHealthProbeMode": "shared", + "userAssignedIdentityID": "/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-mi" + } +kind: ConfigMap +metadata: + name: azure-cloud-config diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/azure/testdata/zz_fixture_TestConfig.yaml b/control-plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/azure/testdata/zz_fixture_TestConfig_When_health_probe_annotations_are_set_it_should_generate_valid_config.yaml similarity index 100% rename from control-plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/azure/testdata/zz_fixture_TestConfig.yaml rename to control-plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/azure/testdata/zz_fixture_TestConfig_When_health_probe_annotations_are_set_it_should_generate_valid_config.yaml diff --git a/docs/content/reference/aggregated-docs.md b/docs/content/reference/aggregated-docs.md index 3d478c247ad4..bcc20fa3b67b 100644 --- a/docs/content/reference/aggregated-docs.md +++ b/docs/content/reference/aggregated-docs.md @@ -33728,6 +33728,107 @@ This is only valid for self-managed Azure.

AzureClientID is a string that represents the client ID of a managed identity.

+###AzureContainerRegistryConfig { #hypershift.openshift.io/v1beta1.AzureContainerRegistryConfig } +

+(Appears on: +AzurePlatformSpec) +

+

+

AzureContainerRegistryConfig configures Azure Container Registry integration for a hosted cluster.

+

+ + + + + + + + + + + + + +
FieldDescription
+credentials,omitzero
+ + +AzureContainerRegistryCredentialConfig + + +		 +

credentials configures authentication for worker nodes pulling images from ACR +using a user-assigned managed identity. +The identity does not need to be in the same subscription or resource group as the +HostedCluster, but it must be in the same Azure AD tenant. The management cluster’s +CAPZ identity must have Microsoft.ManagedIdentity/userAssignedIdentities/*/assign/action +on the identity’s scope to attach it to worker virtual machines at creation time.

+
+###AzureContainerRegistryCredentialConfig { #hypershift.openshift.io/v1beta1.AzureContainerRegistryCredentialConfig } +

+(Appears on: +AzureContainerRegistryConfig) +

+

+

AzureContainerRegistryCredentialConfig configures authentication credentials for Azure Container Registry.

+

+ + + + + + + + + + + + + + + + + +
FieldDescription
+type
+ + +AzureContainerRegistryCredentialType + + +		 +

type specifies the credential type used for ACR image pulls.

+
+managedIdentity,omitzero
+ + +UserAssignedManagedIdentity + + +		 +(Optional) +

managedIdentity identifies the user-assigned managed identity used for ACR image pulls.

+
+###AzureContainerRegistryCredentialType { #hypershift.openshift.io/v1beta1.AzureContainerRegistryCredentialType } +

+(Appears on: +AzureContainerRegistryCredentialConfig) +

+

+

AzureContainerRegistryCredentialType identifies the type of credential used for ACR image pulls.

+

+ + + + + + + + + + +
ValueDescription

"ManagedIdentity"

AzureContainerRegistryCredentialManagedIdentity uses a user-assigned managed identity for ACR authentication.

+
###AzureDiagnosticsStorageAccountType { #hypershift.openshift.io/v1beta1.AzureDiagnosticsStorageAccountType }

(Appears on: @@ -33960,6 +34061,15 @@ and traffic must be routed through the private router (Swift).

+###AzureManagedIdentityResourceID { #hypershift.openshift.io/v1beta1.AzureManagedIdentityResourceID } +

+(Appears on: +UserAssignedManagedIdentity) +

+

+

AzureManagedIdentityResourceID is an ARM resource ID for a user-assigned managed identity +in the format /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{name}.

+

###AzureMarketplaceImage { #hypershift.openshift.io/v1beta1.AzureMarketplaceImage }

(Appears on: @@ -34428,6 +34538,24 @@ string +containerRegistry,omitzero
+ + +AzureContainerRegistryConfig + + + + +(Optional) +

containerRegistry configures how worker nodes authenticate to Azure Container Registry (ACR). +When set, the managed identity is attached to worker virtual machines and its resource ID is +written into the worker cloud provider config so kubelet’s ACR credential provider can +authenticate without image pull secrets. +Changing this value will trigger a rollout for all existing NodePools in the cluster.

+ + + + topology
@@ -47312,6 +47440,41 @@ capacity.

+###UserAssignedManagedIdentity { #hypershift.openshift.io/v1beta1.UserAssignedManagedIdentity } +

+(Appears on: +AzureContainerRegistryCredentialConfig) +

+

+

UserAssignedManagedIdentity identifies a user-assigned managed identity by its ARM resource ID.

+

+ + + + + + + + + + + + + +
FieldDescription
+resourceID
+ + +AzureManagedIdentityResourceID + + +		 +

resourceID is the ARM resource ID of the user-assigned managed identity +in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}. +The identity must have the AcrPull role on the target Azure Container Registry. +It does not need to be in the same subscription or resource group as the HostedCluster, +but it must be in the same Azure AD tenant.

+
###UserManagedDiagnostics { #hypershift.openshift.io/v1beta1.UserManagedDiagnostics }

(Appears on: diff --git a/docs/content/reference/api.md b/docs/content/reference/api.md index a8bdb5209460..19f1abee9971 100644 --- a/docs/content/reference/api.md +++ b/docs/content/reference/api.md @@ -3277,6 +3277,107 @@ This is only valid for self-managed Azure.

AzureClientID is a string that represents the client ID of a managed identity.

+###AzureContainerRegistryConfig { #hypershift.openshift.io/v1beta1.AzureContainerRegistryConfig } +

+(Appears on: +AzurePlatformSpec) +

+

+

AzureContainerRegistryConfig configures Azure Container Registry integration for a hosted cluster.

+

+ + + + + + + + + + + + + +
FieldDescription
+credentials,omitzero
+ + +AzureContainerRegistryCredentialConfig + + +		 +

credentials configures authentication for worker nodes pulling images from ACR +using a user-assigned managed identity. +The identity does not need to be in the same subscription or resource group as the +HostedCluster, but it must be in the same Azure AD tenant. The management cluster’s +CAPZ identity must have Microsoft.ManagedIdentity/userAssignedIdentities/*/assign/action +on the identity’s scope to attach it to worker virtual machines at creation time.

+
+###AzureContainerRegistryCredentialConfig { #hypershift.openshift.io/v1beta1.AzureContainerRegistryCredentialConfig } +

+(Appears on: +AzureContainerRegistryConfig) +

+

+

AzureContainerRegistryCredentialConfig configures authentication credentials for Azure Container Registry.

+

+ + + + + + + + + + + + + + + + + +
FieldDescription
+type
+ + +AzureContainerRegistryCredentialType + + +		 +

type specifies the credential type used for ACR image pulls.

+
+managedIdentity,omitzero
+ + +UserAssignedManagedIdentity + + +		 +(Optional) +

managedIdentity identifies the user-assigned managed identity used for ACR image pulls.

+
+###AzureContainerRegistryCredentialType { #hypershift.openshift.io/v1beta1.AzureContainerRegistryCredentialType } +

+(Appears on: +AzureContainerRegistryCredentialConfig) +

+

+

AzureContainerRegistryCredentialType identifies the type of credential used for ACR image pulls.

+

+ + + + + + + + + + +
ValueDescription

"ManagedIdentity"

AzureContainerRegistryCredentialManagedIdentity uses a user-assigned managed identity for ACR authentication.

+
###AzureDiagnosticsStorageAccountType { #hypershift.openshift.io/v1beta1.AzureDiagnosticsStorageAccountType }

(Appears on: @@ -3509,6 +3610,15 @@ and traffic must be routed through the private router (Swift).

+###AzureManagedIdentityResourceID { #hypershift.openshift.io/v1beta1.AzureManagedIdentityResourceID } +

+(Appears on: +UserAssignedManagedIdentity) +

+

+

AzureManagedIdentityResourceID is an ARM resource ID for a user-assigned managed identity +in the format /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{name}.

+

###AzureMarketplaceImage { #hypershift.openshift.io/v1beta1.AzureMarketplaceImage }

(Appears on: @@ -3977,6 +4087,24 @@ string +containerRegistry,omitzero
+ + +AzureContainerRegistryConfig + + + + +(Optional) +

containerRegistry configures how worker nodes authenticate to Azure Container Registry (ACR). +When set, the managed identity is attached to worker virtual machines and its resource ID is +written into the worker cloud provider config so kubelet’s ACR credential provider can +authenticate without image pull secrets. +Changing this value will trigger a rollout for all existing NodePools in the cluster.

+ + + + topology
@@ -16861,6 +16989,41 @@ capacity.

+###UserAssignedManagedIdentity { #hypershift.openshift.io/v1beta1.UserAssignedManagedIdentity } +

+(Appears on: +AzureContainerRegistryCredentialConfig) +

+

+

UserAssignedManagedIdentity identifies a user-assigned managed identity by its ARM resource ID.

+

+ + + + + + + + + + + + + +
FieldDescription
+resourceID
+ + +AzureManagedIdentityResourceID + + +		 +

resourceID is the ARM resource ID of the user-assigned managed identity +in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}. +The identity must have the AcrPull role on the target Azure Container Registry. +It does not need to be in the same subscription or resource group as the HostedCluster, +but it must be in the same Azure AD tenant.

+
###UserManagedDiagnostics { #hypershift.openshift.io/v1beta1.UserManagedDiagnostics }

(Appears on: diff --git a/hypershift-operator/controllers/hostedcluster/metrics/metrics.go b/hypershift-operator/controllers/hostedcluster/metrics/metrics.go index e68ab0489509..d94a406aab99 100644 --- a/hypershift-operator/controllers/hostedcluster/metrics/metrics.go +++ b/hypershift-operator/controllers/hostedcluster/metrics/metrics.go @@ -95,6 +95,9 @@ const ( HostedClusterAzureInfoMetricName = "hosted_cluster_azure_info" HostedClusterAzureInfoMetricHelp = "Reports Azure information about the given HostedCluster" + + AcrPullIdentityConfiguredMetricName = "hypershift_cluster_acr_pull_identity_configured" + acrPullIdentityConfiguredMetricHelp = "Indicates whether a HostedCluster has an ACR pull managed identity configured (1=configured, 0=not configured). Only emitted for Azure platform clusters." ) // semantically constant - not supposed to be changed at runtime @@ -204,6 +207,10 @@ var ( "location", "microsoft_subscription_id", "microsoft_resource_group_name"), nil) + + acrPullIdentityConfiguredMetricDesc = prometheus.NewDesc( + AcrPullIdentityConfiguredMetricName, acrPullIdentityConfiguredMetricHelp, + hclusterLabels, nil) ) type hostedClustersMetricsCollector struct { @@ -556,6 +563,20 @@ func (c *hostedClustersMetricsCollector) Collect(ch chan<- prometheus.Metric) { } } + // acrPullIdentityConfiguredMetric + if hcluster.Spec.Platform.Azure != nil { + var acrValue float64 + if hcluster.Spec.Platform.Azure.ContainerRegistry.Credentials.Type != "" { + acrValue = 1 + } + ch <- prometheus.MustNewConstMetric( + acrPullIdentityConfiguredMetricDesc, + prometheus.GaugeValue, + acrValue, + hclusterLabelValues..., + ) + } + // invalidAwsCredsMetric { // Use detailed credential status: 0=valid, 1=invalid, 2=unknown diff --git a/hypershift-operator/controllers/hostedcluster/metrics/metrics_test.go b/hypershift-operator/controllers/hostedcluster/metrics/metrics_test.go index 34fce2ccbfa8..f6ee01445419 100644 --- a/hypershift-operator/controllers/hostedcluster/metrics/metrics_test.go +++ b/hypershift-operator/controllers/hostedcluster/metrics/metrics_test.go @@ -1280,6 +1280,104 @@ func TestHostedClusterAzureInfo(t *testing.T) { } } +func TestAcrPullIdentityConfigured(t *testing.T) { + const ( + zero float64 = 0 + one float64 = 1 + ) + testCases := []struct { + name string + platformType hyperv1.PlatformType + azureSpec *hyperv1.AzurePlatformSpec + expectedMetricName string + expected *dto.MetricFamily + }{ + { + name: "When platform is not Azure it should not emit a metric", + platformType: hyperv1.AWSPlatform, + expectedMetricName: AcrPullIdentityConfiguredMetricName, + }, + { + name: "When Azure has no containerRegistry configured it should emit 0", + platformType: hyperv1.AzurePlatform, + azureSpec: &hyperv1.AzurePlatformSpec{ + Cloud: "AzureCloud", + Location: "eastus", + ResourceGroupName: "myRG", + SubscriptionID: "mySub", + }, + expectedMetricName: AcrPullIdentityConfiguredMetricName, + expected: &dto.MetricFamily{ + Name: ptr.To(AcrPullIdentityConfiguredMetricName), + Help: ptr.To(acrPullIdentityConfiguredMetricHelp), + Type: func() *dto.MetricType { v := dto.MetricType(1); return &v }(), + Metric: []*dto.Metric{{ + Label: []*dto.LabelPair{ + {Name: ptr.To("_id"), Value: ptr.To("this-is-the-clusterID")}, + {Name: ptr.To("name"), Value: ptr.To("hc-name")}, + {Name: ptr.To("namespace"), Value: ptr.To("hc-ns")}, + }, + Gauge: &dto.Gauge{Value: ptr.To(zero)}, + }}, + }, + }, + { + name: "When Azure has a containerRegistry configured it should emit 1", + platformType: hyperv1.AzurePlatform, + azureSpec: &hyperv1.AzurePlatformSpec{ + Cloud: "AzureCloud", + Location: "eastus", + ResourceGroupName: "myRG", + SubscriptionID: "mySub", + ContainerRegistry: hyperv1.AzureContainerRegistryConfig{ + Credentials: hyperv1.AzureContainerRegistryCredentialConfig{ + Type: hyperv1.AzureContainerRegistryCredentialManagedIdentity, + ManagedIdentity: hyperv1.UserAssignedManagedIdentity{ + ResourceID: "/subscriptions/mySub/resourceGroups/myRG/providers/Microsoft.ManagedIdentity/userAssignedIdentities/acr-pull-mi", + }, + }, + }, + }, + expectedMetricName: AcrPullIdentityConfiguredMetricName, + expected: &dto.MetricFamily{ + Name: ptr.To(AcrPullIdentityConfiguredMetricName), + Help: ptr.To(acrPullIdentityConfiguredMetricHelp), + Type: func() *dto.MetricType { v := dto.MetricType(1); return &v }(), + Metric: []*dto.Metric{{ + Label: []*dto.LabelPair{ + {Name: ptr.To("_id"), Value: ptr.To("this-is-the-clusterID")}, + {Name: ptr.To("name"), Value: ptr.To("hc-name")}, + {Name: ptr.To("namespace"), Value: ptr.To("hc-ns")}, + }, + Gauge: &dto.Gauge{Value: ptr.To(one)}, + }}, + }, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + hostedCluster := &hyperv1.HostedCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hc-name", + Namespace: "hc-ns", + }, + Spec: hyperv1.HostedClusterSpec{ + ClusterID: "this-is-the-clusterID", + Platform: hyperv1.PlatformSpec{ + Type: tc.platformType, + Azure: tc.azureSpec, + }, + }, + } + checkMetric(t, + fake.NewClientBuilder().WithScheme(api.Scheme).WithObjects(hostedCluster).Build(), + clocktesting.NewFakeClock(now), + tc.expectedMetricName, + tc.expected) + }) + } +} + func TestReportTransitionDurationForAWSEndpointConditions(t *testing.T) { testCases := []struct { name string diff --git a/hypershift-operator/controllers/nodepool/azure.go b/hypershift-operator/controllers/nodepool/azure.go index 40965d853bc0..c0051b479fa0 100644 --- a/hypershift-operator/controllers/nodepool/azure.go +++ b/hypershift-operator/controllers/nodepool/azure.go @@ -14,6 +14,7 @@ import ( "k8s.io/utils/ptr" capiazure "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" + capzutil "sigs.k8s.io/cluster-api-provider-azure/util/azure" "github.com/blang/semver" ) @@ -167,7 +168,7 @@ func getAzureMarketplaceMetadata(releaseImage *releaseinfo.ReleaseImage, arch st return result, nil } -func azureMachineTemplateSpec(nodePool *hyperv1.NodePool) (*capiazure.AzureMachineTemplateSpec, error) { +func azureMachineTemplateSpec(nodePool *hyperv1.NodePool, acrIdentityResourceID string) (*capiazure.AzureMachineTemplateSpec, error) { subnetName, err := azureutil.GetSubnetNameFromSubnetID(nodePool.Spec.Platform.Azure.SubnetID) if err != nil { return nil, fmt.Errorf("failed to determine subnet name for Azure machine: %w", err) @@ -244,6 +245,16 @@ func azureMachineTemplateSpec(nodePool *hyperv1.NodePool) (*capiazure.AzureMachi } } + if acrIdentityResourceID != "" { + azureMachineTemplate.Template.Spec.Identity = capiazure.VMIdentityUserAssigned + azureMachineTemplate.Template.Spec.UserAssignedIdentities = append( + azureMachineTemplate.Template.Spec.UserAssignedIdentities, + capiazure.UserAssignedIdentity{ + ProviderID: capzutil.ProviderIDPrefix + acrIdentityResourceID, + }, + ) + } + azureMachineTemplate.Template.Spec.SSHPublicKey = dummySSHKey return azureMachineTemplate, nil @@ -255,7 +266,12 @@ func (c *CAPI) azureMachineTemplate(ctx context.Context, templateNameGenerator f return nil, fmt.Errorf("failed to apply Azure image defaults: %w", err) } - spec, err := azureMachineTemplateSpec(c.nodePool) + var acrIdentityResourceID string + if c.hostedCluster != nil && c.hostedCluster.Spec.Platform.Azure != nil && c.hostedCluster.Spec.Platform.Azure.ContainerRegistry.Credentials.ManagedIdentity.ResourceID != "" { + acrIdentityResourceID = string(c.hostedCluster.Spec.Platform.Azure.ContainerRegistry.Credentials.ManagedIdentity.ResourceID) + } + + spec, err := azureMachineTemplateSpec(c.nodePool, acrIdentityResourceID) if err != nil { return nil, fmt.Errorf("failed to generate AzureMachineTemplateSpec: %w", err) } diff --git a/hypershift-operator/controllers/nodepool/azure_test.go b/hypershift-operator/controllers/nodepool/azure_test.go index 7b5e996ab8fc..a7882cd0117f 100644 --- a/hypershift-operator/controllers/nodepool/azure_test.go +++ b/hypershift-operator/controllers/nodepool/azure_test.go @@ -22,6 +22,7 @@ func TestAzureMachineTemplateSpec(t *testing.T) { testCases := []struct { name string nodePool *hyperv1.NodePool + acrIdentityResourceID string expectedAzureMachineTemplateSpec *capiazure.AzureMachineTemplateSpec expectedErr bool expectedErrMsg string @@ -507,12 +508,164 @@ func TestAzureMachineTemplateSpec(t *testing.T) { expectedErr: true, expectedErrMsg: "failed to determine subnet name for Azure machine: failed to parse subnet name from \"/subscriptions/testSubscriptionID/resourceGroups/testResourceGroupName/providers/Microsoft.Network/virtualNetworks/testVnetName/subnets/\"", }, + { + name: "When HostedCluster has containerRegistry credentials set it should set UserAssigned identity", + nodePool: &hyperv1.NodePool{ + Spec: hyperv1.NodePoolSpec{ + Platform: hyperv1.NodePoolPlatform{ + Type: hyperv1.AzurePlatform, + Azure: &hyperv1.AzureNodePoolPlatform{ + Image: hyperv1.AzureVMImage{ + Type: hyperv1.ImageID, + ImageID: ptr.To("testImageID"), + }, + SubnetID: "/subscriptions/testSubscriptionID/resourceGroups/testResourceGroupName/providers/Microsoft.Network/virtualNetworks/testVnetName/subnets/testSubnetName", + VMSize: "Standard_D2_v2", + OSDisk: hyperv1.AzureNodePoolOSDisk{ + SizeGiB: 30, + DiskStorageAccountType: "Standard_LRS", + }, + }, + }, + }, + }, + acrIdentityResourceID: "/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-mi", + expectedAzureMachineTemplateSpec: &capiazure.AzureMachineTemplateSpec{ + Template: capiazure.AzureMachineTemplateResource{ + ObjectMeta: clusterv1.ObjectMeta{Labels: nil, Annotations: nil}, + Spec: capiazure.AzureMachineSpec{ + ProviderID: nil, + VMSize: "Standard_D2_v2", + FailureDomain: nil, + Image: &capiazure.Image{ + ID: ptr.To("testImageID"), + SharedGallery: nil, + Marketplace: nil, + ComputeGallery: nil, + }, + Identity: capiazure.VMIdentityUserAssigned, + UserAssignedIdentities: []capiazure.UserAssignedIdentity{ + {ProviderID: "azure:///subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-mi"}, + }, + SystemAssignedIdentityRole: nil, + RoleAssignmentName: "", + OSDisk: capiazure.OSDisk{ + OSType: "", + DiskSizeGB: ptr.To[int32](30), + ManagedDisk: &capiazure.ManagedDiskParameters{ + StorageAccountType: "Standard_LRS", + DiskEncryptionSet: nil, + SecurityProfile: nil, + }, + DiffDiskSettings: nil, + CachingType: "", + }, + DataDisks: nil, + SSHPublicKey: dummySSHKey, + AdditionalTags: nil, + AdditionalCapabilities: nil, + AllocatePublicIP: false, + EnableIPForwarding: false, + AcceleratedNetworking: nil, + Diagnostics: nil, + SpotVMOptions: nil, + SecurityProfile: nil, + SubnetName: "", + DNSServers: nil, + VMExtensions: nil, + NetworkInterfaces: []capiazure.NetworkInterface{ + { + SubnetName: "testSubnetName", + PrivateIPConfigs: 0, + AcceleratedNetworking: nil, + }, + }, + CapacityReservationGroupID: nil, + }, + }, + }, + expectedErr: false, + }, + { + name: "When HostedCluster has no containerRegistry credentials it should not set UserAssigned identity", + nodePool: &hyperv1.NodePool{ + Spec: hyperv1.NodePoolSpec{ + Platform: hyperv1.NodePoolPlatform{ + Type: hyperv1.AzurePlatform, + Azure: &hyperv1.AzureNodePoolPlatform{ + Image: hyperv1.AzureVMImage{ + Type: hyperv1.ImageID, + ImageID: ptr.To("testImageID"), + }, + SubnetID: "/subscriptions/testSubscriptionID/resourceGroups/testResourceGroupName/providers/Microsoft.Network/virtualNetworks/testVnetName/subnets/testSubnetName", + VMSize: "Standard_D2_v2", + OSDisk: hyperv1.AzureNodePoolOSDisk{ + SizeGiB: 30, + DiskStorageAccountType: "Standard_LRS", + }, + }, + }, + }, + }, + expectedAzureMachineTemplateSpec: &capiazure.AzureMachineTemplateSpec{ + Template: capiazure.AzureMachineTemplateResource{ + ObjectMeta: clusterv1.ObjectMeta{Labels: nil, Annotations: nil}, + Spec: capiazure.AzureMachineSpec{ + ProviderID: nil, + VMSize: "Standard_D2_v2", + FailureDomain: nil, + Image: &capiazure.Image{ + ID: ptr.To("testImageID"), + SharedGallery: nil, + Marketplace: nil, + ComputeGallery: nil, + }, + UserAssignedIdentities: nil, + SystemAssignedIdentityRole: nil, + RoleAssignmentName: "", + OSDisk: capiazure.OSDisk{ + OSType: "", + DiskSizeGB: ptr.To[int32](30), + ManagedDisk: &capiazure.ManagedDiskParameters{ + StorageAccountType: "Standard_LRS", + DiskEncryptionSet: nil, + SecurityProfile: nil, + }, + DiffDiskSettings: nil, + CachingType: "", + }, + DataDisks: nil, + SSHPublicKey: dummySSHKey, + AdditionalTags: nil, + AdditionalCapabilities: nil, + AllocatePublicIP: false, + EnableIPForwarding: false, + AcceleratedNetworking: nil, + Diagnostics: nil, + SpotVMOptions: nil, + SecurityProfile: nil, + SubnetName: "", + DNSServers: nil, + VMExtensions: nil, + NetworkInterfaces: []capiazure.NetworkInterface{ + { + SubnetName: "testSubnetName", + PrivateIPConfigs: 0, + AcceleratedNetworking: nil, + }, + }, + CapacityReservationGroupID: nil, + }, + }, + }, + expectedErr: false, + }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { g := NewGomegaWithT(t) - azureSpec, err := azureMachineTemplateSpec(tc.nodePool) + azureSpec, err := azureMachineTemplateSpec(tc.nodePool, tc.acrIdentityResourceID) if tc.expectedErr { g.Expect(err.Error()).To(ContainSubstring(tc.expectedErrMsg)) } else { diff --git a/hypershift-operator/controllers/nodepool/config.go b/hypershift-operator/controllers/nodepool/config.go index b86aab5b713d..74de33f5e609 100644 --- a/hypershift-operator/controllers/nodepool/config.go +++ b/hypershift-operator/controllers/nodepool/config.go @@ -10,6 +10,7 @@ import ( "strings" hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + cpomanifests "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/manifests" "github.com/openshift/hypershift/support/api" "github.com/openshift/hypershift/support/backwardcompat" "github.com/openshift/hypershift/support/capabilities" @@ -23,6 +24,7 @@ import ( "github.com/openshift/api/operator/v1alpha1" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" serializer "k8s.io/apimachinery/pkg/runtime/serializer/json" @@ -351,3 +353,39 @@ func globalConfigString(hcluster *hyperv1.HostedCluster) (string, error) { // Some fields in the ClusterConfiguration have changes that are not backwards compatible with older versions of the CPO. return backwardcompat.GetBackwardCompatibleConfigString(rawConfig), nil } + +func (cg *ConfigGenerator) GetCloudConfigHash(ctx context.Context) (string, error) { + var cm *corev1.ConfigMap + switch cg.hostedCluster.Spec.Platform.Type { + case hyperv1.AzurePlatform: + cm = cpomanifests.AzureProviderConfig(cg.controlplaneNamespace) + case hyperv1.OpenStackPlatform: + cm = cpomanifests.OpenStackProviderConfig(cg.controlplaneNamespace) + default: + return "", nil + } + + if err := cg.Get(ctx, client.ObjectKeyFromObject(cm), cm); err != nil { + if apierrors.IsNotFound(err) { + return "", nil + } + return "", fmt.Errorf("failed to get cloud config ConfigMap %s/%s: %w", cm.Namespace, cm.Name, err) + } + + if len(cm.Data) == 0 { + return "", nil + } + keys := make([]string, 0, len(cm.Data)) + for k := range cm.Data { + keys = append(keys, k) + } + sort.Strings(keys) + var b strings.Builder + for _, k := range keys { + b.WriteString(k) + b.WriteByte(0) + b.WriteString(cm.Data[k]) + b.WriteByte(0) + } + return supportutil.HashSimple(b.String()), nil +} diff --git a/hypershift-operator/controllers/nodepool/config_test.go b/hypershift-operator/controllers/nodepool/config_test.go index ef9096ab218c..4695dd03e89b 100644 --- a/hypershift-operator/controllers/nodepool/config_test.go +++ b/hypershift-operator/controllers/nodepool/config_test.go @@ -3,6 +3,7 @@ package nodepool import ( "bytes" "compress/gzip" + "context" "errors" "fmt" "io" @@ -12,6 +13,7 @@ import ( hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" "github.com/openshift/hypershift/api/util/ipnet" + cpomanifests "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/manifests" api "github.com/openshift/hypershift/support/api" "github.com/openshift/hypershift/support/releaseinfo" supportutil "github.com/openshift/hypershift/support/util" @@ -24,6 +26,7 @@ import ( crclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "github.com/google/go-cmp/cmp" ) @@ -1812,3 +1815,144 @@ func TestGlobalConfigString(t *testing.T) { }) } } + +func TestCloudConfigHash(t *testing.T) { + controlPlaneNamespace := "test-cp" + azureCloudConfig := cpomanifests.AzureProviderConfig(controlPlaneNamespace) + azureCloudConfig.Data = map[string]string{ + "cloud.conf": `{"tenantId":"t1","subscriptionId":"s1"}`, + } + + azureCloudConfigDifferent := cpomanifests.AzureProviderConfig(controlPlaneNamespace) + azureCloudConfigDifferent.Data = map[string]string{ + "cloud.conf": `{"tenantId":"t1","subscriptionId":"s1","userAssignedIdentityID":"new-id"}`, + } + + testCases := []struct { + name string + platform hyperv1.PlatformType + objects []crclient.Object + expectEmpty bool + }{ + { + name: "When platform is Azure and cloud config exists it should return a hash", + platform: hyperv1.AzurePlatform, + objects: []crclient.Object{azureCloudConfig.DeepCopy()}, + }, + { + name: "When platform is Azure and cloud config is missing it should return empty", + platform: hyperv1.AzurePlatform, + objects: []crclient.Object{}, + expectEmpty: true, + }, + { + name: "When platform is AWS it should return empty", + platform: hyperv1.AWSPlatform, + objects: []crclient.Object{}, + expectEmpty: true, + }, + { + name: "When cloud config content changes it should produce a different hash", + platform: hyperv1.AzurePlatform, + objects: []crclient.Object{azureCloudConfigDifferent.DeepCopy()}, + }, + { + name: "When platform is OpenStack and cloud config exists it should return a hash", + platform: hyperv1.OpenStackPlatform, + objects: []crclient.Object{func() crclient.Object { + cm := cpomanifests.OpenStackProviderConfig(controlPlaneNamespace) + cm.Data = map[string]string{"cloud.conf": `[Global]\nauth-url=https://openstack.example.com`} + return cm + }()}, + }, + { + name: "When platform is OpenStack and cloud config is missing it should return empty", + platform: hyperv1.OpenStackPlatform, + objects: []crclient.Object{}, + expectEmpty: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + fakeClient := fake.NewClientBuilder().WithScheme(api.Scheme).WithObjects(tc.objects...).Build() + + cg := &ConfigGenerator{ + Client: fakeClient, + hostedCluster: &hyperv1.HostedCluster{ + Spec: hyperv1.HostedClusterSpec{ + Platform: hyperv1.PlatformSpec{ + Type: tc.platform, + }, + }, + }, + controlplaneNamespace: controlPlaneNamespace, + rolloutConfig: &rolloutConfig{}, + } + + hash, err := cg.GetCloudConfigHash(t.Context()) + g.Expect(err).ToNot(HaveOccurred()) + + if tc.expectEmpty { + g.Expect(hash).To(BeEmpty()) + } else { + g.Expect(hash).ToNot(BeEmpty()) + } + }) + } + + t.Run("When content differs the hashes should differ", func(t *testing.T) { + g := NewWithT(t) + + getHash := func(obj crclient.Object) string { + fakeClient := fake.NewClientBuilder().WithScheme(api.Scheme).WithObjects(obj).Build() + cg := &ConfigGenerator{ + Client: fakeClient, + hostedCluster: &hyperv1.HostedCluster{ + Spec: hyperv1.HostedClusterSpec{ + Platform: hyperv1.PlatformSpec{ + Type: hyperv1.AzurePlatform, + }, + }, + }, + controlplaneNamespace: controlPlaneNamespace, + rolloutConfig: &rolloutConfig{}, + } + hash, err := cg.GetCloudConfigHash(t.Context()) + g.Expect(err).ToNot(HaveOccurred()) + return hash + } + + hash1 := getHash(azureCloudConfig.DeepCopy()) + hash2 := getHash(azureCloudConfigDifferent.DeepCopy()) + g.Expect(hash1).ToNot(Equal(hash2)) + }) + + t.Run("When a non-NotFound error occurs it should return the error", func(t *testing.T) { + g := NewWithT(t) + injectedErr := errors.New("connection refused") + fakeClient := fake.NewClientBuilder().WithScheme(api.Scheme).WithInterceptorFuncs(interceptor.Funcs{ + Get: func(_ context.Context, _ crclient.WithWatch, _ crclient.ObjectKey, _ crclient.Object, _ ...crclient.GetOption) error { + return injectedErr + }, + }).Build() + + cg := &ConfigGenerator{ + Client: fakeClient, + hostedCluster: &hyperv1.HostedCluster{ + Spec: hyperv1.HostedClusterSpec{ + Platform: hyperv1.PlatformSpec{ + Type: hyperv1.AzurePlatform, + }, + }, + }, + controlplaneNamespace: controlPlaneNamespace, + rolloutConfig: &rolloutConfig{}, + } + + _, err := cg.GetCloudConfigHash(t.Context()) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("connection refused")) + }) +} diff --git a/hypershift-operator/controllers/nodepool/nodepool_controller.go b/hypershift-operator/controllers/nodepool/nodepool_controller.go index eb515d893416..20abace889a6 100644 --- a/hypershift-operator/controllers/nodepool/nodepool_controller.go +++ b/hypershift-operator/controllers/nodepool/nodepool_controller.go @@ -47,6 +47,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" "github.com/blang/semver" @@ -135,6 +136,10 @@ func (r *NodePoolReconciler) SetupWithManager(mgr ctrl.Manager) error { Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(enqueueParentNodePool), builder.WithPredicates(supportutil.PredicatesForHostedClusterAnnotationScoping(mgr.GetClient()))). // We want to reconcile when the ConfigMaps referenced by the spec.config and also the core ones change. Watches(&corev1.ConfigMap{}, handler.EnqueueRequestsFromMapFunc(r.enqueueNodePoolsForConfig), builder.WithPredicates(supportutil.PredicatesForHostedClusterAnnotationScoping(mgr.GetClient()))). + // We want to reconcile when cloud provider config ConfigMaps change in the control plane namespace. + Watches(&corev1.ConfigMap{}, handler.EnqueueRequestsFromMapFunc(r.enqueueNodePoolsForCloudConfig), builder.WithPredicates(predicate.NewPredicateFuncs(func(obj client.Object) bool { + return obj.GetName() == "azure-cloud-config" || obj.GetName() == "openstack-cloud-config" + }))). WithOptions(controller.Options{ RateLimiter: workqueue.NewTypedItemExponentialFailureRateLimiter[reconcile.Request](1*time.Second, 10*time.Second), MaxConcurrentReconciles: 10, @@ -897,6 +902,33 @@ func (r *NodePoolReconciler) enqueueNodePoolsForConfig(ctx context.Context, obj return result } +func (r *NodePoolReconciler) enqueueNodePoolsForCloudConfig(ctx context.Context, obj client.Object) []reconcile.Request { + hcpList := &hyperv1.HostedControlPlaneList{} + if err := r.List(ctx, hcpList, client.InNamespace(obj.GetNamespace())); err != nil || len(hcpList.Items) == 0 { + return nil + } + hcName, ok := hcpList.Items[0].Annotations[supportutil.HostedClusterAnnotation] + if !ok { + return nil + } + hc := supportutil.ParseNamespacedName(hcName) + + nodePoolList := &hyperv1.NodePoolList{} + if err := r.List(ctx, nodePoolList, client.InNamespace(hc.Namespace)); err != nil { + return nil + } + + var result []reconcile.Request + for i := range nodePoolList.Items { + if nodePoolList.Items[i].Spec.ClusterName == hc.Name { + result = append(result, reconcile.Request{ + NamespacedName: client.ObjectKeyFromObject(&nodePoolList.Items[i]), + }) + } + } + return result +} + // getNodePoolNamespace returns the namespaced name of a NodePool, given the NodePools name // and the control plane namespace name for the hosted cluster that this NodePool is a part of. func (r *NodePoolReconciler) getNodePoolNamespacedName(nodePoolName string, controlPlaneNamespace string) (types.NamespacedName, error) { diff --git a/hypershift-operator/controllers/nodepool/nodepool_controller_test.go b/hypershift-operator/controllers/nodepool/nodepool_controller_test.go index 6da1240148ab..b43d57e547ac 100644 --- a/hypershift-operator/controllers/nodepool/nodepool_controller_test.go +++ b/hypershift-operator/controllers/nodepool/nodepool_controller_test.go @@ -2338,3 +2338,127 @@ func TestNodePoolReconciler_reconcile(t *testing.T) { }) } } + +func TestEnqueueNodePoolsForCloudConfig(t *testing.T) { + t.Parallel() + hcNamespace := "clusters" + hcName := "my-cluster" + cpNamespace := "clusters-my-cluster" + + hcp := &hyperv1.HostedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: hcName, + Namespace: cpNamespace, + Annotations: map[string]string{ + util.HostedClusterAnnotation: hcNamespace + "/" + hcName, + }, + }, + } + + matchingNodePool := &hyperv1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "np-1", + Namespace: hcNamespace, + }, + Spec: hyperv1.NodePoolSpec{ + ClusterName: hcName, + }, + } + + unrelatedNodePool := &hyperv1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "np-other", + Namespace: hcNamespace, + }, + Spec: hyperv1.NodePoolSpec{ + ClusterName: "other-cluster", + }, + } + + testCases := []struct { + name string + cm *corev1.ConfigMap + objects []client.Object + expected []reconcile.Request + }{ + { + name: "When azure-cloud-config changes, it should enqueue matching NodePools", + cm: &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "azure-cloud-config", + Namespace: cpNamespace, + }, + }, + objects: []client.Object{hcp, matchingNodePool, unrelatedNodePool}, + expected: []reconcile.Request{ + {NamespacedName: types.NamespacedName{Name: "np-1", Namespace: hcNamespace}}, + }, + }, + { + name: "When openstack-cloud-config changes, it should enqueue matching NodePools", + cm: &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "openstack-cloud-config", + Namespace: cpNamespace, + }, + }, + objects: []client.Object{hcp, matchingNodePool}, + expected: []reconcile.Request{ + {NamespacedName: types.NamespacedName{Name: "np-1", Namespace: hcNamespace}}, + }, + }, + { + name: "When no HostedControlPlane exists in the namespace, it should return nil", + cm: &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "azure-cloud-config", + Namespace: cpNamespace, + }, + }, + objects: []client.Object{matchingNodePool}, + expected: nil, + }, + { + name: "When HostedControlPlane has no cluster annotation, it should return nil", + cm: &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "azure-cloud-config", + Namespace: cpNamespace, + }, + }, + objects: []client.Object{ + &hyperv1.HostedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: hcName, + Namespace: cpNamespace, + }, + }, + matchingNodePool, + }, + expected: nil, + }, + { + name: "When no NodePools match the HostedCluster, it should return empty", + cm: &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "azure-cloud-config", + Namespace: cpNamespace, + }, + }, + objects: []client.Object{hcp, unrelatedNodePool}, + expected: nil, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + + c := fake.NewClientBuilder().WithScheme(api.Scheme).WithObjects(tc.objects...).Build() + r := &NodePoolReconciler{Client: c} + + result := r.enqueueNodePoolsForCloudConfig(context.Background(), tc.cm) + g.Expect(result).To(Equal(tc.expected)) + }) + } +} diff --git a/hypershift-operator/controllers/nodepool/token.go b/hypershift-operator/controllers/nodepool/token.go index adf5b0c2c1de..59e9516e1e8f 100644 --- a/hypershift-operator/controllers/nodepool/token.go +++ b/hypershift-operator/controllers/nodepool/token.go @@ -40,6 +40,7 @@ const ( TokenSecretPullSecretHashKey = "pull-secret-hash" TokenSecretHCConfigurationHashKey = "hc-configuration-hash" TokenSecretAdditionalTrustBundleKey = "additional-trust-bundle-hash" + TokenSecretCloudConfigHashKey = "cloud-config-hash" TokenSecretConfigKey = "config" TokenSecretAnnotation = "hypershift.openshift.io/ignition-config" TokenSecretIgnitionReachedAnnotation = "hypershift.openshift.io/ignition-reached" @@ -60,6 +61,7 @@ type Token struct { pullSecretHash []byte additionalTrustBundleHash []byte globalConfigHash []byte + cloudConfigHash []byte userData *userData } @@ -112,6 +114,11 @@ func NewToken(ctx context.Context, configGenerator *ConfigGenerator, cpoCapabili return nil, fmt.Errorf("failed to hash HostedCluster configuration: %w", err) } + cloudConfigHash, err := configGenerator.GetCloudConfigHash(ctx) + if err != nil { + return nil, err + } + token := &Token{ CreateOrUpdateProvider: upsert.New(false), ConfigGenerator: configGenerator, @@ -119,6 +126,7 @@ func NewToken(ctx context.Context, configGenerator *ConfigGenerator, cpoCapabili pullSecretHash: []byte(supportutil.HashSimple(pullSecretBytes)), additionalTrustBundleHash: []byte(supportutil.HashSimple(additionalTrustBundle)), globalConfigHash: []byte(hcConfigurationHash), + cloudConfigHash: []byte(cloudConfigHash), } // User data input. @@ -353,11 +361,13 @@ func (t *Token) reconcileTokenSecret(tokenSecret *corev1.Secret) error { tokenSecret.Data[TokenSecretPullSecretHashKey] = t.pullSecretHash tokenSecret.Data[TokenSecretAdditionalTrustBundleKey] = t.additionalTrustBundleHash tokenSecret.Data[TokenSecretHCConfigurationHashKey] = t.globalConfigHash + tokenSecret.Data[TokenSecretCloudConfigHashKey] = t.cloudConfigHash } // TODO (alberto): Only apply this on creation and change the hash generation to only use triggering upgrade fields. // We let this change to happen inplace now as the tokenSecret and the mcs config use the whole spec.Config for the comparing hash. // Otherwise if something which does not trigger a new token generation from spec.Config changes, like .IDP, both hashes would mismatch forever. tokenSecret.Data[TokenSecretHCConfigurationHashKey] = t.globalConfigHash + tokenSecret.Data[TokenSecretCloudConfigHashKey] = t.cloudConfigHash return nil } diff --git a/hypershift-operator/controllers/nodepool/token_test.go b/hypershift-operator/controllers/nodepool/token_test.go index 420e1fd8aeea..6da03cfc40d1 100644 --- a/hypershift-operator/controllers/nodepool/token_test.go +++ b/hypershift-operator/controllers/nodepool/token_test.go @@ -100,6 +100,7 @@ func TestNewToken(t *testing.T) { }, nodePool: &hyperv1.NodePool{}, controlplaneNamespace: controlplaneNamespace, + rolloutConfig: &rolloutConfig{}, }, fakeObjects: []crclient.Object{ pullSecret, @@ -131,6 +132,7 @@ func TestNewToken(t *testing.T) { }, nodePool: &hyperv1.NodePool{}, controlplaneNamespace: controlplaneNamespace, + rolloutConfig: &rolloutConfig{}, }, fakeObjects: []crclient.Object{ pullSecret, @@ -162,6 +164,7 @@ func TestNewToken(t *testing.T) { }, nodePool: &hyperv1.NodePool{}, controlplaneNamespace: controlplaneNamespace, + rolloutConfig: &rolloutConfig{}, }, fakeObjects: []crclient.Object{ additionalTrustBundle, @@ -192,6 +195,7 @@ func TestNewToken(t *testing.T) { }, nodePool: &hyperv1.NodePool{}, controlplaneNamespace: controlplaneNamespace, + rolloutConfig: &rolloutConfig{}, }, fakeObjects: []crclient.Object{ pullSecret, @@ -222,6 +226,7 @@ func TestNewToken(t *testing.T) { }, nodePool: &hyperv1.NodePool{}, controlplaneNamespace: controlplaneNamespace, + rolloutConfig: &rolloutConfig{}, }, fakeObjects: []crclient.Object{ pullSecret, @@ -263,6 +268,7 @@ func TestNewToken(t *testing.T) { }, nodePool: &hyperv1.NodePool{}, controlplaneNamespace: controlplaneNamespace, + rolloutConfig: &rolloutConfig{}, }, fakeObjects: []crclient.Object{ pullSecret, @@ -368,6 +374,7 @@ func TestTokenCleanupOutdated(t *testing.T) { }, }, controlplaneNamespace: controlplaneNamespace, + rolloutConfig: &rolloutConfig{}, }, }, fakeObjects: []crclient.Object{ @@ -394,6 +401,7 @@ func TestTokenCleanupOutdated(t *testing.T) { }, }, controlplaneNamespace: controlplaneNamespace, + rolloutConfig: &rolloutConfig{}, }, }, fakeObjects: []crclient.Object{}, @@ -417,6 +425,7 @@ func TestTokenCleanupOutdated(t *testing.T) { }, }, controlplaneNamespace: controlplaneNamespace, + rolloutConfig: &rolloutConfig{}, }, }, fakeObjects: []crclient.Object{ diff --git a/ignition-server/cmd/run_local_ignitionprovider.go b/ignition-server/cmd/run_local_ignitionprovider.go index 38e13d246a8f..9652f172c490 100644 --- a/ignition-server/cmd/run_local_ignitionprovider.go +++ b/ignition-server/cmd/run_local_ignitionprovider.go @@ -111,7 +111,7 @@ func (o *RunLocalIgnitionProviderOptions) Run(ctx context.Context) error { FeatureGateManifest: o.FeatureGateManifest, } - payload, err := p.GetPayload(ctx, o.Image, config.String(), "", "", "") + payload, err := p.GetPayload(ctx, o.Image, config.String(), "", "", "", "") if err != nil { return err } diff --git a/ignition-server/controllers/cache.go b/ignition-server/controllers/cache.go index 91e4f1ec6f50..b7ec879173b2 100644 --- a/ignition-server/controllers/cache.go +++ b/ignition-server/controllers/cache.go @@ -32,8 +32,9 @@ type ExpiringCache struct { } type CacheValue struct { - Payload []byte - SecretName string + Payload []byte + SecretName string + CloudConfigHash string } type entry struct { diff --git a/ignition-server/controllers/local_ignitionprovider.go b/ignition-server/controllers/local_ignitionprovider.go index 4137e5893fd3..115d16ed5d41 100644 --- a/ignition-server/controllers/local_ignitionprovider.go +++ b/ignition-server/controllers/local_ignitionprovider.go @@ -12,6 +12,7 @@ import ( "os/exec" "path" "path/filepath" + "sort" "strings" "sync" "time" @@ -131,7 +132,7 @@ const ( managedTrustBundleName = "trusted-ca-bundle-managed" ) -func (p *LocalIgnitionProvider) GetPayload(ctx context.Context, releaseImage, customConfig, pullSecretHash, additionalTrustBundleHash, hcConfigurationHash string) ([]byte, error) { +func (p *LocalIgnitionProvider) GetPayload(ctx context.Context, releaseImage, customConfig, pullSecretHash, additionalTrustBundleHash, hcConfigurationHash, cloudConfigHash string) ([]byte, error) { p.lock.Lock() defer p.lock.Unlock() @@ -352,6 +353,25 @@ func (p *LocalIgnitionProvider) GetPayload(ctx context.Context, releaseImage, cu return nil, fmt.Errorf("failed to get cloud provider configmap: %w", err) } } + if cloudConfigHash != "" { + keys := make([]string, 0, len(cloudConfigMap.Data)) + for k := range cloudConfigMap.Data { + keys = append(keys, k) + } + sort.Strings(keys) + var b strings.Builder + for _, k := range keys { + b.WriteString(k) + b.WriteByte(0) + b.WriteString(cloudConfigMap.Data[k]) + b.WriteByte(0) + } + actualHash := util.HashSimple(b.String()) + if actualHash != cloudConfigHash { + return nil, fmt.Errorf("cloud config %s/%s hash mismatch (expected %s, got %s), waiting for update", + cloudConfigMap.Namespace, cloudConfigMap.Name, cloudConfigHash, actualHash) + } + } cloudConfYaml, err := yaml.Marshal(cloudConfigMap) if err != nil { return nil, fmt.Errorf("failed to marshal cloud config: %w", err) diff --git a/ignition-server/controllers/tokensecret_controller.go b/ignition-server/controllers/tokensecret_controller.go index 18332ff88c0c..e915897c65f8 100644 --- a/ignition-server/controllers/tokensecret_controller.go +++ b/ignition-server/controllers/tokensecret_controller.go @@ -35,7 +35,9 @@ const ( TokenSecretPullSecretHashKey = "pull-secret-hash" TokenSecretHCConfigurationHashKey = "hc-configuration-hash" TokenSecretAdditionalTrustBundleHashKey = "additional-trust-bundle-hash" + TokenSecretCloudConfigHashKey = "cloud-config-hash" InvalidConfigReason = "InvalidConfig" + CloudConfigPendingReason = "CloudConfigPending" TokenSecretReasonKey = "reason" TokenSecretAnnotation = "hypershift.openshift.io/ignition-config" TokenSecretNodePoolUpgradeType = "hypershift.openshift.io/node-pool-upgrade-type" @@ -83,7 +85,7 @@ func NewPayloadStore() *ExpiringCache { type IgnitionProvider interface { // GetPayload returns the ignition payload content for // the provided release image and a config string containing 0..N MachineConfig yaml definitions. - GetPayload(ctx context.Context, payloadImage, config, pullSecretHash, additionalTrustBundleHash, hcConfigurationHash string) ([]byte, error) + GetPayload(ctx context.Context, payloadImage, config, pullSecretHash, additionalTrustBundleHash, hcConfigurationHash, cloudConfigHash string) ([]byte, error) } // TokenSecretReconciler watches token Secrets @@ -226,25 +228,35 @@ func (r *TokenSecretReconciler) Reconcile(ctx context.Context, req ctrl.Request) } token := string(tokenSecret.Data[TokenSecretTokenKey]) + cloudConfigHash := string(tokenSecret.Data[TokenSecretCloudConfigHashKey]) if value, ok := r.PayloadStore.Get(token); ok { - log.Info("Payload found in cache") - - if tokenNeedRotation(timeLived) { - log.Info("Rotating token ID") - if err := r.rotateToken(ctx, tokenSecret, value, now); err != nil { - return ctrl.Result{}, err + if value.CloudConfigHash != cloudConfigHash { + log.Info("Cloud config hash changed, invalidating cached payload") + r.PayloadStore.Delete(token) + } else { + log.Info("Payload found in cache") + + if tokenNeedRotation(timeLived) { + log.Info("Rotating token ID") + if err := r.rotateToken(ctx, tokenSecret, value, now); err != nil { + return ctrl.Result{}, err + } + TokenRotationTotal.Inc() } - TokenRotationTotal.Inc() + return ctrl.Result{RequeueAfter: ttl/2 - durationDeref(timeLived)}, nil } - return ctrl.Result{RequeueAfter: ttl/2 - durationDeref(timeLived)}, nil } // If something else rotated the token (e.g. running in HA), we fall back to set the cache value from the old one. oldToken, ok := tokenSecret.Data[TokenSecretOldTokenKey] if ok { if value, ok := r.PayloadStore.Get(string(oldToken)); ok { - r.PayloadStore.Set(token, value) - return ctrl.Result{RequeueAfter: ttl/2 - durationDeref(timeLived)}, nil + if value.CloudConfigHash == cloudConfigHash { + r.PayloadStore.Set(token, value) + return ctrl.Result{RequeueAfter: ttl/2 - durationDeref(timeLived)}, nil + } + log.Info("Cloud config hash changed, invalidating old token cached payload") + r.PayloadStore.Delete(string(oldToken)) } } @@ -273,7 +285,7 @@ func (r *TokenSecretReconciler) Reconcile(ctx context.Context, req ctrl.Request) additionalTrustBundleHash := string(tokenSecret.Data[TokenSecretAdditionalTrustBundleHashKey]) payload, err := func() ([]byte, error) { start := time.Now() - payload, err := r.IgnitionProvider.GetPayload(ctx, releaseImage, config.String(), pullSecretHash, additionalTrustBundleHash, hcConfigurationHash) + payload, err := r.IgnitionProvider.GetPayload(ctx, releaseImage, config.String(), pullSecretHash, additionalTrustBundleHash, hcConfigurationHash, cloudConfigHash) if err != nil { return nil, fmt.Errorf("error getting ignition payload: %v", err) } @@ -286,12 +298,16 @@ func (r *TokenSecretReconciler) Reconcile(ctx context.Context, req ctrl.Request) // This patch could flood the API server, so we should only do it when the reason or message is different from the current one. // More info here: https://issues.redhat.com/browse/OCPBUGS-42320. errWithFullMsg := fmt.Errorf("failed to generate payload: %w", err) - if hasSameReasonAndMessage(tokenSecret, InvalidConfigReason, errWithFullMsg) { + reason := InvalidConfigReason + if strings.Contains(err.Error(), "hash mismatch") { + reason = CloudConfigPendingReason + } + if hasSameReasonAndMessage(tokenSecret, reason, errWithFullMsg) { return ctrl.Result{}, errWithFullMsg } patch := tokenSecret.DeepCopy() - patch.Data[TokenSecretReasonKey] = []byte(InvalidConfigReason) + patch.Data[TokenSecretReasonKey] = []byte(reason) patch.Data[TokenSecretMessageKey] = []byte(errWithFullMsg.Error()) if err := r.Client.Patch(ctx, patch, client.MergeFrom(tokenSecret)); err != nil { return ctrl.Result{}, fmt.Errorf("failed to patch tokenSecret with payload content: %w", err) @@ -301,12 +317,13 @@ func (r *TokenSecretReconciler) Reconcile(ctx context.Context, req ctrl.Request) } log.Info("IgnitionProvider generated payload") - r.PayloadStore.Set(token, CacheValue{Payload: payload, SecretName: tokenSecret.Name}) + cacheValue := CacheValue{Payload: payload, SecretName: tokenSecret.Name, CloudConfigHash: cloudConfigHash} + r.PayloadStore.Set(token, cacheValue) oldToken, ok = tokenSecret.Data[TokenSecretOldTokenKey] if ok { // If we got here and there's an old token e.g. ignition server pod was restarted, then we set it as well // So Machines that were given that token right before the restart can succeed. - r.PayloadStore.Set(string(oldToken), CacheValue{Payload: payload, SecretName: tokenSecret.Name}) + r.PayloadStore.Set(string(oldToken), cacheValue) } patch := tokenSecret.DeepCopy() diff --git a/ignition-server/controllers/tokensecret_controller_test.go b/ignition-server/controllers/tokensecret_controller_test.go index 1ced8c9bc476..d18e7885fbdd 100644 --- a/ignition-server/controllers/tokensecret_controller_test.go +++ b/ignition-server/controllers/tokensecret_controller_test.go @@ -30,7 +30,7 @@ var ( type fakeIgnitionProvider struct{} -func (p *fakeIgnitionProvider) GetPayload(ctx context.Context, releaseImage, config, pullSecretHash, additionalTrustBundleHash, hcConfigurationHash string) (payload []byte, err error) { +func (p *fakeIgnitionProvider) GetPayload(ctx context.Context, releaseImage, config, pullSecretHash, additionalTrustBundleHash, hcConfigurationHash, cloudConfigHash string) (payload []byte, err error) { return []byte(fakePayload), nil } @@ -610,6 +610,214 @@ func TestProcessedExpiredToken(t *testing.T) { } } +func TestCacheInvalidationOnCloudConfigHashChange(t *testing.T) { + compressedConfig, err := util.CompressAndEncode([]byte("compressedConfig")) + if err != nil { + t.Fatal(err) + } + compressedConfigBytes := compressedConfig.Bytes() + + tokenID := uuid.New().String() + secretName := "test" + + tests := []struct { + name string + cachedHash string + secretHash string + expectRegeneration bool + }{ + { + name: "When cloud config hash matches cached value, it should return cached payload", + cachedHash: "abc123", + secretHash: "abc123", + expectRegeneration: false, + }, + { + name: "When cloud config hash differs from cached value, it should regenerate payload", + cachedHash: "old-hash", + secretHash: "new-hash", + expectRegeneration: true, + }, + { + name: "When cloud config hash changes from non-empty to empty, it should regenerate payload", + cachedHash: "some-hash", + secretHash: "", + expectRegeneration: true, + }, + { + name: "When cloud config hash changes from empty to non-empty, it should regenerate payload", + cachedHash: "", + secretHash: "new-hash", + expectRegeneration: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: "test", + Annotations: map[string]string{ + TokenSecretAnnotation: "true", + TokenSecretTokenGenerationTime: time.Now().Format(time.RFC3339Nano), + }, + CreationTimestamp: metav1.Now(), + }, + Data: map[string][]byte{ + TokenSecretTokenKey: []byte(tokenID), + TokenSecretReleaseKey: []byte("release"), + TokenSecretConfigKey: compressedConfigBytes, + TokenSecretCloudConfigHashKey: []byte(tt.secretHash), + }, + } + + callCount := 0 + provider := &countingIgnitionProvider{count: &callCount} + + r := TokenSecretReconciler{ + Client: fake.NewClientBuilder().WithObjects(secret).Build(), + IgnitionProvider: provider, + PayloadStore: NewPayloadStore(), + } + + r.PayloadStore.Set(tokenID, CacheValue{ + Payload: []byte("old-payload"), + SecretName: secretName, + CloudConfigHash: tt.cachedHash, + }) + + _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(secret)}) + g.Expect(err).ToNot(HaveOccurred()) + + if tt.expectRegeneration { + g.Expect(callCount).To(Equal(1), "expected GetPayload to be called for regeneration") + value, found := r.PayloadStore.Get(tokenID) + g.Expect(found).To(BeTrue()) + g.Expect(value.CloudConfigHash).To(Equal(tt.secretHash)) + } else { + g.Expect(callCount).To(Equal(0), "expected cached payload to be returned without calling GetPayload") + } + }) + } +} + +func TestOldTokenFallbackWithCloudConfigHashMismatch(t *testing.T) { + g := NewWithT(t) + + compressedConfig, err := util.CompressAndEncode([]byte("config")) + g.Expect(err).ToNot(HaveOccurred()) + compressedConfigBytes := compressedConfig.Bytes() + + currentToken := "current-token" + oldToken := "old-token" + secretName := "test" + + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: "test", + Annotations: map[string]string{ + TokenSecretAnnotation: "true", + TokenSecretTokenGenerationTime: time.Now().Format(time.RFC3339Nano), + }, + CreationTimestamp: metav1.Now(), + }, + Data: map[string][]byte{ + TokenSecretTokenKey: []byte(currentToken), + TokenSecretOldTokenKey: []byte(oldToken), + TokenSecretReleaseKey: []byte("release"), + TokenSecretConfigKey: compressedConfigBytes, + TokenSecretCloudConfigHashKey: []byte("new-hash"), + }, + } + + callCount := 0 + provider := &countingIgnitionProvider{count: &callCount} + + r := TokenSecretReconciler{ + Client: fake.NewClientBuilder().WithObjects(secret).Build(), + IgnitionProvider: provider, + PayloadStore: NewPayloadStore(), + } + + r.PayloadStore.Set(oldToken, CacheValue{ + Payload: []byte("old-payload"), + SecretName: secretName, + CloudConfigHash: "old-hash", + }) + + _, err = r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(secret)}) + g.Expect(err).ToNot(HaveOccurred()) + + g.Expect(callCount).To(Equal(1), "expected GetPayload to be called when old token hash mismatches") + + value, found := r.PayloadStore.Get(currentToken) + g.Expect(found).To(BeTrue()) + g.Expect(value.CloudConfigHash).To(Equal("new-hash")) + + oldValue, found := r.PayloadStore.Get(oldToken) + g.Expect(found).To(BeTrue(), "old token should be re-cached with new payload after regeneration") + g.Expect(oldValue.CloudConfigHash).To(Equal("new-hash"), "old token cache should have updated hash") +} + +func TestCloudConfigPendingReasonOnHashMismatch(t *testing.T) { + g := NewWithT(t) + + compressedConfig, err := util.CompressAndEncode([]byte("config")) + g.Expect(err).ToNot(HaveOccurred()) + + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "test", + Annotations: map[string]string{ + TokenSecretAnnotation: "true", + TokenSecretTokenGenerationTime: time.Now().Format(time.RFC3339Nano), + }, + CreationTimestamp: metav1.Now(), + }, + Data: map[string][]byte{ + TokenSecretTokenKey: []byte("token"), + TokenSecretReleaseKey: []byte("release"), + TokenSecretConfigKey: compressedConfig.Bytes(), + TokenSecretCloudConfigHashKey: []byte("expected-hash"), + }, + } + + r := TokenSecretReconciler{ + Client: fake.NewClientBuilder().WithObjects(secret).Build(), + IgnitionProvider: &errorIgnitionProvider{err: fmt.Errorf("cloud config ns/name hash mismatch (expected a, got b), waiting for update")}, + PayloadStore: NewPayloadStore(), + } + + _, err = r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(secret)}) + g.Expect(err).To(HaveOccurred()) + + updated := &corev1.Secret{} + g.Expect(r.Client.Get(t.Context(), client.ObjectKeyFromObject(secret), updated)).To(Succeed()) + g.Expect(string(updated.Data[TokenSecretReasonKey])).To(Equal(CloudConfigPendingReason)) +} + +type countingIgnitionProvider struct { + count *int +} + +func (p *countingIgnitionProvider) GetPayload(_ context.Context, _, _, _, _, _, _ string) ([]byte, error) { + *p.count++ + return []byte("regenerated-payload"), nil +} + +type errorIgnitionProvider struct { + err error +} + +func (p *errorIgnitionProvider) GetPayload(_ context.Context, _, _, _, _, _, _ string) ([]byte, error) { + return nil, p.err +} + func TestHasSameReasonAndMessage(t *testing.T) { testCases := []struct { name string diff --git a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/azure.go b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/azure.go index d6258444e842..dafc4aa67154 100644 --- a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/azure.go +++ b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/azure.go @@ -469,6 +469,16 @@ type AzurePlatformSpec struct { // +kubebuilder:validation:MaxLength=255 TenantID string `json:"tenantID"` + // containerRegistry configures how worker nodes authenticate to Azure Container Registry (ACR). + // When set, the managed identity is attached to worker virtual machines and its resource ID is + // written into the worker cloud provider config so kubelet's ACR credential provider can + // authenticate without image pull secrets. + // Changing this value will trigger a rollout for all existing NodePools in the cluster. + // + // +rollout + // +optional + ContainerRegistry AzureContainerRegistryConfig `json:"containerRegistry,omitzero"` + // topology specifies the network topology of the API server endpoint for the hosted cluster. // - Public: The API server is accessible only via a public endpoint. // - PublicAndPrivate: The API server is accessible via both public and private endpoints. @@ -546,6 +556,67 @@ type AzureResourceManagedIdentities struct { // +kubebuilder:validation:Pattern=`^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$` type AzureClientID string +// AzureManagedIdentityResourceID is an ARM resource ID for a user-assigned managed identity +// in the format /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{name}. +// +// +kubebuilder:validation:XValidation:rule="self.lowerAscii().matches('^/subscriptions/[^/]+/resourcegroups/[^/]+/providers/microsoft\\\\.managedidentity/userassignedidentities/[^/]+$')",message="must be a user-assigned managed identity ARM resource ID in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}" +// +kubebuilder:validation:MinLength=131 +// +kubebuilder:validation:MaxLength=345 +type AzureManagedIdentityResourceID string + +// UserAssignedManagedIdentity identifies a user-assigned managed identity by its ARM resource ID. +type UserAssignedManagedIdentity struct { + // resourceID is the ARM resource ID of the user-assigned managed identity + // in the format /subscriptions/{subscriptionID}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}. + // The identity must have the AcrPull role on the target Azure Container Registry. + // It does not need to be in the same subscription or resource group as the HostedCluster, + // but it must be in the same Azure AD tenant. + // + // +required + ResourceID AzureManagedIdentityResourceID `json:"resourceID,omitempty"` +} + +// AzureContainerRegistryConfig configures Azure Container Registry integration for a hosted cluster. +type AzureContainerRegistryConfig struct { + // credentials configures authentication for worker nodes pulling images from ACR + // using a user-assigned managed identity. + // The identity does not need to be in the same subscription or resource group as the + // HostedCluster, but it must be in the same Azure AD tenant. The management cluster's + // CAPZ identity must have Microsoft.ManagedIdentity/userAssignedIdentities/*/assign/action + // on the identity's scope to attach it to worker virtual machines at creation time. + // + // +required + Credentials AzureContainerRegistryCredentialConfig `json:"credentials,omitzero"` +} + +// AzureContainerRegistryCredentialType identifies the type of credential used for ACR image pulls. +// +// +kubebuilder:validation:Enum=ManagedIdentity +type AzureContainerRegistryCredentialType string + +const ( + // AzureContainerRegistryCredentialManagedIdentity uses a user-assigned managed identity for ACR authentication. + AzureContainerRegistryCredentialManagedIdentity AzureContainerRegistryCredentialType = "ManagedIdentity" +) + +// AzureContainerRegistryCredentialConfig configures authentication credentials for Azure Container Registry. +// +// +kubebuilder:validation:XValidation:rule="self.type == 'ManagedIdentity' ? has(self.managedIdentity) : !has(self.managedIdentity)",message="managedIdentity is required when type is ManagedIdentity, and forbidden otherwise" +// +union +type AzureContainerRegistryCredentialConfig struct { + // type specifies the credential type used for ACR image pulls. + // + // +required + // +unionDiscriminator + Type AzureContainerRegistryCredentialType `json:"type,omitempty"` + + // managedIdentity identifies the user-assigned managed identity used for ACR image pulls. + // + // +optional + // +unionMember + ManagedIdentity UserAssignedManagedIdentity `json:"managedIdentity,omitzero"` +} + // AzureWorkloadIdentities is a struct that contains the client IDs of all the managed identities in self-managed Azure // needing to authenticate with Azure's API. type AzureWorkloadIdentities struct { diff --git a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/zz_generated.deepcopy.go b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/zz_generated.deepcopy.go index 3c1718831a7c..2c767b50dfc9 100644 --- a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/zz_generated.deepcopy.go +++ b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/zz_generated.deepcopy.go @@ -637,6 +637,38 @@ func (in *AzureAuthenticationConfiguration) DeepCopy() *AzureAuthenticationConfi return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureContainerRegistryConfig) DeepCopyInto(out *AzureContainerRegistryConfig) { + *out = *in + out.Credentials = in.Credentials +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureContainerRegistryConfig. +func (in *AzureContainerRegistryConfig) DeepCopy() *AzureContainerRegistryConfig { + if in == nil { + return nil + } + out := new(AzureContainerRegistryConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureContainerRegistryCredentialConfig) DeepCopyInto(out *AzureContainerRegistryCredentialConfig) { + *out = *in + out.ManagedIdentity = in.ManagedIdentity +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureContainerRegistryCredentialConfig. +func (in *AzureContainerRegistryCredentialConfig) DeepCopy() *AzureContainerRegistryCredentialConfig { + if in == nil { + return nil + } + out := new(AzureContainerRegistryCredentialConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AzureKMSKey) DeepCopyInto(out *AzureKMSKey) { *out = *in @@ -735,6 +767,7 @@ func (in *AzureNodePoolPlatform) DeepCopy() *AzureNodePoolPlatform { func (in *AzurePlatformSpec) DeepCopyInto(out *AzurePlatformSpec) { *out = *in in.AzureAuthenticationConfig.DeepCopyInto(&out.AzureAuthenticationConfig) + out.ContainerRegistry = in.ContainerRegistry in.Private.DeepCopyInto(&out.Private) } @@ -4709,6 +4742,21 @@ func (in *UnmanagedEtcdSpec) DeepCopy() *UnmanagedEtcdSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *UserAssignedManagedIdentity) DeepCopyInto(out *UserAssignedManagedIdentity) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UserAssignedManagedIdentity. +func (in *UserAssignedManagedIdentity) DeepCopy() *UserAssignedManagedIdentity { + if in == nil { + return nil + } + out := new(UserAssignedManagedIdentity) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *UserManagedDiagnostics) DeepCopyInto(out *UserManagedDiagnostics) { *out = *in