diff --git a/incubator/kruise/Chart.yaml b/incubator/kruise/Chart.yaml index 62fcf877..d1b14140 100644 --- a/incubator/kruise/Chart.yaml +++ b/incubator/kruise/Chart.yaml @@ -1,28 +1,27 @@ -annotations: - artifacthub.io/changes: | - - "[Changed]: https://github.com/openkruise/kruise/blob/master/CHANGELOG.md" - - "[Security]: Fix potential security issues of dependent packages" - - "[Changed]: Change kruise daemon dns policy to ClusterFirstWithHostNet" apiVersion: v1 -appVersion: 1.7.2 +name: kruise description: Helm chart for kruise components -home: https://openkruise.io +version: 1.8.3 +appVersion: 1.8.3 +kubeVersion: ">= 1.18.0-0" icon: https://cloudcache.tencent-cloud.com/qcloud/ui/static/Industry_tke/4a7ca09f-eb3a-461d-a830-ce4728a7258f.png maintainers: - name: rockerchen email: rockerchen@tencent.com keywords: -- openkruise -- kubernetes -- kruise -- workload -- statefulset -- sidecar -- job -- deployment -- cloneset -kubeVersion: '>= 1.18.0-0' -name: kruise + - openkruise + - kubernetes + - kruise + - workload + - statefulset + - sidecar + - job + - deployment + - cloneset +home: https://openkruise.io sources: -- https://github.com/openkruise/kruise -version: 1.7.2 + - https://github.com/openkruise/kruise +annotations: + artifacthub.io/changes: | + - "[Changed]: https://github.com/openkruise/kruise/blob/master/CHANGELOG.md" + - "[Security]: Fix potential security issues of dependent packages" diff --git a/incubator/kruise/README.md b/incubator/kruise/README.md index 55f79787..5568ac20 100644 --- a/incubator/kruise/README.md +++ b/incubator/kruise/README.md @@ -1,99 +1,113 @@ -# Kruise v1.7.2 +# Kruise v1.8.3 ## Configuration The following table lists the configurable parameters of the kruise chart and their default values. ## setup parameters -| Parameter | Description | Default | -| ----------------------------------------- | ------------------------------------------------------------ | ----------------------------- | -| `featureGates` | Feature gates for Kruise, empty string means all enabled | `""` | -| `installation.namespace` | Namespace for kruise installation | `kruise-system` | -| `installation.createNamespace` | Whether to create the installation.namespace | `true` | -| `installation.roleListGroups` | ApiGroups which kruise is permit to list, default set to be all | `*` | -| `crds.managed` | Kruise will not install CRDs with chart if this is false | `true` | -| `imagePullSecrets` | The list of image pull secrets for kruise image | `[]` | + +| Parameter | Description | Default | +|--------------------------------|-----------------------------------------------------------------|-----------------| +| `featureGates` | Feature gates for Kruise, empty string means all enabled | `""` | +| `installation.namespace` | Namespace for kruise installation | `kruise-system` | +| `installation.createNamespace` | Whether to create the installation.namespace | `true` | +| `installation.roleListGroups` | ApiGroups which kruise is permit to list, default set to be all | `*` | +| `crds.managed` | Kruise will not install CRDs with chart if this is false | `true` | +| `imagePullSecrets` | The list of image pull secrets for kruise image | `[]` | #### manager parameters -| Parameter | Description | Default | -| ----------------------------------------- | ------------------------------------------------------------ | ----------------------------- | -| `manager.log.level` | Log level that kruise-manager printed | `4` | -| `manager.replicas` | Replicas of kruise-controller-manager deployment | `2` | -| `manager.image.repository` | Repository for kruise-manager image | `ccr.ccs.tencentyun.com/tke-market/kruise-manager` | -| `manager.image.tag` | Tag for kruise-manager image | `v1.7.2` | -| `manager.resources.limits.cpu` | CPU resource limit of kruise-manager container | `200m` | -| `manager.resources.limits.memory` | Memory resource limit of kruise-manager container | `512Mi` | -| `manager.resources.requests.cpu` | CPU resource request of kruise-manager container | `100m` | -| `manager.resources.requests.memory` | Memory resource request of kruise-manager container | `256Mi` | -| `manager.metrics.port` | Port of metrics served | `8080` | -| `manager.webhook.port` | Port of webhook served | `9443` | -| `manager.pprofAddr` | Address of pprof served | `localhost:8090` | -| `manager.nodeAffinity` | Node affinity policy for kruise-manager pod | `{}` | -| `manager.nodeSelector` | Node labels for kruise-manager pod | `{}` | -| `manager.tolerations` | Tolerations for kruise-manager pod | `[]` | -| `manager.resyncPeriod` | Resync period of informer kruise-manager, defaults no resync | `0` | -| `manager.hostNetwork` | Whether kruise-manager pod should run with hostnetwork | `false` | -| `manager.loggingFormat` | Logging format, valid formats includes ` `(plain text), `json` | ` ` | + +| Parameter | Description | Default | +|-------------------------------------|----------------------------------------------------------------|-----------------------------| +| `manager.log.level` | Log level that kruise-manager printed | `4` | +| `manager.replicas` | Replicas of kruise-controller-manager deployment | `2` | +| `manager.image.repository` | Repository for kruise-manager image | `ccr.ccs.tencentyun.com/tke-market/kruise-manager` | +| `manager.image.tag` | Tag for kruise-manager image | `v1.8.3` | +| `manager.resources.limits.cpu` | CPU resource limit of kruise-manager container | `200m` | +| `manager.resources.limits.memory` | Memory resource limit of kruise-manager container | `512Mi` | +| `manager.resources.requests.cpu` | CPU resource request of kruise-manager container | `100m` | +| `manager.resources.requests.memory` | Memory resource request of kruise-manager container | `256Mi` | +| `manager.metrics.port` | Port of metrics served | `8080` | +| `manager.webhook.port` | Port of webhook served | `9443` | +| `manager.pprofAddr` | Address of pprof served | `localhost:8090` | +| `manager.nodeAffinity` | Node affinity policy for kruise-manager pod | `{}` | +| `manager.nodeSelector` | Node labels for kruise-manager pod | `{}` | +| `manager.tolerations` | Tolerations for kruise-manager pod | `[]` | +| `manager.resyncPeriod` | Resync period of informer kruise-manager, defaults no resync | `0` | +| `manager.hostNetwork` | Whether kruise-manager pod should run with hostnetwork | `false` | +| `manager.loggingFormat` | Logging format, valid formats includes ` `(plain text), `json` | ` ` | #### daemon parameters -| Parameter | Description | Default | -| ----------------------------------------- | ------------------------------------------------------------ | ----------------------------- | -| `daemon.extraEnvs` | Extra environment variables that will be pass onto pods | `[]` | -| `daemon.log.level` | Log level that kruise-daemon printed | `4` | -| `daemon.port` | Port of metrics and healthz that kruise-daemon served | `10221` | -| `daemon.pprofAddr` | Address of pprof served | `localhost:10222` | -| `daemon.resources.limits.cpu` | CPU resource limit of kruise-daemon container | `50m` | -| `daemon.resources.limits.memory` | Memory resource limit of kruise-daemon container | `128Mi` | -| `daemon.resources.requests.cpu` | CPU resource request of kruise-daemon container | `0` | -| `daemon.resources.requests.memory` | Memory resource request of kruise-daemon container | `0` | -| `daemon.affinity` | Affinity policy for kruise-daemon pod | `{}` | -| `daemon.socketLocation` | Location of the container manager control socket | `/var/run` | -| `daemon.socketFile` | Specify the socket file name in `socketLocation` (if you are not using containerd/docker/pouch/cri-o) | ` ` | -| `daemon.credentialProvider.enable` | Whether to enable credential provider for image pull job | `false` | -| `daemon.credentialProvider.hostPath` | node dir of the credential provider plugin, kruise-daemon will mount the dir as a hostpath volume | `credential-provider-plugin` | -| `daemon.credentialProvider.configmap` | configmap name of the credential provider in kruise-system ns | `credential-provider-config` | -| `daemon.credentialProvider.awsCredentialsDir` | aws credentials dir if using AWS, for example: `/root/.aws` | ` ` | + +| Parameter | Description | Default | +|-----------------------------------------------|-------------------------------------------------------------------------------------------------------|------------------------------| +| `daemon.extraEnvs` | Extra environment variables that will be pass onto pods | `[]` | +| `daemon.log.level` | Log level that kruise-daemon printed | `4` | +| `daemon.port` | Port of metrics and healthz that kruise-daemon served | `10221` | +| `daemon.pprofAddr` | Address of pprof served | `localhost:10222` | +| `daemon.resources.limits.cpu` | CPU resource limit of kruise-daemon container | `50m` | +| `daemon.resources.limits.memory` | Memory resource limit of kruise-daemon container | `128Mi` | +| `daemon.resources.requests.cpu` | CPU resource request of kruise-daemon container | `0` | +| `daemon.resources.requests.memory` | Memory resource request of kruise-daemon container | `0` | +| `daemon.affinity` | Affinity policy for kruise-daemon pod | `{}` | +| `daemon.socketLocation` | Location of the container manager control socket | `/var/run` | +| `daemon.socketFile` | Specify the socket file name in `socketLocation` (if you are not using containerd/docker/pouch/cri-o) | ` ` | +| `daemon.credentialProvider.enable` | Whether to enable credential provider for image pull job | `false` | +| `daemon.credentialProvider.hostPath` | node dir of the credential provider plugin, kruise-daemon will mount the dir as a hostpath volume | `credential-provider-plugin` | +| `daemon.credentialProvider.configmap` | configmap name of the credential provider in kruise-system ns | `credential-provider-config` | +| `daemon.credentialProvider.awsCredentialsDir` | aws credentials dir if using AWS, for example: `/root/.aws` | ` ` | ### other parameters -| Parameter | Description | Default | -| ----------------------------------------- | ------------------------------------------------------------ | ----------------------------- | -| `enableKubeCacheMutationDetector` | Whether to enable KUBE_CACHE_MUTATION_DETECTOR | `false` | -| `webhookConfiguration.timeoutSeconds` | The timeoutSeconds for all webhook configuration | `30` | -| `serviceAccount.annotations` | Annotations to patch for serviceAccounts | `{}` | -| `externalCerts.annotations` | Annotations to patch for webhook configuration and crd when featuregate `EnableExternalCerts` is enabled. For example, `cert-manager.io/inject-ca-from: kruise-system/kruise-webhook-certs`. | `{}` | -Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example, `helm install kruise https://... --set featureGates="AllAlpha=true"`. +| Parameter | Description | Default | +|---------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------| +| `enableKubeCacheMutationDetector` | Whether to enable KUBE_CACHE_MUTATION_DETECTOR | `false` | +| `webhookConfiguration.timeoutSeconds` | The timeoutSeconds for all webhook configuration | `30` | +| `serviceAccount.annotations` | Annotations to patch for serviceAccounts | `{}` | +| `externalCerts.annotations` | Annotations to patch for webhook configuration and crd when featuregate `EnableExternalCerts` is enabled. For example, `cert-manager.io/inject-ca-from: kruise-system/kruise-webhook-certs`. | `{}` | +| `helmHooks.image.repository` | Repository for kruise pre-delete hook image | `ccr.ccs.tencentyun.com/tke-market/kruise-helm-hook` | +| `helmHooks.image.tag` | Tag for kruise pre-delete hook image | `v0.1.0` | + +Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example, +`helm install kruise https://... --set featureGates="AllAlpha=true"`. ### Optional: feature-gate Feature-gate controls some influential features in Kruise: -| Name | Description | Default | Effect (if closed) | -|---------------------------------------------|-----------------------------------------------------------------------------------------------------------------------| ------- |-------------------------------------------------------------------------------------------------------------------| -| `PodWebhook` | Whether to open a webhook for Pod **create** | `true` | SidecarSet/KruisePodReadinessGate disabled | -| `KruiseDaemon` | Whether to deploy `kruise-daemon` DaemonSet | `true` | ImagePulling/ContainerRecreateRequest disabled | -| `DaemonWatchingPod` | Should each `kruise-daemon` watch pods on the same node | `true` | For in-place update with same imageID or env from labels/annotations | -| `CloneSetShortHash` | Enables CloneSet controller only set revision hash name to pod label | `false` | CloneSet name can not be longer than 54 characters | -| `KruisePodReadinessGate` | Enables Kruise webhook to inject 'KruisePodReady' readiness-gate to all Pods during creation | `false` | The readiness-gate will only be injected to Pods created by Kruise workloads | -| `PreDownloadImageForInPlaceUpdate` | Enables CloneSet controller to create ImagePullJobs to pre-download images for in-place update | `true` | No image pre-download for in-place update | -| `CloneSetPartitionRollback` | Enables CloneSet controller to rollback Pods to currentRevision when number of updateRevision pods is bigger than (replicas - partition) | `false` | CloneSet will only update Pods to updateRevision | -| `ResourcesDeletionProtection` | Enables protection for resources deletion | `true` | No protection for resources deletion | -| `TemplateNoDefaults` | Whether to disable defaults injection for pod/pvc template in workloads | `false` | Should not close this feature if it has open | -| `PodUnavailableBudgetDeleteGate` | Enables PodUnavailableBudget for pod deletion, eviction | `true` | No protection for pod deletion, eviction | -| `PodUnavailableBudgetUpdateGate` | Enables PodUnavailableBudget for pod.Spec update | `false` | No protection for in-place update | -| `WorkloadSpread` | Enables WorkloadSpread to manage multi-domain and elastic deploy | `true` | WorkloadSpread disabled | -| `InPlaceUpdateEnvFromMetadata` | Enables Kruise to in-place update a container in Pod when its env from labels/annotations changed and pod is in-place updating | `true` | Only container image can be in-place update | -| `StatefulSetAutoDeletePVC` | Enables policies controlling deletion of PVCs created by a StatefulSet | `true` | No deletion of PVCs by StatefulSet | -| `PreDownloadImageForDaemonSetUpdate` | Enables DaemonSet controller to create ImagePullJobs to pre-download images for in-place update | `false` | No image pre-download for in-place update | -| `PodProbeMarkerGate` | Whether to turn on PodProbeMarker ability | `true` | PodProbeMarker disabled | -| `SidecarSetPatchPodMetadataDefaultsAllowed` | Allow SidecarSet patch any annotations to Pod Object | `false` | Annotations are not allowed to patch randomly and need to be configured via SidecarSet_PatchPodMetadata_WhiteList | -| `SidecarTerminator` | SidecarTerminator enables SidecarTerminator to stop sidecar containers when all main containers exited | `false` | SidecarTerminator disabled | -| `CloneSetEventHandlerOptimization` | CloneSetEventHandlerOptimization enable optimization for cloneset-controller to reduce the queuing frequency cased by pod update | `false` | optimization for cloneset-controller to reduce the queuing frequency cased by pod update disabled | -| `PreparingUpdateAsUpdate` | PreparingUpdateAsUpdate enable CloneSet/Advanced StatefulSet controller to regard preparing-update Pod as updated when calculating update/current revision during scaling. | `false` | Pods at preparing update state will be regarded as current revision instead of update revision | -| `ImagePullJobGate` | ImagePullJobGate enable imagepulljob-controller execute ImagePullJob | `false` | ImagePullJob and PreDownloadImageForInPlaceUpdate are disabled | -| `ResourceDistributionGate` | ResourceDistributionGate enable resourcedistribution-controller execute ResourceDistribution. | `false` | ResourceDistribution disabled | -| `DeletionProtectionForCRDCascadingGate` | DeletionProtectionForCRDCascadingGate enable deletionProtection for crd Cascading | `false` | CustomResourceDefinition deletion protection disabled | -| `EnableExternalCerts` | Using certs generated externally, cert-manager e.g., for webhook server | `false` | kruise-manager will generate self-signed certs for webhook server | +| Name | Description | Default | Effect (if closed) | +|---------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------|-------------------------------------------------------------------------------------------------------------------------| +| `PodWebhook` | Whether to open a webhook for Pod **create** | `true` | SidecarSet/KruisePodReadinessGate disabled | +| `KruiseDaemon` | Whether to deploy `kruise-daemon` DaemonSet | `true` | ImagePulling/ContainerRecreateRequest disabled | +| `DaemonWatchingPod` | Should each `kruise-daemon` watch pods on the same node | `true` | For in-place update with same imageID or env from labels/annotations | +| `CloneSetShortHash` | Enables CloneSet controller only set revision hash name to pod label | `false` | CloneSet name can not be longer than 54 characters | +| `KruisePodReadinessGate` | Enables Kruise webhook to inject 'KruisePodReady' readiness-gate to all Pods during creation | `false` | The readiness-gate will only be injected to Pods created by Kruise workloads | +| `PreDownloadImageForInPlaceUpdate` | Enables CloneSet controller to create ImagePullJobs to pre-download images for in-place update | `true` | No image pre-download for in-place update | +| `CloneSetPartitionRollback` | Enables CloneSet controller to rollback Pods to currentRevision when number of updateRevision pods is bigger than (replicas - partition) | `false` | CloneSet will only update Pods to updateRevision | +| `ResourcesDeletionProtection` | Enables protection for resources deletion | `false` | No protection for resources deletion | +| `TemplateNoDefaults` | Whether to disable defaults injection for pod/pvc template in workloads | `false` | Should not close this feature if it has open | +| `PodUnavailableBudgetDeleteGate` | Enables PodUnavailableBudget for pod deletion, eviction | `true` | No protection for pod deletion, eviction | +| `PodUnavailableBudgetUpdateGate` | Enables PodUnavailableBudget for pod.Spec update | `false` | No protection for in-place update | +| `WorkloadSpread` | Enables WorkloadSpread to manage multi-domain and elastic deploy | `true` | WorkloadSpread disabled | +| `InPlaceUpdateEnvFromMetadata` | Enables Kruise to in-place update a container in Pod when its env from labels/annotations changed and pod is in-place updating | `true` | Only container image can be in-place update | +| `StatefulSetAutoDeletePVC` | Enables policies controlling deletion of PVCs created by a StatefulSet | `true` | No deletion of PVCs by StatefulSet | +| `PreDownloadImageForDaemonSetUpdate` | Enables DaemonSet controller to create ImagePullJobs to pre-download images for in-place update | `false` | No image pre-download for in-place update | +| `PodProbeMarkerGate` | Whether to turn on PodProbeMarker ability | `true` | PodProbeMarker disabled | +| `SidecarSetPatchPodMetadataDefaultsAllowed` | Allow SidecarSet patch any annotations to Pod Object | `false` | Annotations are not allowed to patch randomly and need to be configured via SidecarSet_PatchPodMetadata_WhiteList | +| `SidecarTerminator` | SidecarTerminator enables SidecarTerminator to stop sidecar containers when all main containers exited | `false` | SidecarTerminator disabled | +| `CloneSetEventHandlerOptimization` | CloneSetEventHandlerOptimization enable optimization for cloneset-controller to reduce the queuing frequency cased by pod update | `false` | optimization for cloneset-controller to reduce the queuing frequency cased by pod update disabled | +| `PreparingUpdateAsUpdate` | PreparingUpdateAsUpdate enable CloneSet/Advanced StatefulSet controller to regard preparing-update Pod as updated when calculating update/current revision during scaling. | `false` | Pods at preparing update state will be regarded as current revision instead of update revision | +| `ImagePullJobGate` | ImagePullJobGate enable imagepulljob-controller execute ImagePullJob | `false` | ImagePullJob and PreDownloadImageForInPlaceUpdate are disabled | +| `ResourceDistributionGate` | ResourceDistributionGate enable resourcedistribution-controller execute ResourceDistribution. | `false` | ResourceDistribution disabled | +| `DeletionProtectionForCRDCascadingGate` | DeletionProtectionForCRDCascadingGate enable deletionProtection for crd Cascading | `false` | CustomResourceDefinition deletion protection disabled | +| `EnableExternalCerts` | Using certs generated externally, cert-manager e.g., for webhook server | `false` | kruise-manager will generate self-signed certs for webhook server | +| `RecreatePodWhenChangeVCTInCloneSetGate` | Recreate the pod upon changing volume claim templates in a clone set to ensure PVC consistency | `false` | kruise-manager will recreate the pod upon changing volume claim templates in a clone set to ensure PVC consistency | +| `StatefulSetStartOrdinal` | Enables a StatefulSet to start from an arbitrary non zero ordinal | `false` | kruise-manager will enables a StatefulSet to start from an arbitrary non zero ordinal | +| `PodIndexLabel` | Set pod completion index as a pod label for Indexed Jobs. | `true` | kruise-manager will set pod completion index as a pod label for Indexed Jobs. | +| `StatefulSetAutoResizePVCGate` | Enables policies auto resizing PVCs created by a StatefulSet when user expands volumeClaimTemplates. | `false` | kruise-manager will enable policies auto resizing PVCs created by a StatefulSet when user expands volumeClaimTemplates. | +| `InPlaceWorkloadVerticalScaling` | Enables CloneSet/Advanced StatefulSet controller to support vertical scaling of managed Pods. | `false` | kruise-manager will enable CloneSet/Advanced StatefulSet controller to support vertical scaling of managed Pods. | +| `EnablePodProbeMarkerOnServerless` | Enables PodProbeMarker on Serverless Pod. | `false` | kruise-manager will enable PodProbeMarker on Serverless Pod. | +| `ForceDeleteTimeoutExpectationFeatureGate` | Enables delete timeout expectation, for example: cloneSet ScaleExpectation | `false` | kruise-manager will enable delete timeout expectation, for example: cloneSet ScaleExpectation | If you want to configure the feature-gate, just set the parameter when install or upgrade. Such as: @@ -106,7 +120,8 @@ If you want to enable all feature-gates, set the parameter as `featureGates=AllA ### Optional: the local image for China -If you are in China and have problem to pull image from official DockerHub, you can use the registry hosted on Alibaba Cloud: +If you are in China and have problem to pull image from official DockerHub, you can use the registry hosted on Alibaba +Cloud: ```bash $ helm install kruise https://... --set manager.image.repository=openkruise-registry.cn-hangzhou.cr.aliyuncs.com/openkruise/kruise-manager @@ -115,11 +130,14 @@ $ helm install kruise https://... --set manager.image.repository=openkruise-reg ### Optional: Support webhook CA injection using external certification management tool -Kruise needs certificates to enable mutating, validating and conversion webhooks. By default, kruise will generate self-signed certificates for webhook server. -If you want to use external certification management tool, e.g. cert-manager, you can follow these steps when install or upgrade: +Kruise needs certificates to enable mutating, validating and conversion webhooks. By default, kruise will generate +self-signed certificates for webhook server. +If you want to use external certification management tool, e.g. cert-manager, you can follow these steps when install or +upgrade: 1. Install external certification management tool, e.g. [cert-manager](https://cert-manager.io/docs/installation/helm/). 2. Create issuer and certificate resources if you have not done this before. + ```yaml apiVersion: cert-manager.io/v1 kind: Certificate @@ -133,8 +151,8 @@ spec: # DO NOT CHANGE THE SECRET NAME SINCE KRUISE READ CERTS FROM THIS SECRET secretName: kruise-webhook-certs dnsNames: - - kruise-webhook-service.kruise-system.svc - - localhost + - kruise-webhook-service.kruise-system.svc + - localhost issuerRef: name: selfsigned-kruise kind: Issuer @@ -145,9 +163,12 @@ metadata: name: selfsigned-kruise namespace: kruise-system spec: - selfSigned: {} + selfSigned: { } ``` -3. During installation and upgrade, enable external certs support by setting featureGates=EnableExternalCerts=true and specify extra annotations that should be added to webhookconfiguration and CRD. + +3. During installation and upgrade, enable external certs support by setting featureGates=EnableExternalCerts=true and + specify extra annotations that should be added to webhookconfiguration and CRD. + ``` helm install kruise https://... --set featureGates="EnableExternalCerts=true" --set-json externalCerts.annotations='{"cert-manager.io/inject-ca-from":"kruise-system/kruise-webhook-certs"}' ``` diff --git a/incubator/kruise/templates/apps.kruise.io_clonesets.yaml b/incubator/kruise/templates/apps.kruise.io_clonesets.yaml index 0946f8a4..75c40152 100644 --- a/incubator/kruise/templates/apps.kruise.io_clonesets.yaml +++ b/incubator/kruise/templates/apps.kruise.io_clonesets.yaml @@ -31,6 +31,10 @@ spec: jsonPath: .status.updatedReadyReplicas name: UPDATED_READY type: integer + - description: The number of pods updated and available. + jsonPath: .status.updatedAvailableReplicas + name: UPDATED_AVAILABLE + type: integer - description: The number of pods ready. jsonPath: .status.readyReplicas name: READY @@ -514,6 +518,8 @@ spec: description: |- UpdatedAvailableReplicas is the number of Pods created by the CloneSet controller from the CloneSet version indicated by updateRevision and have a Ready Condition for at least minReadySeconds. + Notice: when enable InPlaceWorkloadVerticalScaling, pod during resource resizing will also be unavailable. + This means these pod will be counted in maxUnavailable. format: int32 type: integer updatedReadyReplicas: diff --git a/incubator/kruise/templates/apps.kruise.io_sidecarsets.yaml b/incubator/kruise/templates/apps.kruise.io_sidecarsets.yaml index 2c82d027..0b34f9b4 100644 --- a/incubator/kruise/templates/apps.kruise.io_sidecarsets.yaml +++ b/incubator/kruise/templates/apps.kruise.io_sidecarsets.yaml @@ -260,9 +260,11 @@ spec: history SidecarSet to inject specific version of the sidecar to pods. type: string policy: - description: |- - Policy describes the behavior of revision injection. - Defaults to Always. + default: Always + description: Policy describes the behavior of revision injection. + enum: + - Always + - Partial type: string revisionName: description: RevisionName corresponds to a specific ControllerRevision @@ -416,8 +418,7 @@ spec: paused: description: |- Paused indicates that the SidecarSet is paused to update the injected pods, - but it don't affect the webhook inject sidecar container into the newly created pods. - default is false + For the impact on the injection behavior for newly created Pods, please refer to the comments of Selector. type: boolean priorityStrategy: description: |- @@ -529,8 +530,14 @@ spec: type: object type: array selector: - description: If selector is not nil, this upgrade will only update - the selected pods. + description: |- + If selector is not nil, this upgrade will only update the selected pods. + + + Starting from Kruise 1.8.0, the updateStrategy.Selector affects the version of the Sidecar container + injected into newly created Pods by a SidecarSet configured with an injectionStrategy. + In most cases, all newly created Pods are injected with the specified Sidecar version as configured in injectionStrategy.revision, + which is consistent with previous versions. properties: matchExpressions: description: matchExpressions is a list of label selector diff --git a/incubator/kruise/templates/apps.kruise.io_statefulsets.yaml b/incubator/kruise/templates/apps.kruise.io_statefulsets.yaml index 4331fd57..16f67af8 100644 --- a/incubator/kruise/templates/apps.kruise.io_statefulsets.yaml +++ b/incubator/kruise/templates/apps.kruise.io_statefulsets.yaml @@ -1,5 +1,5 @@ {{- if .Values.crds.managed }} - + --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -668,8 +668,12 @@ spec: Then controller will delete Pod-1 and create Pod-3 (existing Pods will be [0, 2, 3]) - If you just want to delete Pod-1, you should set spec.reserveOrdinal to [1] and spec.replicas to 2. Then controller will delete Pod-1 (existing Pods will be [0, 2]) + You can also use ranges along with numbers, such as [1, 3-5], which is a shortcut for [1, 3, 4, 5]. items: - type: integer + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true type: array revisionHistoryLimit: description: |- @@ -800,9 +804,7 @@ spec: type: integer partition: description: |- - Partition indicates the ordinal at which the StatefulSet should be partitioned by default. - But if unorderedUpdate has been set: - - Partition indicates the number of pods with non-updated revisions when rolling update. + Partition indicates the number of pods the StatefulSet should be partitioned by default. - It means controller will update $(replicas - partition) number of pod. Default value is 0. format: int32 @@ -937,6 +939,18 @@ spec: any volumes in the template, with the same name. TODO: Define the behavior if a claim already exists with the same name. x-kubernetes-preserve-unknown-fields: true + volumeClaimUpdateStrategy: + description: |- + VolumeClaimUpdateStrategy specifies the strategy for updating VolumeClaimTemplates within a StatefulSet. + This field is currently only effective if the StatefulSetAutoResizePVCGate is enabled. + properties: + type: + description: |- + Type specifies the type of update strategy, possible values include: + OnPodRollingUpdateVolumeClaimUpdateStrategyType: Apply the update strategy during pod rolling updates. + OnPVCDeleteVolumeClaimUpdateStrategyType: Apply the update strategy when a PersistentVolumeClaim is deleted. + type: string + type: object required: - selector - template @@ -1040,6 +1054,43 @@ spec: indicated by updateRevision. format: int32 type: integer + volumeClaims: + description: |- + VolumeClaims represents the status of compatibility between existing PVCs + and their respective templates. It tracks whether the PersistentVolumeClaims have been updated + to match any changes made to the volumeClaimTemplates, ensuring synchronization + between the defined templates and the actual PersistentVolumeClaims in use. + items: + description: |- + VolumeClaimStatus describes the status of a volume claim template. + It provides details about the compatibility and readiness of the volume claim. + properties: + compatibleReadyReplicas: + description: |- + CompatibleReadyReplicas is the number of replicas that are both ready and compatible with the volume claim. + It highlights that these replicas are not only compatible but also ready to be put into service immediately. + Compatibility is determined by whether the pvc spec storage requests are greater than or equal to the template spec storage requests + The "ready" status is determined by whether the PVC status capacity is greater than or equal to the PVC spec storage requests. + format: int32 + type: integer + compatibleReplicas: + description: |- + CompatibleReplicas is the number of replicas currently compatible with the volume claim. + It indicates how many replicas can function properly, being compatible with this volume claim. + Compatibility is determined by whether the PVC spec storage requests are greater than or equal to the template spec storage requests + format: int32 + type: integer + volumeClaimName: + description: |- + VolumeClaimName is the name of the volume claim. + This is a unique identifier used to reference a specific volume claim. + type: string + required: + - compatibleReadyReplicas + - compatibleReplicas + - volumeClaimName + type: object + type: array required: - availableReplicas - currentReplicas diff --git a/incubator/kruise/templates/apps.kruise.io_uniteddeployments.yaml b/incubator/kruise/templates/apps.kruise.io_uniteddeployments.yaml index 9f4f6b10..376d267c 100644 --- a/incubator/kruise/templates/apps.kruise.io_uniteddeployments.yaml +++ b/incubator/kruise/templates/apps.kruise.io_uniteddeployments.yaml @@ -277,8 +277,12 @@ spec: Then controller will delete Pod-1 and create Pod-3 (existing Pods will be [0, 2, 3]) - If you just want to delete Pod-1, you should set spec.reserveOrdinal to [1] and spec.replicas to 2. Then controller will delete Pod-1 (existing Pods will be [0, 2]) + You can also use ranges along with numbers, such as [1, 3-5], which is a shortcut for [1, 3, 4, 5]. items: - type: integer + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true type: array revisionHistoryLimit: description: |- @@ -409,9 +413,7 @@ spec: type: integer partition: description: |- - Partition indicates the ordinal at which the StatefulSet should be partitioned by default. - But if unorderedUpdate has been set: - - Partition indicates the number of pods with non-updated revisions when rolling update. + Partition indicates the number of pods the StatefulSet should be partitioned by default. - It means controller will update $(replicas - partition) number of pod. Default value is 0. format: int32 @@ -549,6 +551,18 @@ spec: any volumes in the template, with the same name. TODO: Define the behavior if a claim already exists with the same name. x-kubernetes-preserve-unknown-fields: true + volumeClaimUpdateStrategy: + description: |- + VolumeClaimUpdateStrategy specifies the strategy for updating VolumeClaimTemplates within a StatefulSet. + This field is currently only effective if the StatefulSetAutoResizePVCGate is enabled. + properties: + type: + description: |- + Type specifies the type of update strategy, possible values include: + OnPodRollingUpdateVolumeClaimUpdateStrategyType: Apply the update strategy during pod rolling updates. + OnPVCDeleteVolumeClaimUpdateStrategyType: Apply the update strategy when a PersistentVolumeClaim is deleted. + type: string + type: object required: - selector - template @@ -945,6 +959,38 @@ spec: description: Topology describes the pods distribution detail between each of subsets. properties: + scheduleStrategy: + description: ScheduleStrategy indicates the strategy the UnitedDeployment + used to preform the schedule between each of subsets. + properties: + adaptive: + description: Adaptive is used to communicate parameters when + Type is AdaptiveUnitedDeploymentScheduleStrategyType. + properties: + rescheduleCriticalSeconds: + description: |- + RescheduleCriticalSeconds indicates how long controller will reschedule a schedule failed Pod to the subset that has + redundant capacity after the subset where the Pod lives. If a Pod was scheduled failed and still in an unschedulabe status + over RescheduleCriticalSeconds duration, the controller will reschedule it to a suitable subset. Default is 30 seconds. + format: int32 + type: integer + unschedulableLastSeconds: + description: |- + UnschedulableLastSeconds is used to set the number of seconds for a Subset to recover from an unschedulable state, + with a default value of 300 seconds. + format: int32 + type: integer + type: object + type: + description: |- + Type indicates the type of the UnitedDeploymentScheduleStrategy. + Default is Fixed + enum: + - Adaptive + - Fixed + - "" + type: string + type: object subsets: description: |- Contains the details of each subset. Each element in this array represents one subset @@ -1163,7 +1209,7 @@ spec: format: date-time type: string message: - description: A human readable message indicating details about + description: A human-readable message indicating details about the transition. type: string reason: @@ -1206,6 +1252,44 @@ spec: description: Records the topology detail information of the replicas of each subset. type: object + subsetStatuses: + description: Record the conditions of each subset. + items: + properties: + conditions: + description: Conditions is an array of current observed subset + conditions. + items: + properties: + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + type: string + status: + type: string + type: + type: string + required: + - status + - type + type: object + type: array + name: + description: Subset name specified in Topology.Subsets + type: string + partition: + description: Records the current partition. Currently unused. + format: int32 + type: integer + replicas: + description: Recores the current replicas. Currently unused. + format: int32 + type: integer + type: object + type: array updateStatus: description: Records the information of update progress. properties: diff --git a/incubator/kruise/templates/apps.kruise.io_workloadspreads.yaml b/incubator/kruise/templates/apps.kruise.io_workloadspreads.yaml index 7ca7e75b..68f4c3cb 100644 --- a/incubator/kruise/templates/apps.kruise.io_workloadspreads.yaml +++ b/incubator/kruise/templates/apps.kruise.io_workloadspreads.yaml @@ -312,6 +312,69 @@ spec: - name type: object type: array + targetFilter: + description: |- + TargetFilter allows WorkloadSpread to manage only a portion of the Pods in the TargetReference: + by specifying the criteria for the Pods to be managed through a label selector, + and by specifying how to obtain the total number of these selected Pods from the workload using replicasPaths. + properties: + replicasPathList: + description: |- + ReplicasPathList is a list of resource paths used to specify how to determine the total number of replicas of + the target workload after filtering. If this list is not empty, WorkloadSpread will look for the corresponding + values in the target resource according to each path, and treat the sum of these values as the total number of replicas after filtering. + + + The replicas path is a dot-separated path, similar to "spec.replicas". If there are arrays, you can use numbers to denote indexes, like "subsets.1.replicas". + The real values of these paths must be integers. + items: + type: string + type: array + selector: + description: Selector is used to filter the Pods to be managed. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: object targetRef: description: TargetReference is the target workload that WorkloadSpread want to control. diff --git a/incubator/kruise/templates/manager.yaml b/incubator/kruise/templates/manager.yaml index 320e80b1..e9426741 100644 --- a/incubator/kruise/templates/manager.yaml +++ b/incubator/kruise/templates/manager.yaml @@ -128,6 +128,17 @@ spec: nodeAffinity: {{ toYaml . | indent 10 }} {{- end }} + topologySpreadConstraints: + - labelSelector: + matchLabels: + control-plane: controller-manager +{{- if and ( eq (int .Capabilities.KubeVersion.Major) 1) ( gt (int .Capabilities.KubeVersion.Minor) 26 ) }} + matchLabelKeys: + - pod-template-hash +{{- end }} + maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway {{- if .Values.manager.nodeSelector }} nodeSelector: diff --git a/incubator/kruise/templates/pre_delete_hook_job.yaml b/incubator/kruise/templates/pre_delete_hook_job.yaml new file mode 100644 index 00000000..204606d4 --- /dev/null +++ b/incubator/kruise/templates/pre_delete_hook_job.yaml @@ -0,0 +1,92 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: kruise-helm-hook-role + annotations: + "helm.sh/hook": pre-delete + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + "helm.sh/hook-weight": "1" +rules: + - apiGroups: + - apps.kruise.io + resources: + - clonesets + verbs: + - list + - apiGroups: + - apps.kruise.io + resources: + - daemonsets + verbs: + - list + - apiGroups: + - apps.kruise.io + resources: + - statefulsets + verbs: + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kruise-helm-hook-rolebinding + annotations: + "helm.sh/hook": pre-delete + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + "helm.sh/hook-weight": "2" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kruise-helm-hook-role +subjects: + - kind: ServiceAccount + name: kruise-helm-hook + namespace: {{ .Values.installation.namespace }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: "{{ .Release.Name }}-finalizer" + namespace: {{ .Values.installation.namespace }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + kruise: helm-finalizer + annotations: + # This is what defines this resource as a hook. Without this line, the + # job is considered part of the release. + "helm.sh/hook": pre-delete + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + "helm.sh/hook-weight": "4" +spec: + backoffLimit: 0 + template: + metadata: + name: "{{ .Release.Name }}-finalizer" + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + kruise: helm-finalizer + spec: + restartPolicy: Never + serviceAccountName: kruise-helm-hook + containers: + - name: pre-delete-job + image: {{ .Values.helmHooks.image.repository }}:{{ .Values.helmHooks.image.tag }} + imagePullPolicy: IfNotPresent +--- +# write a service account named kruise-helm-hook: +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kruise-helm-hook + namespace: {{ .Values.installation.namespace }} + annotations: + "helm.sh/hook": pre-delete + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + "helm.sh/hook-weight": "3" diff --git a/incubator/kruise/templates/rbac_role.yaml b/incubator/kruise/templates/rbac_role.yaml index 57f004fb..695de735 100644 --- a/incubator/kruise/templates/rbac_role.yaml +++ b/incubator/kruise/templates/rbac_role.yaml @@ -790,6 +790,16 @@ rules: - get - patch - update +{{- if (contains "StatefulSetAutoResizePVCGate=true" .Values.featureGates) }} +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch +{{- end }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding @@ -941,3 +951,114 @@ subjects: - kind: ServiceAccount name: kruise-daemon namespace: {{ .Values.installation.namespace }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: aggregate-kruise-view + labels: + # Add these permissions to the "view" default role. + rbac.authorization.k8s.io/aggregate-to-view: "true" +rules: +- apiGroups: ["apps.kruise.io"] + resources: + - advancedcronjobs + - advancedcronjobs/status + - broadcastjobs + - broadcastjobs/status + - clonesets + - clonesets/scale + - clonesets/status + - containerrecreaterequests + - containerrecreaterequests/status + - daemonsets + - daemonsets/status + - imagelistpulljobs + - imagelistpulljobs/status + - imagepulljobs + - imagepulljobs/status + - nodeimages + - nodeimages/status + - nodepodprobes + - nodepodprobes/status + - persistentpodstates + - persistentpodstates/status + - podprobemarkers + - podprobemarkers/status + - sidecarsets + - sidecarsets/status + - statefulsets + - statefulsets/scale + - statefulsets/status + - uniteddeployments + - uniteddeployments/scale + - uniteddeployments/status + - workloadspreads + - workloadspreads/status + verbs: + - get + - list + - watch +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: aggregate-kruise-edit + labels: + # Add these permissions to the "admin" and "edit" default roles. + rbac.authorization.k8s.io/aggregate-to-edit: "true" +rules: +- apiGroups: ["apps.kruise.io"] + resources: + - advancedcronjobs + - broadcastjobs + - clonesets + - clonesets/scale + - containerrecreaterequests + - daemonsets + - imagelistpulljobs + - imagepulljobs + - nodeimages + - nodepodprobes + - persistentpodstates + - podprobemarkers + - sidecarsets + - statefulsets + - statefulsets/scale + - uniteddeployments + - uniteddeployments/scale + - workloadspreads + verbs: + - create + - delete + - deletecollection + - patch + - update +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: aggregate-kruise-admin + labels: + # Add these permissions to the "admin" default roles. + rbac.authorization.k8s.io/aggregate-to-admin: "true" +rules: +- apiGroups: ["apps.kruise.io"] + resources: + - resourcedistributions/status + verbs: + - get + - list + - watch +- apiGroups: ["apps.kruise.io"] + resources: + - resourcedistributions + verbs: + - get + - list + - watch + - create + - delete + - deletecollection + - patch + - update \ No newline at end of file diff --git a/incubator/kruise/values.yaml b/incubator/kruise/values.yaml index 8bae9ddb..25ddcbf3 100644 --- a/incubator/kruise/values.yaml +++ b/incubator/kruise/values.yaml @@ -32,7 +32,7 @@ manager: replicas: 2 image: repository: ccr.ccs.tencentyun.com/tke-market/kruise-manager - tag: v1.7.2 + tag: v1.8.3 webhook: port: 9876 metrics: @@ -113,3 +113,8 @@ daemon: serviceAccount: annotations: {} + +helmHooks: + image: + repository: ccr.ccs.tencentyun.com/tke-market/kruise-helm-hook + tag: v0.1.0