[v0.16] - Deploy charts that define CRDs and use lookup (#5689)

integrationtests/agent/assets/crd-chart/Chart.yaml

@@ -0,0 +1,6 @@
+apiVersion: v2
+name: crd-chart
+description: A test chart defining a CRD, a custom resource and a template using lookup
+type: application
+version: 0.1.0
+appVersion: "1.16.0"

integrationtests/agent/assets/crd-chart/crds/appconfig.yaml

@@ -0,0 +1,25 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: appconfigs.stable.example.com
+spec:
+  group: stable.example.com
+  names:
+    kind: AppConfig
+    listKind: AppConfigList
+    plural: appconfigs
+    singular: appconfig
+  scope: Namespaced
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            spec:
+              type: object
+              properties:
+                value:
+                  type: string

integrationtests/agent/assets/crd-chart/templates/appconfig.yaml

@@ -0,0 +1,6 @@
+apiVersion: stable.example.com/v1
+kind: AppConfig
+metadata:
+  name: frontend-config
+spec:
+  value: {{ .Values.value | quote }}

integrationtests/agent/assets/crd-chart/templates/configmap.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: lookup-chart-config
+data:
+  namespace: {{ (lookup "v1" "Namespace" "" .Release.Namespace).metadata.name }}

integrationtests/agent/assets/missing-crd-chart/Chart.yaml

@@ -0,0 +1,6 @@
+apiVersion: v2
+name: missing-crd-chart
+description: A test chart using lookup and a custom resource whose CRD is not part of the chart
+type: application
+version: 0.1.0
+appVersion: "1.16.0"

integrationtests/agent/assets/missing-crd-chart/templates/configmap.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: missing-crd-chart-config
+data:
+  namespace: {{ (lookup "v1" "Namespace" "" .Release.Namespace).metadata.name }}

integrationtests/agent/assets/missing-crd-chart/templates/unknownconfig.yaml

@@ -0,0 +1,6 @@
+apiVersion: stable.example.com/v1
+kind: UnknownConfig
+metadata:
+  name: unknown-config
+spec:
+  value: {{ .Values.value | quote }}

integrationtests/agent/bundle_deployment_crds_test.go

@@ -0,0 +1,105 @@
+package agent_test
+
+import (
+	"context"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/rancher/fleet/pkg/apis/fleet.cattle.io/v1alpha1"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
+)
+
+var _ = Describe("Helm chart defining CRDs and using lookup", Ordered, func() {
+	var (
+		env  *specEnv
+		name string
+	)
+
+	createBundleDeployment := func(name, id, chart string) {
+		bundled := v1alpha1.BundleDeployment{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      name,
+				Namespace: clusterNS,
+			},
+			Spec: v1alpha1.BundleDeploymentSpec{
+				DeploymentID: id,
+				Options: v1alpha1.BundleDeploymentOptions{
+					DefaultNamespace: env.namespace,
+					Helm: &v1alpha1.HelmOptions{
+						Chart: chart,
+						Values: &v1alpha1.GenericMap{
+							Data: map[string]any{"value": "example-value"},
+						},
+					},
+				},
+			},
+		}
+
+		Expect(k8sClient.Create(ctx, &bundled)).ToNot(HaveOccurred())
+
+		DeferCleanup(func() {
+			Expect(k8sClient.Delete(context.TODO(), &v1alpha1.BundleDeployment{
+				ObjectMeta: metav1.ObjectMeta{Namespace: clusterNS, Name: name},
+			})).ToNot(HaveOccurred())
+		})
+	}
+
+	BeforeAll(func() {
+		env = &specEnv{namespace: createNamespace()}
+	})
+
+	When("the chart provides the CRDs for its custom resources", func() {
+		BeforeAll(func() {
+			name = "crds-and-lookup"
+			createBundleDeployment(name, "crdsAndLookup", "crd-chart")
+		})
+
+		It("deploys the chart, although its CRDs cannot be installed during the dry run", func() {
+			By("Making the BundleDeployment ready")
+			Eventually(env.isBundleDeploymentReadyAndNotModified).WithArguments(name).Should(BeTrue())
+
+			By("Deploying the custom resource defined by the chart's CRD")
+			appConfig := &unstructured.Unstructured{}
+			appConfig.SetGroupVersionKind(schema.GroupVersionKind{
+				Group:   "stable.example.com",
+				Version: "v1",
+				Kind:    "AppConfig",
+			})
+			err := k8sClient.Get(
+				ctx,
+				types.NamespacedName{Namespace: env.namespace, Name: "frontend-config"},
+				appConfig,
+			)
+			Expect(err).ToNot(HaveOccurred())
+
+			By("Resolving the lookup function against the cluster")
+			cm, err := env.getConfigMap("lookup-chart-config")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(cm.Data["namespace"]).To(Equal(env.namespace))
+		})
+	})
+
+	When("the chart does not provide the CRDs for its custom resources", func() {
+		BeforeAll(func() {
+			name = "missing-crd-and-lookup"
+			createBundleDeployment(name, "missingCRDAndLookup", "missing-crd-chart")
+		})
+
+		It("reports the missing CRD in the BundleDeployment status", func() {
+			Eventually(func(g Gomega) {
+				bd := &v1alpha1.BundleDeployment{}
+				err := k8sClient.Get(ctx, types.NamespacedName{Namespace: clusterNS, Name: name}, bd)
+				g.Expect(err).ToNot(HaveOccurred())
+
+				checkCondition(g, bd.Status.Conditions, "Deployed", "False", `no matches for kind "UnknownConfig"`)
+			}).Should(Succeed())
+
+			Expect(env.isBundleDeploymentReadyAndNotModified(name)).To(BeFalse())
+		})
+	})
+})

integrationtests/agent/suite_test.go

@@ -3,7 +3,9 @@ package agent_test
 import (
 	"context"
 	"fmt"
+	"io/fs"
 	"os"
+	"path/filepath"
 	"sync"
 	"testing"
 	"time"
@@ -448,6 +450,12 @@ data:
 				Name:    "fleet.yaml",
 			},
 		},
+		// A chart which defines a CRD in crds/, deploys a custom resource of
+		// that kind and uses the lookup function in one of its templates.
+		"crdsAndLookup": chartResources("crd-chart"),
+		// The same, but the CRD for the deployed custom resource is missing
+		// from the chart.
+		"missingCRDAndLookup": chartResources("missing-crd-chart"),
 		"capabilitiesv2": []v1alpha1.BundleResource{
 			{
 				Content: "apiVersion: v2\nname: config-chart\ndescription: A test chart that verifies its config\ntype: application\nversion: 0.1.0\nappVersion: \"1.16.0\"\nkubeVersion: '>= 920.920.0-0'\n",
@@ -464,3 +472,36 @@ data:
 		},
 	}
 }
+
+// chartResources reads a chart from the assets directory and turns it into
+// bundle resources, named after their path relative to the assets directory.
+func chartResources(chartDir string) []v1alpha1.BundleResource {
+	var resources []v1alpha1.BundleResource
+
+	root := filepath.Join(assetsPath, chartDir)
+	err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error {
+		if err != nil || entry.IsDir() {
+			return err
+		}
+
+		content, err := os.ReadFile(path)
+		if err != nil {
+			return err
+		}
+
+		name, err := filepath.Rel(assetsPath, path)
+		if err != nil {
+			return err
+		}
+
+		resources = append(resources, v1alpha1.BundleResource{
+			Name:    name,
+			Content: string(content),
+		})
+
+		return nil
+	})
+	Expect(err).ToNot(HaveOccurred())
+
+	return resources
+}

internal/helmdeployer/crds.go

@@ -0,0 +1,94 @@
+package helmdeployer
+
+import (
+	"bytes"
+	"errors"
+
+	chartv2 "helm.sh/helm/v4/pkg/chart/v2"
+
+	apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/yaml"
+)
+
+// isMissingOwnCRDsError reports whether err consists solely of REST mapping
+// failures for kinds which the chart itself declares in its crds/ directory.
+//
+// A server-side dry run cannot install those CRDs, hence custom resources
+// defined by the chart have no REST mapping yet and Helm fails to build the
+// Kubernetes objects for the rendered manifest. The actual install installs
+// the contents of crds/ first, so it does not suffer from this limitation.
+func isMissingOwnCRDsError(err error, chart *chartv2.Chart) bool {
+	leaves := leafErrors(err)
+	if len(leaves) == 0 {
+		return false
+	}
+
+	declared := chartCRDGroupKinds(chart)
+	for _, leaf := range leaves {
+		noKindMatch, ok := errors.AsType[*meta.NoKindMatchError](leaf)
+		if !ok || !declared.Has(noKindMatch.GroupKind) {
+			return false
+		}
+	}
+
+	return true
+}
+
+// leafErrors returns the leaf errors contained in err, expanding aggregates
+// recursively. Errors reported while building a resource list are wrapped into
+// an aggregate when more than one resource fails, and aggregates cannot be
+// traversed by errors.AsType, hence the explicit recursion.
+func leafErrors(err error) []error {
+	if err == nil {
+		return nil
+	}
+
+	aggregate, ok := errors.AsType[utilerrors.Aggregate](err)
+	if !ok {
+		return []error{err}
+	}
+
+	var result []error
+	for _, e := range aggregate.Errors() {
+		result = append(result, leafErrors(e)...)
+	}
+
+	return result
+}
+
+// chartCRDGroupKinds returns the group kinds defined by the CRDs found in the
+// crds/ directory of the chart and of its dependencies.
+func chartCRDGroupKinds(chart *chartv2.Chart) sets.Set[schema.GroupKind] {
+	groupKinds := sets.New[schema.GroupKind]()
+	if chart == nil {
+		return groupKinds
+	}
+
+	for _, crd := range chart.CRDObjects() {
+		if crd.File == nil {
+			continue
+		}
+
+		// CRD files may contain multiple documents.
+		decoder := yaml.NewYAMLToJSONDecoder(bytes.NewReader(crd.File.Data))
+		for {
+			var def apiextv1.CustomResourceDefinition
+			if err := decoder.Decode(&def); err != nil {
+				// Stop at the first unreadable document: an incomplete list of
+				// group kinds can only make the caller stricter, and Helm
+				// reports invalid CRDs when it installs them for real.
+				break
+			}
+			if def.Spec.Group == "" || def.Spec.Names.Kind == "" {
+				continue
+			}
+			groupKinds.Insert(schema.GroupKind{Group: def.Spec.Group, Kind: def.Spec.Names.Kind})
+		}
+	}
+
+	return groupKinds
+}