diff --git a/integrationtests/agentmanagement/clusterregistrationtoken_test.go b/integrationtests/agentmanagement/clusterregistrationtoken_test.go new file mode 100644 index 0000000000..3f8eac09b5 --- /dev/null +++ b/integrationtests/agentmanagement/clusterregistrationtoken_test.go @@ -0,0 +1,292 @@ +package agentmanagement_test + +import ( + "strconv" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/rancher/fleet/integrationtests/utils" + fleetns "github.com/rancher/fleet/internal/cmd/controller/namespace" + "github.com/rancher/fleet/internal/config" + "github.com/rancher/fleet/internal/names" + fleet "github.com/rancher/fleet/pkg/apis/fleet.cattle.io/v1alpha1" + + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + yaml "sigs.k8s.io/yaml" +) + +var _ = Describe("ClusterRegistrationToken", func() { + var regNamespace string + + systemRegistrationNamespace := fleetns.SystemRegistrationNamespace(systemNamespace) + + BeforeEach(func() { + ns := newGeneratedNamespace("cluster-registration-token-test-") + Expect(k8sClient.Create(ctx, ns)).To(Succeed()) + regNamespace = ns.Name + }) + + // waitForServiceAccount waits for the ServiceAccount the handler creates + // for token and returns its name. + waitForServiceAccount := func(token *fleet.ClusterRegistrationToken) string { + GinkgoHelper() + + saName := names.SafeConcatName(token.Name, string(token.UID)) + Eventually(func(g Gomega) { + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: token.Namespace, Name: saName}, &corev1.ServiceAccount{})).To(Succeed()) + }).Should(Succeed()) + return saName + } + + // unblockServiceAccountTokenSecret waits for the "-token" Secret the + // handler creates for saName and fills in fake token data, standing in + // for the token controller that would populate it in a real cluster + // (envtest runs no controller-manager to do this on its own). Without + // this, the handler blocks indefinitely waiting for the token to appear. + unblockServiceAccountTokenSecret := func(namespace, saName, tokenValue string) { + GinkgoHelper() + + key := types.NamespacedName{Namespace: namespace, Name: saName + "-token"} + // Get and update must share the retry, or a resourceVersion conflict + // between them fails the spec instead of being retried. Only the token + // key is set, so the rest of a service account token Secret's data + // (ca.crt, namespace) survives if anything ever populates it. + Eventually(func(g Gomega) { + secret := &corev1.Secret{} + g.Expect(k8sClient.Get(ctx, key, secret)).To(Succeed()) + if secret.Data == nil { + secret.Data = map[string][]byte{} + } + secret.Data[corev1.ServiceAccountTokenKey] = []byte(tokenValue) + g.Expect(k8sClient.Update(ctx, secret)).To(Succeed()) + }).Should(Succeed()) + } + + // createTokenWithPopulatedSecret creates a token with the given TTL, + // unblocks the ServiceAccount token Secret the handler waits on (see + // unblockServiceAccountTokenSecret), and drives the token's own + // reconciliation until the resulting cluster-registration-values Secret + // reflects that token data. Returns the created token and the + // ServiceAccount name. + createTokenWithPopulatedSecret := func(name string, ttl *metav1.Duration, tokenValue string) (*fleet.ClusterRegistrationToken, string) { + GinkgoHelper() + + token := newClusterRegistrationToken(regNamespace, name, ttl) + Expect(k8sClient.Create(ctx, token)).To(Succeed()) + + saName := waitForServiceAccount(token) + unblockServiceAccountTokenSecret(regNamespace, saName, tokenValue) + + // The handler only re-reads the SA token Secret through its cache the + // instant it first observes a populated live Secret, and nothing + // re-triggers it on later Secret changes. Nudge the token itself + // (which is watched directly) until the derived values Secret + // reflects the token data, so this does not depend on informer cache + // timing relative to the update above. + valuesSecretKey := types.NamespacedName{Namespace: regNamespace, Name: token.Name} + Eventually(func(g Gomega) { + var current fleet.ClusterRegistrationToken + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: regNamespace, Name: token.Name}, ¤t)).To(Succeed()) + if current.Annotations == nil { + current.Annotations = map[string]string{} + } + current.Annotations["test.fleet.cattle.io/nudge"] = strconv.FormatInt(time.Now().UnixNano(), 10) + g.Expect(k8sClient.Update(ctx, ¤t)).To(Succeed()) + + var values corev1.Secret + g.Expect(k8sClient.Get(ctx, valuesSecretKey, &values)).To(Succeed()) + + var parsed map[string]any + g.Expect(yaml.Unmarshal(values.Data[config.ImportTokenSecretValuesKey], &parsed)).To(Succeed()) + g.Expect(parsed["token"]).To(Equal(tokenValue)) + // Keep the suite's timeout: this is the slowest wait in the file, + // gated on a 2s durations.ServiceTokenSleep boundary plus an + // informer cache sync. Only the interval is tightened from the + // suite's 3s default, and not below 500ms, because every iteration + // issues a token Update. + }, utils.Timeout, 500*time.Millisecond).Should(Succeed()) + + return token, saName + } + + Describe("tokens without a usable TTL are rejected when enforceTTL is enabled", func() { + It("deletes a token with no TTL and creates no resources for it", func() { + token := newClusterRegistrationToken(regNamespace, "no-ttl", nil) + Expect(k8sClient.Create(ctx, token)).To(Succeed()) + + objectGone(token).Should(Succeed()) + + saName := names.SafeConcatName(token.Name, string(token.UID)) + Consistently(func(g Gomega) { + err := k8sClient.Get(ctx, types.NamespacedName{Namespace: regNamespace, Name: saName}, &corev1.ServiceAccount{}) + g.Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }).Should(Succeed()) + }) + + It("deletes a token whose TTL duration is zero", func() { + token := newClusterRegistrationToken(regNamespace, "zero-ttl", &metav1.Duration{Duration: 0}) + Expect(k8sClient.Create(ctx, token)).To(Succeed()) + + objectGone(token).Should(Succeed()) + }) + }) + + Describe("a valid token", func() { + It("creates a ServiceAccount, a Role scoped to creating clusterregistrations, and a RoleBinding, all in the token's own namespace", func() { + _, saName := createTokenWithPopulatedSecret("valid-token", &metav1.Duration{Duration: 24 * time.Hour}, "fake-sa-token") + + var sa corev1.ServiceAccount + Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: regNamespace, Name: saName}, &sa)).To(Succeed()) + Expect(sa.Labels).To(HaveKeyWithValue(fleet.ManagedLabel, "true")) + + roleName := names.SafeConcatName(saName, "role") + var role rbacv1.Role + Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: regNamespace, Name: roleName}, &role)).To(Succeed()) + Expect(role.Rules).To(ConsistOf(rbacv1.PolicyRule{ + Verbs: []string{"create"}, + APIGroups: []string{fleet.SchemeGroupVersion.Group}, + Resources: []string{fleet.ClusterRegistrationResourceNamePlural}, + }), "the token handler must not grant access to secrets; that is the clusterregistration controller's responsibility") + + roleBindingName := names.SafeConcatName(saName, "to", "role") + var rb rbacv1.RoleBinding + Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: regNamespace, Name: roleBindingName}, &rb)).To(Succeed()) + Expect(rb.RoleRef).To(Equal(rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Kind: "Role", + Name: roleName, + })) + Expect(rb.Subjects).To(ConsistOf(rbacv1.Subject{ + Kind: "ServiceAccount", + Name: saName, + Namespace: regNamespace, + })) + + Expect(regNamespace).NotTo(Equal(systemRegistrationNamespace), + "sanity check: the generated namespace must not collide with the system registration namespace") + Expect(sa.Namespace).NotTo(Equal(systemRegistrationNamespace)) + Expect(role.Namespace).NotTo(Equal(systemRegistrationNamespace)) + Expect(rb.Namespace).NotTo(Equal(systemRegistrationNamespace)) + }) + + It("populates the cluster-registration-values Secret and status with the service account token", func() { + token, _ := createTokenWithPopulatedSecret("valid-token-values", &metav1.Duration{Duration: 24 * time.Hour}, "expected-token-value") + + var values corev1.Secret + Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: regNamespace, Name: token.Name}, &values)).To(Succeed()) + Expect(values.Labels).To(HaveKeyWithValue(fleet.ManagedLabel, "true")) + Expect(values.Type).To(BeEquivalentTo("fleet.cattle.io/cluster-registration-values")) + + var parsed map[string]any + Expect(yaml.Unmarshal(values.Data[config.ImportTokenSecretValuesKey], &parsed)).To(Succeed()) + Expect(parsed["clusterNamespace"]).To(Equal(regNamespace)) + Expect(parsed["systemRegistrationNamespace"]).To(Equal(systemRegistrationNamespace)) + Expect(parsed["tokenName"]).To(Equal(token.Name)) + Expect(parsed["token"]).To(Equal("expected-token-value")) + Expect(parsed["apiServerURL"]).To(Equal(config.Get().APIServerURL)) + Expect(parsed["apiServerCA"]).To(Equal(string(config.Get().APIServerCA))) + // systemNamespace equals config.DefaultNamespace in this suite, so the + // "internal" override block is never populated; see PROGRESS.md. + Expect(parsed).NotTo(HaveKey("internal")) + + Eventually(func(g Gomega) { + var current fleet.ClusterRegistrationToken + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: regNamespace, Name: token.Name}, ¤t)).To(Succeed()) + g.Expect(current.Status.SecretName).To(Equal(token.Name)) + g.Expect(current.Status.Expires).NotTo(BeNil()) + g.Expect(current.Status.Expires.Time).To(BeTemporally("~", current.CreationTimestamp.Add(24*time.Hour), 10*time.Second)) + }).Should(Succeed()) + }) + }) + + Describe("expiration", func() { + It("deletes a token once its TTL has elapsed", func() { + // Settle the token first, then shorten its TTL to one that has + // certainly already elapsed (expiry is measured from + // CreationTimestamp, and setup takes seconds). Creating the token + // with a short TTL up front instead races the controller against + // itself: whether the reconcile that creates the ServiceAccount + // beats the one EnqueueAfter(TTL) schedules decides the outcome, + // and if expiry wins, apply prunes the ServiceAccount and the + // helpers waiting on it spin until they time out. + token, _ := createTokenWithPopulatedSecret("expired-ttl", &metav1.Duration{Duration: time.Hour}, "expired-ttl-value") + + Eventually(func(g Gomega) { + var current fleet.ClusterRegistrationToken + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: regNamespace, Name: token.Name}, ¤t)).To(Succeed()) + current.Spec.TTL = &metav1.Duration{Duration: time.Nanosecond} + g.Expect(k8sClient.Update(ctx, ¤t)).To(Succeed()) + }).Should(Succeed()) + + objectGone(token).Should(Succeed()) + }) + + It("keeps a token alive until its TTL elapses, then deletes it", func() { + // Settle the token under a long TTL first: a short one at + // creation races setup, which takes seconds (see the spec + // above), and leaves the handler parked in its unbounded wait + // for the SA token Secret. + token, _ := createTokenWithPopulatedSecret("short-ttl", &metav1.Duration{Duration: time.Hour}, "short-ttl-value") + + // Expiry is measured from CreationTimestamp, so aim the TTL at a + // wall clock deadline. margin keeps the survival check clear of + // that deadline, absorbing CI jitter. + const ttlWindow = 15 * time.Second + const margin = 5 * time.Second + + key := types.NamespacedName{Namespace: regNamespace, Name: token.Name} + deadline := time.Now().Add(ttlWindow) + Eventually(func(g Gomega) { + var current fleet.ClusterRegistrationToken + g.Expect(k8sClient.Get(ctx, key, ¤t)).To(Succeed()) + current.Spec.TTL = &metav1.Duration{Duration: deadline.Sub(current.CreationTimestamp.Time)} + g.Expect(k8sClient.Update(ctx, ¤t)).To(Succeed()) + }).Should(Succeed()) + + // The token has to survive right up to its expiry... + survives := time.Until(deadline) - margin + Expect(survives).To(BeNumerically(">", 0), + "setting the TTL took longer than ttlWindow, raise ttlWindow") + Consistently(func(g Gomega) { + g.Expect(k8sClient.Get(ctx, key, &fleet.ClusterRegistrationToken{})).To(Succeed()) + }, survives, 500*time.Millisecond).Should(Succeed()) + + // ...and be gone once it passes. Deletion here comes from the + // handler's EnqueueAfter, which no other spec covers. + objectGone(token).Should(Succeed()) + }) + }) + + Describe("deleting a token", func() { + It("removes the ServiceAccount, Role, RoleBinding, and values Secret it created", func() { + token, saName := createTokenWithPopulatedSecret("pruned-token", &metav1.Duration{Duration: 24 * time.Hour}, "pruned-token-value") + + sa := &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Namespace: regNamespace, Name: saName}} + role := &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Namespace: regNamespace, Name: names.SafeConcatName(saName, "role")}} + roleBinding := &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Namespace: regNamespace, Name: names.SafeConcatName(saName, "to", "role")}} + values := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: regNamespace, Name: token.Name}} + + // Confirm the full set exists before deletion, so their absence + // afterwards is actually caused by pruning and not by them never + // having been created. + objectExists(sa).Should(Succeed()) + objectExists(role).Should(Succeed()) + objectExists(roleBinding).Should(Succeed()) + objectExists(values).Should(Succeed()) + + Expect(k8sClient.Delete(ctx, token)).To(Succeed()) + objectGone(token).Should(Succeed()) + + objectGone(sa).Should(Succeed()) + objectGone(role).Should(Succeed()) + objectGone(roleBinding).Should(Succeed()) + objectGone(values).Should(Succeed()) + }) + }) +}) diff --git a/integrationtests/agentmanagement/helpers_test.go b/integrationtests/agentmanagement/helpers_test.go index 8810bcf216..008aa483fc 100644 --- a/integrationtests/agentmanagement/helpers_test.go +++ b/integrationtests/agentmanagement/helpers_test.go @@ -48,8 +48,15 @@ func namespaceExists(name string) AsyncAssertion { // namespaceIsGone returns an AsyncAssertion that succeeds when no namespace with // the given name exists. func namespaceIsGone(name string) AsyncAssertion { + return objectGone(newNamespace(name)) +} + +// objectGone returns an AsyncAssertion that succeeds once obj can no longer be +// fetched from the API server. +func objectGone(obj client.Object) AsyncAssertion { + key := types.NamespacedName{Name: obj.GetName(), Namespace: obj.GetNamespace()} return Eventually(func(g Gomega) { - err := k8sClient.Get(ctx, types.NamespacedName{Name: name}, &corev1.Namespace{}) + err := k8sClient.Get(ctx, key, obj) g.Expect(apierrors.IsNotFound(err)).To(BeTrue()) }) } @@ -115,3 +122,12 @@ func newBundleDeployment(namespace, name string) *fleet.BundleDeployment { ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name}, } } + +// newClusterRegistrationToken returns a minimal ClusterRegistrationToken with +// the given TTL (nil for no TTL). +func newClusterRegistrationToken(namespace, name string, ttl *metav1.Duration) *fleet.ClusterRegistrationToken { + return &fleet.ClusterRegistrationToken{ + ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name}, + Spec: fleet.ClusterRegistrationTokenSpec{TTL: ttl}, + } +}