diff --git a/Makefile b/Makefile index ca5229fd0..e30a4f474 100644 --- a/Makefile +++ b/Makefile @@ -118,7 +118,7 @@ all: build check: verify lint test test-e2e test-e2e-contribs .PHONY: check -GOMODS := $(shell find . -name 'go.mod' -exec dirname {} \; | grep -v hack/tools) +GOMODS := $(shell find . -name 'go.mod' -exec dirname {} \; | grep -v hack/tools | grep -v ./dex) ldflags: @echo $(LDFLAGS) @@ -245,8 +245,12 @@ GO_TEST = $(GOTESTSUM) $(GOTESTSUM_ARGS) -- endif COUNT ?= 1 -NPROC ?= $$(( $(shell nproc) / 2 )) -E2E_PARALLELISM ?= $$(( $(NPROC) > 1 ? $(NPROC) : 1)) +# Only set parallelism if user specified E2E_PARALLELISM +ifdef E2E_PARALLELISM +E2E_PARALLELISM_FLAG := -p $(E2E_PARALLELISM) -parallel $(E2E_PARALLELISM) +else +E2E_PARALLELISM_FLAG := +endif $(DEX): mkdir -p $(TOOLS_DIR) @@ -280,7 +284,7 @@ test-e2e: $(KCP) $(DEX) build ## Run e2e tests $(MAKE) run-kcp &>.kcp/kcp.log & KCP_PID=$$!; \ trap 'kill -TERM $$KCP_PID; rm -rf .kcp' TERM INT EXIT && \ echo "Waiting for kcp to be ready (check .kcp/kcp.log)." && while ! KUBECONFIG=.kcp/admin.kubeconfig kubectl get --raw /readyz &>/dev/null; do sleep 1; echo -n "."; done && echo && \ - KUBECONFIG=$$PWD/.kcp/admin.kubeconfig GOOS=$(OS) GOARCH=$(ARCH) $(GO_TEST) -race -count $(COUNT) -p $(E2E_PARALLELISM) -parallel $(E2E_PARALLELISM) $(WHAT) $(TEST_ARGS) + KUBECONFIG=$$PWD/.kcp/admin.kubeconfig GOOS=$(OS) GOARCH=$(ARCH) $(GO_TEST) -race -count $(COUNT) $(E2E_PARALLELISM_FLAG) $(WHAT) $(TEST_ARGS) CONTRIBS_E2E := $(patsubst %,test-e2e-contrib-%,$(CONTRIBS)) @@ -288,7 +292,7 @@ CONTRIBS_E2E := $(patsubst %,test-e2e-contrib-%,$(CONTRIBS)) test-e2e-contribs: $(CONTRIBS_E2E) ## Run e2e tests for external integrations test-e2e-contrib-kcp: $(DEX) $(KCP) $(CONTRIBS_E2E): - cd contrib/$(patsubst test-e2e-contrib-%,%,$@) && $(GO_TEST) -race -count $(COUNT) -p $(E2E_PARALLELISM) -parallel $(E2E_PARALLELISM) ./test/e2e/... + cd contrib/$(patsubst test-e2e-contrib-%,%,$@) && $(GO_TEST) -race -count $(COUNT) $(E2E_PARALLELISM_FLAG) ./test/e2e/... .PHONY: test ifdef USE_GOTESTSUM diff --git a/backend/controllers/clusterbinding/clusterbinding_reconcile.go b/backend/controllers/clusterbinding/clusterbinding_reconcile.go index d72603e84..e0578adfc 100644 --- a/backend/controllers/clusterbinding/clusterbinding_reconcile.go +++ b/backend/controllers/clusterbinding/clusterbinding_reconcile.go @@ -158,6 +158,11 @@ func (r *reconciler) ensureRBACClusterRole(ctx context.Context, client client.Cl Resources: []string{"boundschemas"}, Verbs: []string{"get", "list", "watch"}, }, + { + APIGroups: []string{kubebindv1alpha2.GroupName}, + Resources: []string{"boundschemas/status"}, + Verbs: []string{"get", "update", "patch"}, + }, }} for _, export := range exports { // Collect unique GroupResources and sort for stable rule ordering. diff --git a/backend/controllers/serviceexport/serviceexport_controller.go b/backend/controllers/serviceexport/serviceexport_controller.go index 7d2cdd2e7..fa40c15c4 100644 --- a/backend/controllers/serviceexport/serviceexport_controller.go +++ b/backend/controllers/serviceexport/serviceexport_controller.go @@ -59,7 +59,7 @@ func NewAPIServiceExportReconciler( opts controller.TypedOptions[mcreconcile.Request], ) (*APIServiceExportReconciler, error) { if err := mgr.GetFieldIndexer().IndexField(ctx, &kubebindv1alpha2.APIServiceExport{}, indexers.ServiceExportByBoundSchema, - indexers.IndexServiceExportByBoundSchema); err != nil { + indexers.IndexServiceExportByBoundSchemaControllerRuntime); err != nil { return nil, fmt.Errorf("failed to setup ServiceExportByBoundSchema indexer: %w", err) } diff --git a/backend/controllers/serviceexportrequest/serviceexportrequest_controller.go b/backend/controllers/serviceexportrequest/serviceexportrequest_controller.go index d34985ab3..2c1969a66 100644 --- a/backend/controllers/serviceexportrequest/serviceexportrequest_controller.go +++ b/backend/controllers/serviceexportrequest/serviceexportrequest_controller.go @@ -83,10 +83,10 @@ func NewAPIServiceExportRequestReconciler( informerScope: scope, clusterScopedIsolation: isolation, schemaSource: schemaSource, - getBoundSchema: func(ctx context.Context, cache cache.Cache, namespace, name string) (*kubebindv1alpha2.BoundSchema, error) { + getBoundSchema: func(ctx context.Context, cl client.Client, namespace, name string) (*kubebindv1alpha2.BoundSchema, error) { var schema kubebindv1alpha2.BoundSchema key := types.NamespacedName{Namespace: namespace, Name: name} - if err := cache.Get(ctx, key, &schema); err != nil { + if err := cl.Get(ctx, key, &schema); err != nil { return nil, err } return &schema, nil diff --git a/backend/controllers/serviceexportrequest/serviceexportrequest_reconcile.go b/backend/controllers/serviceexportrequest/serviceexportrequest_reconcile.go index 0bcf4cf9c..630cf4a8f 100644 --- a/backend/controllers/serviceexportrequest/serviceexportrequest_reconcile.go +++ b/backend/controllers/serviceexportrequest/serviceexportrequest_reconcile.go @@ -44,7 +44,7 @@ type reconciler struct { clusterScopedIsolation kubebindv1alpha2.Isolation schemaSource string - getBoundSchema func(ctx context.Context, cache cache.Cache, namespace, name string) (*kubebindv1alpha2.BoundSchema, error) + getBoundSchema func(ctx context.Context, cl client.Client, namespace, name string) (*kubebindv1alpha2.BoundSchema, error) createBoundSchema func(ctx context.Context, cl client.Client, schema *kubebindv1alpha2.BoundSchema) error getServiceExport func(ctx context.Context, cache cache.Cache, ns, name string) (*kubebindv1alpha2.APIServiceExport, error) @@ -53,14 +53,21 @@ type reconciler struct { } func (r *reconciler) reconcile(ctx context.Context, cl client.Client, cache cache.Cache, req *kubebindv1alpha2.APIServiceExportRequest) error { + // We must ensure schemas are created in form of boundSchemas first for the validation. + // Worst case scenario if validation fails, we will reuse schemas for same consumer once issues are fixed. if err := r.ensureBoundSchemas(ctx, cl, cache, req); err != nil { conditions.SetSummary(req) - return err + return fmt.Errorf("failed to ensure bound schemas: %w", err) + } + + if err := r.validate(ctx, cl, req); err != nil { + conditions.SetSummary(req) + return fmt.Errorf("failed to validate APIServiceExportRequest: %w", err) } if err := r.ensureExports(ctx, cl, cache, req); err != nil { conditions.SetSummary(req) - return err + return fmt.Errorf("failed to ensure exports: %w", err) } // TODO(mjudeikis): we could potentially add finallizer to APIServiceExport above or "adopt" boundschemas @@ -72,104 +79,77 @@ func (r *reconciler) reconcile(ctx context.Context, cl client.Client, cache cach return nil } -func (r *reconciler) ensureBoundSchemas(ctx context.Context, cl client.Client, cache cache.Cache, req *kubebindv1alpha2.APIServiceExportRequest) error { - // Ensure all bound schemas exist - for _, res := range req.Spec.Resources { - parts := strings.SplitN(r.schemaSource, ".", 3) - if len(parts) != 3 { // We check this in validation, but just in case. - return fmt.Errorf("malformed schema source: %q", r.schemaSource) - } +// getExportedSchemas will list all schemas, exported by current backend. +// Important: getExportedSchemas is using client.Client to list resources, not cache. +// This is due to fact we use dynamic client and unstructured.Unstructured to get schemas and it +// does not quite work with dynamic cache informers: +// failed to get informer for *unstructured.UnstructuredList apis.kcp.io/v1alpha1, Kind=APIResourceSchemaList: failed to find newly started informer for apis.kcp.io/v1alpha1, Kind=APIResourceSchema"}. +func (r *reconciler) getExportedSchemas(ctx context.Context, cl client.Client) (kubebindv1alpha2.ExportedSchemas, error) { + parts := strings.SplitN(r.schemaSource, ".", 3) + if len(parts) != 3 { // We check this in validation, but just in case. + return nil, fmt.Errorf("malformed schema source: %q", r.schemaSource) + } - gvk := schema.GroupVersionKind{ - Kind: parts[0], - Version: parts[1], - Group: parts[2], - } + gvk := schema.GroupVersionKind{ + Kind: parts[0], + Version: parts[1], + Group: parts[2], + } - // Ensure we have the List kind - listGVK := gvk - if !strings.HasSuffix(listGVK.Kind, "List") { - listGVK.Kind += "List" - } + // Ensure we have the List kind + listGVK := gvk + if !strings.HasSuffix(listGVK.Kind, "List") { + listGVK.Kind += "List" + } - list := &unstructured.UnstructuredList{} - list.SetGroupVersionKind(listGVK) + list := &unstructured.UnstructuredList{} + list.SetGroupVersionKind(listGVK) - // TODO(mjudeikis): This is hardcoded here and in handlers.go for now. - labelSelector := labels.Set{ - resources.ExportedCRDsLabel: "true", - } + // TODO(mjudeikis): This is hardcoded here and in handlers.go for now. + labelSelector := labels.Set{ + resources.ExportedCRDsLabel: "true", + } - listOpts := []client.ListOption{} - listOpts = append(listOpts, client.MatchingLabelsSelector{Selector: labelSelector.AsSelector()}) + listOpts := []client.ListOption{} + listOpts = append(listOpts, client.MatchingLabelsSelector{Selector: labelSelector.AsSelector()}) - if err := cl.List(ctx, list, listOpts...); err != nil { - return err - } + if err := cl.List(ctx, list, listOpts...); err != nil { + return nil, err + } - for _, item := range list.Items { - var schemaFailed bool - obj := item.UnstructuredContent() - group, ok, err := unstructured.NestedString(obj, "spec", "group") - if !ok || err != nil || group == "" { - klog.FromContext(ctx).Error(err, "Skipping invalid schema: missing group", "ns", item.GetNamespace(), "name", item.GetName()) - schemaFailed = true - } - plural, ok, err := unstructured.NestedString(obj, "spec", "names", "plural") - if !ok || err != nil || plural == "" { - klog.FromContext(ctx).Error(err, "Skipping invalid schema: missing names.plural", "ns", item.GetNamespace(), "name", item.GetName()) - schemaFailed = true - } + boundSchemas := make(kubebindv1alpha2.ExportedSchemas, len(list.Items)) + for _, item := range list.Items { + boundSchema, err := helpers.UnstructuredToBoundSchema(item) + if err != nil { + return nil, err + } + boundSchemas[boundSchema.ResourceGroupName()] = boundSchema + } - scope, ok, err := unstructured.NestedString(obj, "spec", "scope") - if !ok || err != nil || scope == "" { - klog.FromContext(ctx).Error(err, "Skipping invalid schema: missing scope", "ns", item.GetNamespace(), "name", item.GetName()) - schemaFailed = true - } + return boundSchemas, nil +} - if schemaFailed { - conditions.MarkFalse( - req, - kubebindv1alpha2.APIServiceExportRequestConditionExportsReady, - "APIServiceExportRequestInvalid", - conditionsapi.ConditionSeverityError, - "APIServiceExportRequest %s is invalid: resource %s/%s has invalid schema", - req.Name, group, plural, - ) - req.Status.Phase = kubebindv1alpha2.APIServiceExportRequestPhaseFailed - return fmt.Errorf("resource %s/%s is invalid", group, plural) - } +func (r *reconciler) ensureBoundSchemas(ctx context.Context, cl client.Client, cache cache.Cache, req *kubebindv1alpha2.APIServiceExportRequest) error { + exportedSchemas, err := r.getExportedSchemas(ctx, cl) + if err != nil { + return err + } - if group == res.Group && plural == res.Resource { - // Important: This checks if the resource are correctly scoped. If consumer is namespaced, we can't allow this. - // We terminate early to prevent triggering other controllers. - if r.informerScope.String() != scope && r.informerScope != kubebindv1alpha2.ClusterScope { - conditions.MarkFalse( - req, - kubebindv1alpha2.APIServiceExportRequestConditionExportsReady, - "APIServiceExportRequestInvalid", - conditionsapi.ConditionSeverityError, - "APIServiceExportRequest %s is invalid: resource %s/%s has scope %q which is incompatible with backend informer scope %q", - req.Name, group, plural, scope, r.informerScope, - ) - req.Status.Phase = kubebindv1alpha2.APIServiceExportRequestPhaseFailed - req.Status.TerminalMessage = conditions.GetMessage(req, kubebindv1alpha2.APIServiceExportRequestConditionExportsReady) - // We can't proceed with this request. - return fmt.Errorf("resource %s/%s has scope %q which is incompatible with backend informer scope %q", group, plural, scope, r.informerScope) - } + // Ensure all bound schemas exist + for _, res := range req.Spec.Resources { + if len(res.Versions) == 0 { + continue + } - // https://github.com/kube-bind/kube-bind/issues/297 to fix. - boundSchema, err := helpers.UnstructuredToBoundSchema(item) - if err != nil { - return err - } + for _, boundSchema := range exportedSchemas { + if boundSchema.Spec.Group == res.Group && boundSchema.Spec.Names.Plural == res.Resource { boundSchema.Name = res.ResourceGroupName() boundSchema.Namespace = req.Namespace boundSchema.Spec.InformerScope = r.informerScope boundSchema.ResourceVersion = "" - obj, err := r.getBoundSchema(ctx, cache, boundSchema.Namespace, boundSchema.Name) - if err != nil && !apierrors.IsNotFound(err) { + obj, err := r.getBoundSchema(ctx, cl, boundSchema.Namespace, boundSchema.Name) + if err != nil && !apierrors.IsNotFound(err) && !strings.Contains(err.Error(), "no matches for kind") { return err } @@ -196,7 +176,7 @@ func (r *reconciler) ensureExports(ctx context.Context, cl client.Client, cache if req.Status.Phase == kubebindv1alpha2.APIServiceExportRequestPhasePending { for _, res := range req.Spec.Resources { name := res.ResourceGroupName() - boundSchema, err := r.getBoundSchema(ctx, cache, req.Namespace, name) + boundSchema, err := r.getBoundSchema(ctx, cl, req.Namespace, name) if err != nil { if apierrors.IsNotFound(err) { conditions.MarkFalse( @@ -258,6 +238,7 @@ func (r *reconciler) ensureExports(ctx context.Context, cl client.Client, cache Versions: res.Versions, }) } + export.Spec.PermissionClaims = req.Spec.PermissionClaims logger.V(1).Info("Creating APIServiceExport", "name", export.Name, "namespace", export.Namespace) if err := r.createServiceExport(ctx, cl, export); err != nil { @@ -283,3 +264,106 @@ func (r *reconciler) ensureExports(ctx context.Context, cl client.Client, cache return nil } + +// Validate validates if the APIServiceExportRequest is in a valid state. +// Currently it validates if all requested schemas are of the same scope and +// if claimable apis are allowed and valid. +// +// TODO: Move this to validatingAdmissionWebhook as this is not really part of reconciliation. +// https://github.com/kube-bind/kube-bind/issues/325 +func (r *reconciler) validate(ctx context.Context, cl client.Client, req *kubebindv1alpha2.APIServiceExportRequest) error { + exportedSchemas, err := r.getExportedSchemas(ctx, cl) + if err != nil { + return err + } + + if len(exportedSchemas) == 0 { + conditions.MarkFalse( + req, + kubebindv1alpha2.APIServiceExportRequestConditionExportsReady, + "SchemaNotFound", + conditionsapi.ConditionSeverityError, + "Schema not found", + ) + return fmt.Errorf("no exported schemas found") + } + + first := apiextensionsv1.ResourceScope("") + for _, res := range req.Spec.Resources { + boundSchema, ok := exportedSchemas[res.ResourceGroupName()] + if !ok { + conditions.MarkFalse( + req, + kubebindv1alpha2.APIServiceExportRequestConditionExportsReady, + "SchemaNotFound", + conditionsapi.ConditionSeverityError, + "Schema %s not found", + res.ResourceGroupName(), + ) + return fmt.Errorf("schema %s not found", res.ResourceGroupName()) + } + if first == apiextensionsv1.ResourceScope("") { + first = boundSchema.Spec.Scope + continue + } + if boundSchema.Spec.Scope != first { + conditions.MarkFalse(req, + kubebindv1alpha2.APIServiceExportRequestConditionExportsReady, + "DifferentScopes", + conditionsapi.ConditionSeverityError, + "Different scopes found: %v", + boundSchema.Spec.Scope, + ) + return fmt.Errorf("different scopes found for claimed resources: %v", boundSchema.Name) + } + } + + // Add validation if claimable apis are valid here + for _, claim := range req.Spec.PermissionClaims { + if !isClaimableAPI(claim) { + conditions.MarkFalse( + req, + kubebindv1alpha2.APIServiceExportConditionPermissionClaim, + "InvalidPermissionClaim", + conditionsapi.ConditionSeverityError, + "Resource %s is not a valid claimable API", + claim.GroupResource.String(), + ) + req.Status.Phase = kubebindv1alpha2.APIServiceExportRequestPhaseFailed + req.Status.TerminalMessage = conditions.GetMessage(req, kubebindv1alpha2.APIServiceExportConditionPermissionClaim) + return fmt.Errorf("resource %s is not a valid claimable API", claim.GroupResource.String()) + } + } + + // Add validation for duplicate group/resource combinations + seenGroupResources := make(map[string]bool) + for _, claim := range req.Spec.PermissionClaims { + key := claim.Group + "/" + claim.Resource + if seenGroupResources[key] { + conditions.MarkFalse( + req, + kubebindv1alpha2.APIServiceExportConditionPermissionClaim, + "DuplicatePermissionClaim", + conditionsapi.ConditionSeverityError, + "Duplicate permission claim found for group/resource %s", + claim.GroupResource.String(), + ) + req.Status.Phase = kubebindv1alpha2.APIServiceExportRequestPhaseFailed + req.Status.TerminalMessage = conditions.GetMessage(req, kubebindv1alpha2.APIServiceExportConditionPermissionClaim) + return fmt.Errorf("duplicate permission claim found for group/resource %s", claim.GroupResource.String()) + } + seenGroupResources[key] = true + } + + return nil +} + +// isClaimableAPI checks if a permission claim is for a claimable API. +func isClaimableAPI(claim kubebindv1alpha2.PermissionClaim) bool { + for _, api := range kubebindv1alpha2.ClaimableAPIs { + if claim.Group == api.GroupVersionResource.Group && claim.Resource == api.Names.Plural { + return true + } + } + return false +} diff --git a/backend/controllers/servicenamespace/servicenamespace_controller.go b/backend/controllers/servicenamespace/servicenamespace_controller.go index de8ca9882..e9f4e52b0 100644 --- a/backend/controllers/servicenamespace/servicenamespace_controller.go +++ b/backend/controllers/servicenamespace/servicenamespace_controller.go @@ -230,7 +230,7 @@ func (r *APIServiceNamespaceReconciler) Reconcile(ctx context.Context, req mcrec // Fetch the APIServiceNamespace instance apiServiceNamespace := &kubebindv1alpha2.APIServiceNamespace{} - if err := client.Get(ctx, req.NamespacedName, apiServiceNamespace); err != nil { + if err := cache.Get(ctx, req.NamespacedName, apiServiceNamespace); err != nil { if errors.IsNotFound(err) { // Request object not found, could have been deleted after reconcile request. // Handle deletion logic here diff --git a/backend/controllers/servicenamespace/servicenamespace_reconcile.go b/backend/controllers/servicenamespace/servicenamespace_reconcile.go index ffdb1bd53..98ad6d3dc 100644 --- a/backend/controllers/servicenamespace/servicenamespace_reconcile.go +++ b/backend/controllers/servicenamespace/servicenamespace_reconcile.go @@ -25,6 +25,7 @@ import ( rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" @@ -75,6 +76,156 @@ func (c *reconciler) reconcile(ctx context.Context, client client.Client, cache sns.Status.Namespace = nsName } + // List APIServiceExports in the namespace + apiServiceExports, err := c.listAPIServiceExports(ctx, cache, sns.Namespace) + if err != nil { + return fmt.Errorf("failed to list APIServiceExports: %w", err) + } + + for _, export := range apiServiceExports.Items { + name := fmt.Sprintf("kube-binder-%s-export-%s", sns.Name, export.Name) // per-sns unique name + permissions := []rbacv1.PolicyRule{} + for _, claim := range export.Spec.PermissionClaims { + permissions = append(permissions, rbacv1.PolicyRule{ + APIGroups: []string{claim.Group}, + Resources: []string{claim.Resource}, + // We need list and watch for informers to be able to start. And create to create initial object. + Verbs: []string{"*"}, + }) + } + if c.scope == kubebindv1alpha2.ClusterScope { + role, err := c.getPermissionClaimsClusterRole(ctx, cache, name) + if err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("failed to get ClusterRole %s: %w", name, err) + } + if role == nil { + role = &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Rules: permissions, + } + // Create new ClusterRole + if err := client.Create(ctx, role); err != nil { + return fmt.Errorf("failed to create ClusterRole %s: %w", name, err) + } + } else { + role.Rules = permissions + if err := client.Update(ctx, role); err != nil { + return fmt.Errorf("failed to update ClusterRole %s: %w", name, err) + } + } + + clusterBinding, err := c.getPermissionClaimsClusterRoleBinding(ctx, cache, name) + if err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("failed to get ClusterRoleBinding %s: %w", name, err) + } + if clusterBinding == nil { + clusterBinding = &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Namespace: sns.Namespace, + Name: kuberesources.ServiceAccountName, + }, + }, + RoleRef: rbacv1.RoleRef{ + Kind: "ClusterRole", + Name: name, + APIGroup: "rbac.authorization.k8s.io", + }, + } + if err := client.Create(ctx, clusterBinding); err != nil { + return fmt.Errorf("failed to create ClusterRoleBinding %s: %w", name, err) + } + } else { + expectedSubjects := []rbacv1.Subject{{ + Kind: "ServiceAccount", + Namespace: sns.Namespace, + Name: kuberesources.ServiceAccountName, + }} + expectedRef := rbacv1.RoleRef{Kind: "ClusterRole", Name: name, APIGroup: "rbac.authorization.k8s.io"} + if !reflect.DeepEqual(clusterBinding.Subjects, expectedSubjects) || !reflect.DeepEqual(clusterBinding.RoleRef, expectedRef) { + rb := clusterBinding.DeepCopy() + rb.Subjects = expectedSubjects + rb.RoleRef = expectedRef + if err := client.Update(ctx, rb); err != nil { + return fmt.Errorf("failed to update ClusterRoleBinding %s: %w", name, err) + } + } + } + } else { + role, err := c.getPermissionClaimsRole(ctx, cache, sns.Status.Namespace, name) + if err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("failed to get Role %s: %w", name, err) + } + if role == nil { + role := &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: sns.Status.Namespace, + }, + Rules: permissions, + } + // Create new Role + if err := client.Create(ctx, role); err != nil { + return fmt.Errorf("failed to create Role %s: %w", name, err) + } + } else { + role.Rules = permissions + if err := client.Update(ctx, role); err != nil { + return fmt.Errorf("failed to update Role %s: %w", name, err) + } + } + + rolebinding, err := c.getPermissionClaimsRoleBinding(ctx, cache, sns.Status.Namespace, name) + if err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("failed to get RoleBinding %s: %w", name, err) + } + + if rolebinding == nil { + rolebinding := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: sns.Status.Namespace, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Namespace: sns.Namespace, + Name: kuberesources.ServiceAccountName, + }, + }, + RoleRef: rbacv1.RoleRef{ + Kind: "Role", + Name: name, + APIGroup: "rbac.authorization.k8s.io", + }, + } + if err := client.Create(ctx, rolebinding); err != nil { + return fmt.Errorf("failed to create RoleBinding %s: %w", name, err) + } + } else { + expectedSubjects := []rbacv1.Subject{{ + Kind: "ServiceAccount", + Namespace: sns.Namespace, + Name: kuberesources.ServiceAccountName, + }} + if !reflect.DeepEqual(rolebinding.Subjects, expectedSubjects) || rolebinding.RoleRef.Kind != "Role" || rolebinding.RoleRef.Name != name || rolebinding.RoleRef.APIGroup != "rbac.authorization.k8s.io" { + rb := rolebinding.DeepCopy() + rb.Subjects = expectedSubjects + rb.RoleRef = rbacv1.RoleRef{Kind: "Role", Name: name, APIGroup: "rbac.authorization.k8s.io"} + if err := client.Update(ctx, rb); err != nil { + return fmt.Errorf("failed to update RoleBinding %s: %w", name, err) + } + } + } + } + } + return nil } @@ -119,3 +270,47 @@ func (c *reconciler) ensureRBACRoleBinding(ctx context.Context, client client.Cl return nil } + +func (c *reconciler) getPermissionClaimsClusterRole(ctx context.Context, cache cache.Cache, name string) (*rbacv1.ClusterRole, error) { + var role rbacv1.ClusterRole + key := types.NamespacedName{Name: name} + if err := cache.Get(ctx, key, &role); err != nil { + return nil, err + } + return &role, nil +} + +func (c *reconciler) getPermissionClaimsRole(ctx context.Context, cache cache.Cache, namespace, name string) (*rbacv1.Role, error) { + var role rbacv1.Role + key := types.NamespacedName{Namespace: namespace, Name: name} + if err := cache.Get(ctx, key, &role); err != nil { + return nil, err + } + return &role, nil +} + +func (c *reconciler) getPermissionClaimsClusterRoleBinding(ctx context.Context, cache cache.Cache, name string) (*rbacv1.ClusterRoleBinding, error) { + var roleBinding rbacv1.ClusterRoleBinding + key := types.NamespacedName{Name: name} + if err := cache.Get(ctx, key, &roleBinding); err != nil { + return nil, err + } + return &roleBinding, nil +} + +func (c *reconciler) getPermissionClaimsRoleBinding(ctx context.Context, cache cache.Cache, namespace, name string) (*rbacv1.RoleBinding, error) { + var roleBinding rbacv1.RoleBinding + key := types.NamespacedName{Name: name, Namespace: namespace} + if err := cache.Get(ctx, key, &roleBinding); err != nil { + return nil, err + } + return &roleBinding, nil +} + +func (c *reconciler) listAPIServiceExports(ctx context.Context, cache cache.Cache, namespace string) (*kubebindv1alpha2.APIServiceExportList, error) { + exports := &kubebindv1alpha2.APIServiceExportList{} + if err := cache.List(ctx, exports, client.InNamespace(namespace)); err != nil { + return nil, err + } + return exports, nil +} diff --git a/backend/http/handler.go b/backend/http/handler.go index 0b0dca46e..db30335d1 100644 --- a/backend/http/handler.go +++ b/backend/http/handler.go @@ -26,7 +26,6 @@ import ( htmltemplate "html/template" "net/http" "net/url" - "sort" "strings" "time" @@ -34,7 +33,6 @@ import ( "github.com/gorilla/securecookie" "golang.org/x/oauth2" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -47,6 +45,7 @@ import ( "github.com/kube-bind/kube-bind/backend/template" bindversion "github.com/kube-bind/kube-bind/pkg/version" kubebindv1alpha2 "github.com/kube-bind/kube-bind/sdk/apis/kubebind/v1alpha2" + "github.com/kube-bind/kube-bind/sdk/apis/kubebind/v1alpha2/helpers" ) var ( @@ -377,57 +376,45 @@ func (h *handler) handleResources(w http.ResponseWriter, r *http.Request) { return } - apiResourceSchemas, err := h.getBackendDynamicResource(r.Context(), providerCluster) + exportedSchemas, err := h.getBackendDynamicResource(r.Context(), providerCluster) if err != nil { logger.Error(err, "failed to get dynamic resources") http.Error(w, "internal error", http.StatusInternalServerError) return } - var result []UISchema - for _, item := range apiResourceSchemas.Items { - scope := item.UnstructuredContent()["spec"].(map[string]interface{})["scope"] - if scope == nil { - scope = "-" - } - - // TODO(mjudeikis): This logic is very brittle, needs rework. - // This will be improved in the permissionClaims PR. - if !strings.EqualFold(h.scope.String(), scope.(string)) && h.scope != kubebindv1alpha2.ClusterScope { + result := make([]UISchema, 0, len(exportedSchemas)) + for _, item := range exportedSchemas { + if !strings.EqualFold(h.scope.String(), string(item.Spec.Scope)) && h.scope != kubebindv1alpha2.ClusterScope { continue } - group := item.UnstructuredContent()["spec"].(map[string]interface{})["group"] - if group == nil { - group = "-" - } - resource := item.UnstructuredContent()["spec"].(map[string]interface{})["names"].(map[string]interface{})["plural"] - if resource == nil { - resource = "-" - } - - kind := item.UnstructuredContent()["spec"].(map[string]interface{})["names"].(map[string]interface{})["kind"] - if kind == nil { - kind = "-" + if len(item.Spec.Versions) == 0 { + logger.Error(fmt.Errorf("no versions found"), "skipping schema", "name", item.Name) + continue } - - versions := item.UnstructuredContent()["spec"].(map[string]interface{})["versions"] - if versions == nil { - versions = []interface{}{""} + // pick first served version + ver := "" + for _, v := range item.Spec.Versions { + if v.Served { + ver = v.Name + break + } } - for _, v := range versions.([]interface{}) { - version := v.(map[string]interface{})["name"] - result = append(result, UISchema{ - Name: item.GetName(), - Kind: kind.(string), - Scope: scope.(string), - Version: version.(string), - Group: group.(string), - // Important: This MUST be used as UI button class in the url, so tests can 'click it' based on it. - Resource: resource.(string), - SessionID: sessionID, - }) + if ver == "" { + logger.Error(fmt.Errorf("no served versions found"), "skipping schema", "name", item.Name) + continue } + result = append(result, UISchema{ + Name: item.GetName(), + Kind: item.Spec.Names.Kind, + Scope: string(item.Spec.Scope), + Version: ver, + Group: item.Spec.Group, + // Important: This MUST be used as UI button class in the url, so tests can 'click it' based on it. + Resource: item.Spec.Names.Plural, + SessionID: sessionID, + }) } bs := bytes.Buffer{} @@ -556,7 +543,7 @@ func mustRead(f func(name string) ([]byte, error), name string) string { return string(bs) } -func (h *handler) getBackendDynamicResource(ctx context.Context, cluster string) (*unstructured.UnstructuredList, error) { +func (h *handler) getBackendDynamicResource(ctx context.Context, cluster string) (kubebindv1alpha2.ExportedSchemas, error) { labelSelector := labels.Set{ resources.ExportedCRDsLabel: "true", } @@ -571,12 +558,19 @@ func (h *handler) getBackendDynamicResource(ctx context.Context, cluster string) Version: parts[1], Group: parts[2], } - apiResourceSchemas, err := h.kubeManager.ListDynamicResources(ctx, cluster, gvk, labelSelector.AsSelector()) + list, err := h.kubeManager.ListDynamicResources(ctx, cluster, gvk, labelSelector.AsSelector()) if err != nil { - return nil, fmt.Errorf("failed to list crds: %w", err) + return nil, fmt.Errorf("failed to list resources: %w", err) + } + + boundSchemas := make(kubebindv1alpha2.ExportedSchemas, len(list.Items)) + for _, item := range list.Items { + boundSchema, err := helpers.UnstructuredToBoundSchema(item) + if err != nil { + return nil, err + } + boundSchemas[boundSchema.ResourceGroupName()] = boundSchema } - sort.SliceStable(apiResourceSchemas.Items, func(i, j int) bool { - return apiResourceSchemas.Items[i].GetName() < apiResourceSchemas.Items[j].GetName() - }) - return apiResourceSchemas, nil + + return boundSchemas, nil } diff --git a/backend/kubernetes/resources/namespace.go b/backend/kubernetes/resources/namespace.go index 4e3f01a26..83176d937 100644 --- a/backend/kubernetes/resources/namespace.go +++ b/backend/kubernetes/resources/namespace.go @@ -28,9 +28,31 @@ import ( ) const ( - IdentityAnnotationKey = "example-backend.kube-bind.io/identity" + IdentityAnnotationKey = "backend.kube-bind.io/identity" + legacyIdentityAnnotationKey = "example-backend.kube-bind.io/identity" ) +func handleLegacyAnnotations(ctx context.Context, cl client.Client, namespace *corev1.Namespace, id string) error { + if namespace.Annotations == nil { + return nil + } + + legacyValue, hasLegacy := namespace.Annotations[legacyIdentityAnnotationKey] + currentValue, hasCurrent := namespace.Annotations[IdentityAnnotationKey] + + if hasLegacy && (!hasCurrent || currentValue != id) && legacyValue == id { + original := namespace.DeepCopy() + if namespace.Annotations == nil { + namespace.Annotations = map[string]string{} + } + namespace.Annotations[IdentityAnnotationKey] = id + delete(namespace.Annotations, legacyIdentityAnnotationKey) + return cl.Patch(ctx, namespace, client.MergeFrom(original)) + } + + return nil +} + func CreateNamespace(ctx context.Context, client client.Client, generateName, id string) (*corev1.Namespace, error) { if !strings.HasSuffix(generateName, "-") { generateName += "-" @@ -52,9 +74,14 @@ func CreateNamespace(ctx context.Context, client client.Client, generateName, id if err != nil { return nil, err } - if namespace.Annotations[IdentityAnnotationKey] != id { + + if namespace.Annotations[IdentityAnnotationKey] != id && namespace.Annotations[legacyIdentityAnnotationKey] != id { return nil, errors.NewAlreadyExists(corev1.Resource("namespace"), namespace.Name) } + + if err := handleLegacyAnnotations(ctx, client, namespace, id); err != nil { + return nil, err + } } return namespace, nil diff --git a/backend/kubernetes/resources/namespace_test.go b/backend/kubernetes/resources/namespace_test.go new file mode 100644 index 000000000..b172186de --- /dev/null +++ b/backend/kubernetes/resources/namespace_test.go @@ -0,0 +1,297 @@ +/* +Copyright 2022 The Kube Bind Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resources + +import ( + "context" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestCreateNamespace(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + + tests := []struct { + name string + generateName string + id string + wantErr bool + wantNameGenerated bool + wantAnnotations map[string]string + }{ + { + name: "create new namespace with generateName", + generateName: "test-ns", + id: "test-id-123", + wantErr: false, + wantNameGenerated: true, + wantAnnotations: map[string]string{ + IdentityAnnotationKey: "test-id-123", + }, + }, + { + name: "create new namespace with generateName already ending with dash", + generateName: "test-ns-", + id: "test-id-456", + wantErr: false, + wantNameGenerated: true, + wantAnnotations: map[string]string{ + IdentityAnnotationKey: "test-id-456", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + cl := fake.NewClientBuilder().WithScheme(scheme).Build() + + result, err := CreateNamespace(ctx, cl, tt.generateName, tt.id) + + if tt.wantErr { + if err == nil { + t.Errorf("CreateNamespace() expected error but got none") + return + } + return + } + + if err != nil { + t.Errorf("CreateNamespace() unexpected error = %v", err) + return + } + + if result == nil { + t.Errorf("CreateNamespace() returned nil namespace") + return + } + + expectedGenerateNamePrefix := tt.generateName + if !strings.HasSuffix(expectedGenerateNamePrefix, "-") { + expectedGenerateNamePrefix += "-" + } + + if tt.wantNameGenerated { + if !strings.HasPrefix(result.Name, expectedGenerateNamePrefix) { + t.Errorf("CreateNamespace() name = %v, expected to start with %v", result.Name, expectedGenerateNamePrefix) + } + if result.GenerateName != expectedGenerateNamePrefix { + t.Errorf("CreateNamespace() generateName = %v, expected %v", result.GenerateName, expectedGenerateNamePrefix) + } + } + + for key, expectedValue := range tt.wantAnnotations { + if actualValue, exists := result.Annotations[key]; !exists || actualValue != expectedValue { + t.Errorf("CreateNamespace() annotation %s = %v, expected %v (exists: %v)", key, actualValue, expectedValue, exists) + } + } + + var actualNamespace corev1.Namespace + if err := cl.Get(ctx, client.ObjectKey{Name: result.Name}, &actualNamespace); err != nil { + t.Fatalf("Failed to get created/updated namespace from client: %v", err) + } + + for key, expectedValue := range tt.wantAnnotations { + if actualValue, exists := actualNamespace.Annotations[key]; !exists || actualValue != expectedValue { + t.Errorf("CreateNamespace() stored annotation %s = %v, expected %v (exists: %v)", key, actualValue, expectedValue, exists) + } + } + }) + } +} + +func TestCreateNamespaceLegacyAnnotationHandling(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + + tests := []struct { + name string + existingNamespace *corev1.Namespace + generateName string + id string + wantErr bool + wantErrType func(error) bool + wantLegacyMigration bool + wantAnnotations map[string]string + }{ + { + name: "namespace exists with matching current annotation - no migration needed", + existingNamespace: &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ns-abc123", + Annotations: map[string]string{ + IdentityAnnotationKey: "matching-id", + }, + }, + }, + generateName: "test-ns", + id: "matching-id", + wantErr: false, + wantAnnotations: map[string]string{ + IdentityAnnotationKey: "matching-id", + }, + wantLegacyMigration: false, + }, + { + name: "namespace exists with matching legacy annotation - should migrate", + existingNamespace: &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ns-xyz789", + Annotations: map[string]string{ + legacyIdentityAnnotationKey: "legacy-id", + }, + }, + }, + generateName: "test-ns", + id: "legacy-id", + wantErr: false, + wantAnnotations: map[string]string{ + IdentityAnnotationKey: "legacy-id", + }, + wantLegacyMigration: true, + }, + { + name: "namespace exists with both annotations, legacy matches - should migrate", + existingNamespace: &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ns-mixed", + Annotations: map[string]string{ + IdentityAnnotationKey: "wrong-id", + legacyIdentityAnnotationKey: "correct-id", + }, + }, + }, + generateName: "test-ns", + id: "correct-id", + wantErr: false, + wantAnnotations: map[string]string{ + IdentityAnnotationKey: "correct-id", + }, + wantLegacyMigration: true, + }, + { + name: "namespace exists with non-matching annotations - should return AlreadyExists error", + existingNamespace: &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ns-conflict", + Annotations: map[string]string{ + IdentityAnnotationKey: "different-id", + legacyIdentityAnnotationKey: "another-id", + }, + }, + }, + generateName: "test-ns", + id: "new-id", + wantErr: true, + wantErrType: errors.IsAlreadyExists, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + + interceptorClient := &alreadyExistsClient{ + Client: fake.NewClientBuilder().WithScheme(scheme).WithObjects(tt.existingNamespace).Build(), + existingNamespace: tt.existingNamespace, + } + + result, err := CreateNamespace(ctx, interceptorClient, tt.generateName, tt.id) + + if tt.wantErr { + if err == nil { + t.Errorf("CreateNamespace() expected error but got none") + return + } + if tt.wantErrType != nil && !tt.wantErrType(err) { + t.Errorf("CreateNamespace() error type mismatch, got %T: %v", err, err) + } + return + } + + if err != nil { + t.Errorf("CreateNamespace() unexpected error = %v", err) + return + } + + if result == nil { + t.Errorf("CreateNamespace() returned nil namespace") + return + } + + if result.Name != tt.existingNamespace.Name { + t.Errorf("CreateNamespace() name = %v, expected %v", result.Name, tt.existingNamespace.Name) + } + + for key, expectedValue := range tt.wantAnnotations { + if actualValue, exists := result.Annotations[key]; !exists || actualValue != expectedValue { + t.Errorf("CreateNamespace() annotation %s = %v, expected %v (exists: %v)", key, actualValue, expectedValue, exists) + } + } + + if tt.wantLegacyMigration { + if _, exists := result.Annotations[legacyIdentityAnnotationKey]; exists { + t.Errorf("CreateNamespace() legacy annotation should be removed but still exists: %v", result.Annotations) + } + } + + var actualNamespace corev1.Namespace + if err := interceptorClient.Get(ctx, client.ObjectKey{Name: result.Name}, &actualNamespace); err != nil { + t.Fatalf("Failed to get updated namespace from client: %v", err) + } + + for key, expectedValue := range tt.wantAnnotations { + if actualValue, exists := actualNamespace.Annotations[key]; !exists || actualValue != expectedValue { + t.Errorf("CreateNamespace() stored annotation %s = %v, expected %v (exists: %v)", key, actualValue, expectedValue, exists) + } + } + + if tt.wantLegacyMigration { + if _, exists := actualNamespace.Annotations[legacyIdentityAnnotationKey]; exists { + t.Errorf("CreateNamespace() legacy annotation should be removed from stored namespace but still exists: %v", actualNamespace.Annotations) + } + } + }) + } +} + +type alreadyExistsClient struct { + client.Client + existingNamespace *corev1.Namespace +} + +func (c *alreadyExistsClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + namespace, ok := obj.(*corev1.Namespace) + if !ok { + return c.Client.Create(ctx, obj, opts...) + } + + if namespace.GenerateName != "" { + namespace.Name = c.existingNamespace.Name + return errors.NewAlreadyExists(corev1.Resource("namespace"), namespace.Name) + } + + return c.Client.Create(ctx, obj, opts...) +} diff --git a/contrib/kcp/README.md b/contrib/kcp/README.md index efc4ab65e..804818261 100644 --- a/contrib/kcp/README.md +++ b/contrib/kcp/README.md @@ -60,7 +60,8 @@ k ws use :root:kube-bind --namespace-prefix="kube-bind-" \ --cookie-signing-key=bGMHz7SR9XcI9JdDB68VmjQErrjbrAR9JdVqjAOKHzE= \ --cookie-encryption-key=wadqi4u+w0bqnSrVFtM38Pz2ykYVIeeadhzT34XlC1Y= \ - --schema-source apiresourceschemas + --schema-source apiresourceschemas \ + --consumer-scope=cluster ``` This process will keep running, so open a new terminal. @@ -197,5 +198,4 @@ kubectl label secret provider-secret app=wildwest kubectl create namespace bob kubectl create secret generic wildwest-secrets1 -n bob kubectl label secret wildwest-secrets1 app=wildwest -n bob - ``` diff --git a/contrib/kcp/deploy/bootstrap.go b/contrib/kcp/deploy/bootstrap.go index d1b9bd2e0..7ef602a0b 100644 --- a/contrib/kcp/deploy/bootstrap.go +++ b/contrib/kcp/deploy/bootstrap.go @@ -234,6 +234,21 @@ func bindAPIExport(ctx context.Context, kcpClient kcpclient.Interface, exportNam }, State: apisv1alpha2.ClaimAccepted, }, + { + ScopedPermissionClaim: apisv1alpha2.ScopedPermissionClaim{ + PermissionClaim: apisv1alpha2.PermissionClaim{ + GroupResource: apisv1alpha2.GroupResource{ + Group: "apis.kcp.io", + Resource: "apiresourceschemas", + }, + Verbs: []string{"*"}, + }, + Selector: apisv1alpha2.PermissionClaimSelector{ + MatchAll: true, + }, + }, + State: apisv1alpha2.ClaimAccepted, + }, } _, err := kcpClient.ApisV1alpha2().APIBindings().Create(ctx, binding, metav1.CreateOptions{}) diff --git a/contrib/kcp/deploy/examples/apiserviceexport-cluster.yaml b/contrib/kcp/deploy/examples/apiserviceexport-cluster.yaml index 72a3bd448..14fec2669 100644 --- a/contrib/kcp/deploy/examples/apiserviceexport-cluster.yaml +++ b/contrib/kcp/deploy/examples/apiserviceexport-cluster.yaml @@ -8,4 +8,11 @@ spec: resource: sheriffs versions: - v1alpha1 + permissionClaims: + - group: "" + resource: configmaps + selector: + labelSelector: + matchLabels: + app: wildwest status: {} diff --git a/contrib/kcp/deploy/examples/apiserviceexport-namespaced.yaml b/contrib/kcp/deploy/examples/apiserviceexport-namespaced.yaml index a6ff6e1f6..53c43920d 100644 --- a/contrib/kcp/deploy/examples/apiserviceexport-namespaced.yaml +++ b/contrib/kcp/deploy/examples/apiserviceexport-namespaced.yaml @@ -8,4 +8,20 @@ spec: resource: cowboys versions: - v1alpha1 + permissionClaims: + - apiGroup: "" + resource: configmaps + selector: + labelSelector: + matchLabels: + app: wildwest + - apiGroup: "" + resource: secrets + selector: + labelSelector: + matchLabels: + app: wildwest + namedResource: + - name: wildwest-secrets + namespace: bob status: {} diff --git a/contrib/kcp/deploy/examples/cowboy.yaml b/contrib/kcp/deploy/examples/cowboy.yaml index 36eec6152..369dc9695 100644 --- a/contrib/kcp/deploy/examples/cowboy.yaml +++ b/contrib/kcp/deploy/examples/cowboy.yaml @@ -2,4 +2,6 @@ apiVersion: wildwest.dev/v1alpha1 kind: Cowboy metadata: name: my-cowboy - namespace: default \ No newline at end of file + namespace: default +spec: + intent: "ride into the sunset" diff --git a/contrib/kcp/deploy/examples/sheriff.yaml b/contrib/kcp/deploy/examples/sheriff.yaml index 7a174bbd7..dff34e843 100644 --- a/contrib/kcp/deploy/examples/sheriff.yaml +++ b/contrib/kcp/deploy/examples/sheriff.yaml @@ -1,4 +1,6 @@ apiVersion: wildwest.dev/v1alpha1 kind: Sheriff metadata: - name: sheriff \ No newline at end of file + name: sheriff +spec: + intent: "ride into the sunset" \ No newline at end of file diff --git a/contrib/kcp/deploy/resources/apiexport-kube-bind.io.yaml b/contrib/kcp/deploy/resources/apiexport-kube-bind.io.yaml index e0db92618..5a8b03b89 100644 --- a/contrib/kcp/deploy/resources/apiexport-kube-bind.io.yaml +++ b/contrib/kcp/deploy/resources/apiexport-kube-bind.io.yaml @@ -49,17 +49,17 @@ spec: crd: {} - group: kube-bind.io name: apiservicebindings - schema: v250809-5ed76a1.apiservicebindings.kube-bind.io + schema: v250929-62fc678.apiservicebindings.kube-bind.io storage: crd: {} - group: kube-bind.io name: apiserviceexportrequests - schema: v250809-5ed76a1.apiserviceexportrequests.kube-bind.io + schema: v250929-62fc678.apiserviceexportrequests.kube-bind.io storage: crd: {} - group: kube-bind.io name: apiserviceexports - schema: v250902-878858c.apiserviceexports.kube-bind.io + schema: v250929-62fc678.apiserviceexports.kube-bind.io storage: crd: {} - group: kube-bind.io diff --git a/contrib/kcp/deploy/resources/apiresourceschema-apiservicebindings.kube-bind.io.yaml b/contrib/kcp/deploy/resources/apiresourceschema-apiservicebindings.kube-bind.io.yaml index 33d76eda1..6a1d7c57a 100644 --- a/contrib/kcp/deploy/resources/apiresourceschema-apiservicebindings.kube-bind.io.yaml +++ b/contrib/kcp/deploy/resources/apiresourceschema-apiservicebindings.kube-bind.io.yaml @@ -2,7 +2,7 @@ apiVersion: apis.kcp.io/v1alpha1 kind: APIResourceSchema metadata: creationTimestamp: null - name: v250809-5ed76a1.apiservicebindings.kube-bind.io + name: v250929-62fc678.apiservicebindings.kube-bind.io spec: conversion: strategy: None @@ -300,6 +300,108 @@ spec: - type type: object type: array + permissionClaims: + description: |- + PermissionClaims records decisions about permission claims requested by the service provider. + Access is granted per GroupResource. + items: + description: |- + PermissionClaim selects objects of a GVR that a service provider may + request and that a consumer may accept and allow the service provider access to. + properties: + group: + default: "" + description: |- + group is the name of an API group. + For core groups this is the empty string '""'. + pattern: ^(|[a-z0-9]([-a-z0-9]*[a-z0-9](\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)?)$ + type: string + resource: + description: |- + resource is the name of the resource. + Note: it is worth noting that you can not ask for permissions for resource provided by a CRD + not provided by an service binding export. + pattern: ^[a-z][-a-z0-9]*[a-z0-9]$ + type: string + selector: + description: Selector is a resource selector that selects objects + of a GVR. + properties: + labelSelector: + description: LabelSelector is a label selector that selects + objects of a GVR. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namedResource: + description: NamedResource is a shorthand for selecting a + single resource by name and namespace. + items: + description: NamedResource selects a specific resource by + name and namespace. + properties: + name: + description: |- + Name is the name of the resource. + Name matches the metadata.name field of the underlying object. + type: string + namespace: + description: |- + Namespace represents namespace where an object of the given group/resource may be managed. + Namespaces matches against the metadata.namespace field. If not provided, the object is assumed to be cluster-scoped. + Namespaces field is ignored for namespaced isolation mode. + type: string + required: + - name + type: object + type: array + type: object + required: + - resource + - selector + type: object + type: array providerPrettyName: description: |- providerPrettyName is the pretty name of the service provider cluster. This diff --git a/contrib/kcp/deploy/resources/apiresourceschema-apiserviceexportrequests.kube-bind.io.yaml b/contrib/kcp/deploy/resources/apiresourceschema-apiserviceexportrequests.kube-bind.io.yaml index d1d26d448..76a220263 100644 --- a/contrib/kcp/deploy/resources/apiresourceschema-apiserviceexportrequests.kube-bind.io.yaml +++ b/contrib/kcp/deploy/resources/apiresourceschema-apiserviceexportrequests.kube-bind.io.yaml @@ -2,7 +2,7 @@ apiVersion: apis.kcp.io/v1alpha1 kind: APIResourceSchema metadata: creationTimestamp: null - name: v250809-5ed76a1.apiserviceexportrequests.kube-bind.io + name: v250929-62fc678.apiserviceexportrequests.kube-bind.io spec: conversion: strategy: None @@ -217,6 +217,111 @@ spec: x-kubernetes-validations: - message: parameters are immutable rule: self == oldSelf + permissionClaims: + description: |- + PermissionClaims records decisions about permission claims requested by the service provider. + Access is granted per GroupResource. + items: + description: |- + PermissionClaim selects objects of a GVR that a service provider may + request and that a consumer may accept and allow the service provider access to. + properties: + group: + default: "" + description: |- + group is the name of an API group. + For core groups this is the empty string '""'. + pattern: ^(|[a-z0-9]([-a-z0-9]*[a-z0-9](\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)?)$ + type: string + resource: + description: |- + resource is the name of the resource. + Note: it is worth noting that you can not ask for permissions for resource provided by a CRD + not provided by an service binding export. + pattern: ^[a-z][-a-z0-9]*[a-z0-9]$ + type: string + selector: + description: Selector is a resource selector that selects objects + of a GVR. + properties: + labelSelector: + description: LabelSelector is a label selector that selects + objects of a GVR. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namedResource: + description: NamedResource is a shorthand for selecting a + single resource by name and namespace. + items: + description: NamedResource selects a specific resource by + name and namespace. + properties: + name: + description: |- + Name is the name of the resource. + Name matches the metadata.name field of the underlying object. + type: string + namespace: + description: |- + Namespace represents namespace where an object of the given group/resource may be managed. + Namespaces matches against the metadata.namespace field. If not provided, the object is assumed to be cluster-scoped. + Namespaces field is ignored for namespaced isolation mode. + type: string + required: + - name + type: object + type: array + type: object + required: + - resource + - selector + type: object + type: array + x-kubernetes-validations: + - message: permissionClaims are immutable + rule: self == oldSelf resources: description: resources is a list of resources that should be exported. items: diff --git a/contrib/kcp/deploy/resources/apiresourceschema-apiserviceexports.kube-bind.io.yaml b/contrib/kcp/deploy/resources/apiresourceschema-apiserviceexports.kube-bind.io.yaml index 1e8a83da1..208a38641 100644 --- a/contrib/kcp/deploy/resources/apiresourceschema-apiserviceexports.kube-bind.io.yaml +++ b/contrib/kcp/deploy/resources/apiresourceschema-apiserviceexports.kube-bind.io.yaml @@ -2,7 +2,7 @@ apiVersion: apis.kcp.io/v1alpha1 kind: APIResourceSchema metadata: creationTimestamp: null - name: v250902-878858c.apiserviceexports.kube-bind.io + name: v250929-62fc678.apiserviceexports.kube-bind.io spec: conversion: strategy: None @@ -487,6 +487,111 @@ spec: x-kubernetes-validations: - message: informerScope is immutable rule: self == oldSelf + permissionClaims: + description: |- + PermissionClaims records decisions about permission claims requested by the service provider. + Access is granted per GroupResource. + items: + description: |- + PermissionClaim selects objects of a GVR that a service provider may + request and that a consumer may accept and allow the service provider access to. + properties: + group: + default: "" + description: |- + group is the name of an API group. + For core groups this is the empty string '""'. + pattern: ^(|[a-z0-9]([-a-z0-9]*[a-z0-9](\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)?)$ + type: string + resource: + description: |- + resource is the name of the resource. + Note: it is worth noting that you can not ask for permissions for resource provided by a CRD + not provided by an service binding export. + pattern: ^[a-z][-a-z0-9]*[a-z0-9]$ + type: string + selector: + description: Selector is a resource selector that selects objects + of a GVR. + properties: + labelSelector: + description: LabelSelector is a label selector that selects + objects of a GVR. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namedResource: + description: NamedResource is a shorthand for selecting a + single resource by name and namespace. + items: + description: NamedResource selects a specific resource by + name and namespace. + properties: + name: + description: |- + Name is the name of the resource. + Name matches the metadata.name field of the underlying object. + type: string + namespace: + description: |- + Namespace represents namespace where an object of the given group/resource may be managed. + Namespaces matches against the metadata.namespace field. If not provided, the object is assumed to be cluster-scoped. + Namespaces field is ignored for namespaced isolation mode. + type: string + required: + - name + type: object + type: array + type: object + required: + - resource + - selector + type: object + type: array + x-kubernetes-validations: + - message: permissionClaims are immutable + rule: self == oldSelf resources: description: resources is a list of resources that should be exported. items: @@ -527,53 +632,6 @@ spec: status: description: status contains reconciliation information for the resource. properties: - acceptedNames: - description: |- - acceptedNames are the names that are actually being used to serve discovery. - They may be different than the names in spec. - properties: - categories: - description: |- - categories is a list of grouped resources this custom resource belongs to (e.g. 'all'). - This is published in API discovery documents, and used by clients to support invocations like - `kubectl get all`. - items: - type: string - type: array - x-kubernetes-list-type: atomic - kind: - description: |- - kind is the serialized kind of the resource. It is normally CamelCase and singular. - Custom resource instances will use this value as the `kind` attribute in API calls. - type: string - listKind: - description: listKind is the serialized kind of the list for this - resource. Defaults to "`kind`List". - type: string - plural: - description: |- - plural is the plural name of the resource to serve. - The custom resources are served under `/apis///.../`. - Must match the name of the CustomResourceDefinition (in the form `.`). - Must be all lowercase. - type: string - shortNames: - description: |- - shortNames are short names for the resource, exposed in API discovery documents, - and used by clients to support invocations like `kubectl get `. - It must be all lowercase. - items: - type: string - type: array - x-kubernetes-list-type: atomic - singular: - description: singular is the singular name of the resource. It must - be all lowercase. Defaults to lowercased `kind`. - type: string - required: - - kind - - plural - type: object conditions: description: |- conditions is a list of conditions that apply to the APIServiceExport. It is @@ -621,17 +679,6 @@ spec: - type type: object type: array - storedVersions: - description: |- - storedVersions lists all versions of CustomResources that were ever persisted. Tracking these - versions allows a migration path for stored versions in etcd. The field is mutable - so a migration controller can finish a migration to another version (ensuring - no old objects are left in storage), and then remove the rest of the - versions from this list. - Versions may not be removed from `spec.versions` while they exist in this list. - items: - type: string - type: array type: object required: - spec diff --git a/deploy/crd/kube-bind.io_apiservicebindings.yaml b/deploy/crd/kube-bind.io_apiservicebindings.yaml index c68ca4e9f..d94c50a99 100644 --- a/deploy/crd/kube-bind.io_apiservicebindings.yaml +++ b/deploy/crd/kube-bind.io_apiservicebindings.yaml @@ -307,6 +307,108 @@ spec: - type type: object type: array + permissionClaims: + description: |- + PermissionClaims records decisions about permission claims requested by the service provider. + Access is granted per GroupResource. + items: + description: |- + PermissionClaim selects objects of a GVR that a service provider may + request and that a consumer may accept and allow the service provider access to. + properties: + group: + default: "" + description: |- + group is the name of an API group. + For core groups this is the empty string '""'. + pattern: ^(|[a-z0-9]([-a-z0-9]*[a-z0-9](\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)?)$ + type: string + resource: + description: |- + resource is the name of the resource. + Note: it is worth noting that you can not ask for permissions for resource provided by a CRD + not provided by an service binding export. + pattern: ^[a-z][-a-z0-9]*[a-z0-9]$ + type: string + selector: + description: Selector is a resource selector that selects objects + of a GVR. + properties: + labelSelector: + description: LabelSelector is a label selector that selects + objects of a GVR. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namedResource: + description: NamedResource is a shorthand for selecting + a single resource by name and namespace. + items: + description: NamedResource selects a specific resource + by name and namespace. + properties: + name: + description: |- + Name is the name of the resource. + Name matches the metadata.name field of the underlying object. + type: string + namespace: + description: |- + Namespace represents namespace where an object of the given group/resource may be managed. + Namespaces matches against the metadata.namespace field. If not provided, the object is assumed to be cluster-scoped. + Namespaces field is ignored for namespaced isolation mode. + type: string + required: + - name + type: object + type: array + type: object + required: + - resource + - selector + type: object + type: array providerPrettyName: description: |- providerPrettyName is the pretty name of the service provider cluster. This diff --git a/deploy/crd/kube-bind.io_apiserviceexportrequests.yaml b/deploy/crd/kube-bind.io_apiserviceexportrequests.yaml index b7aaf85c8..2dc9adcfc 100644 --- a/deploy/crd/kube-bind.io_apiserviceexportrequests.yaml +++ b/deploy/crd/kube-bind.io_apiserviceexportrequests.yaml @@ -221,6 +221,111 @@ spec: x-kubernetes-validations: - message: parameters are immutable rule: self == oldSelf + permissionClaims: + description: |- + PermissionClaims records decisions about permission claims requested by the service provider. + Access is granted per GroupResource. + items: + description: |- + PermissionClaim selects objects of a GVR that a service provider may + request and that a consumer may accept and allow the service provider access to. + properties: + group: + default: "" + description: |- + group is the name of an API group. + For core groups this is the empty string '""'. + pattern: ^(|[a-z0-9]([-a-z0-9]*[a-z0-9](\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)?)$ + type: string + resource: + description: |- + resource is the name of the resource. + Note: it is worth noting that you can not ask for permissions for resource provided by a CRD + not provided by an service binding export. + pattern: ^[a-z][-a-z0-9]*[a-z0-9]$ + type: string + selector: + description: Selector is a resource selector that selects objects + of a GVR. + properties: + labelSelector: + description: LabelSelector is a label selector that selects + objects of a GVR. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namedResource: + description: NamedResource is a shorthand for selecting + a single resource by name and namespace. + items: + description: NamedResource selects a specific resource + by name and namespace. + properties: + name: + description: |- + Name is the name of the resource. + Name matches the metadata.name field of the underlying object. + type: string + namespace: + description: |- + Namespace represents namespace where an object of the given group/resource may be managed. + Namespaces matches against the metadata.namespace field. If not provided, the object is assumed to be cluster-scoped. + Namespaces field is ignored for namespaced isolation mode. + type: string + required: + - name + type: object + type: array + type: object + required: + - resource + - selector + type: object + type: array + x-kubernetes-validations: + - message: permissionClaims are immutable + rule: self == oldSelf resources: description: resources is a list of resources that should be exported. items: diff --git a/deploy/crd/kube-bind.io_apiserviceexports.yaml b/deploy/crd/kube-bind.io_apiserviceexports.yaml index 138ba8419..2aaf068b9 100644 --- a/deploy/crd/kube-bind.io_apiserviceexports.yaml +++ b/deploy/crd/kube-bind.io_apiserviceexports.yaml @@ -490,6 +490,111 @@ spec: x-kubernetes-validations: - message: informerScope is immutable rule: self == oldSelf + permissionClaims: + description: |- + PermissionClaims records decisions about permission claims requested by the service provider. + Access is granted per GroupResource. + items: + description: |- + PermissionClaim selects objects of a GVR that a service provider may + request and that a consumer may accept and allow the service provider access to. + properties: + group: + default: "" + description: |- + group is the name of an API group. + For core groups this is the empty string '""'. + pattern: ^(|[a-z0-9]([-a-z0-9]*[a-z0-9](\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)?)$ + type: string + resource: + description: |- + resource is the name of the resource. + Note: it is worth noting that you can not ask for permissions for resource provided by a CRD + not provided by an service binding export. + pattern: ^[a-z][-a-z0-9]*[a-z0-9]$ + type: string + selector: + description: Selector is a resource selector that selects objects + of a GVR. + properties: + labelSelector: + description: LabelSelector is a label selector that selects + objects of a GVR. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namedResource: + description: NamedResource is a shorthand for selecting + a single resource by name and namespace. + items: + description: NamedResource selects a specific resource + by name and namespace. + properties: + name: + description: |- + Name is the name of the resource. + Name matches the metadata.name field of the underlying object. + type: string + namespace: + description: |- + Namespace represents namespace where an object of the given group/resource may be managed. + Namespaces matches against the metadata.namespace field. If not provided, the object is assumed to be cluster-scoped. + Namespaces field is ignored for namespaced isolation mode. + type: string + required: + - name + type: object + type: array + type: object + required: + - resource + - selector + type: object + type: array + x-kubernetes-validations: + - message: permissionClaims are immutable + rule: self == oldSelf resources: description: resources is a list of resources that should be exported. items: @@ -530,53 +635,6 @@ spec: status: description: status contains reconciliation information for the resource. properties: - acceptedNames: - description: |- - acceptedNames are the names that are actually being used to serve discovery. - They may be different than the names in spec. - properties: - categories: - description: |- - categories is a list of grouped resources this custom resource belongs to (e.g. 'all'). - This is published in API discovery documents, and used by clients to support invocations like - `kubectl get all`. - items: - type: string - type: array - x-kubernetes-list-type: atomic - kind: - description: |- - kind is the serialized kind of the resource. It is normally CamelCase and singular. - Custom resource instances will use this value as the `kind` attribute in API calls. - type: string - listKind: - description: listKind is the serialized kind of the list for this - resource. Defaults to "`kind`List". - type: string - plural: - description: |- - plural is the plural name of the resource to serve. - The custom resources are served under `/apis///.../`. - Must match the name of the CustomResourceDefinition (in the form `.`). - Must be all lowercase. - type: string - shortNames: - description: |- - shortNames are short names for the resource, exposed in API discovery documents, - and used by clients to support invocations like `kubectl get `. - It must be all lowercase. - items: - type: string - type: array - x-kubernetes-list-type: atomic - singular: - description: singular is the singular name of the resource. It - must be all lowercase. Defaults to lowercased `kind`. - type: string - required: - - kind - - plural - type: object conditions: description: |- conditions is a list of conditions that apply to the APIServiceExport. It is @@ -624,17 +682,6 @@ spec: - type type: object type: array - storedVersions: - description: |- - storedVersions lists all versions of CustomResources that were ever persisted. Tracking these - versions allows a migration path for stored versions in etcd. The field is mutable - so a migration controller can finish a migration to another version (ensuring - no old objects are left in storage), and then remove the rest of the - versions from this list. - Versions may not be removed from `spec.versions` while they exist in this list. - items: - type: string - type: array type: object required: - spec diff --git a/pkg/indexers/serviceexport.go b/pkg/indexers/serviceexport.go index ce76d97a3..40f5c0993 100644 --- a/pkg/indexers/serviceexport.go +++ b/pkg/indexers/serviceexport.go @@ -37,7 +37,7 @@ func IndexServiceExportByCustomResourceDefinition(obj any) ([]string, error) { } // IndexServiceExportByBoundSchema is a controller-runtime compatible indexer function. -func IndexServiceExportByBoundSchema(obj client.Object) []string { +func IndexServiceExportByBoundSchemaControllerRuntime(obj client.Object) []string { export, ok := obj.(*v1alpha2.APIServiceExport) if !ok { return nil @@ -47,6 +47,19 @@ func IndexServiceExportByBoundSchema(obj client.Object) []string { for _, res := range export.Spec.Resources { names = append(names, res.ResourceGroupName()) } - return names } + +// IndexServiceExportByBoundSchema is a controller-runtime compatible indexer function. +func IndexServiceExportByBoundSchema(obj any) ([]string, error) { + export, ok := obj.(*v1alpha2.APIServiceExport) + if !ok { + return nil, nil + } + + names := make([]string, 0, len(export.Spec.Resources)) + for _, res := range export.Spec.Resources { + names = append(names, res.ResourceGroupName()) + } + return names, nil +} diff --git a/pkg/indexers/servicenamespace.go b/pkg/indexers/servicenamespace.go index 80d6d559f..5cef0f0da 100644 --- a/pkg/indexers/servicenamespace.go +++ b/pkg/indexers/servicenamespace.go @@ -23,7 +23,7 @@ import ( ) const ( - ServiceNamespaceByNamespace = "ServiceNamespaceByNamespace" + ServiceNamespaceByNamespace = "serviceNamespaceByNamespace" ) func IndexServiceNamespaceByNamespace(obj any) ([]string, error) { @@ -31,6 +31,7 @@ func IndexServiceNamespaceByNamespace(obj any) ([]string, error) { if !ok || sn.Status.Namespace == "" { return nil, nil } + return []string{sn.Status.Namespace}, nil } diff --git a/pkg/konnector/controllers/cluster/claimedresources/claimedresources_controller.go b/pkg/konnector/controllers/cluster/claimedresources/claimedresources_controller.go new file mode 100644 index 000000000..d49f483ff --- /dev/null +++ b/pkg/konnector/controllers/cluster/claimedresources/claimedresources_controller.go @@ -0,0 +1,525 @@ +/* +Copyright 2022 The Kube Bind Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package claimedresources + +import ( + "context" + "fmt" + "time" + + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" + dynamicclient "k8s.io/client-go/dynamic" + "k8s.io/client-go/dynamic/dynamiclister" + "k8s.io/client-go/informers" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + + "github.com/kube-bind/kube-bind/pkg/indexers" + "github.com/kube-bind/kube-bind/pkg/konnector/controllers/cluster/serviceexport/multinsinformer" + "github.com/kube-bind/kube-bind/pkg/konnector/controllers/dynamic" + "github.com/kube-bind/kube-bind/pkg/resources" + kubebindv1alpha2 "github.com/kube-bind/kube-bind/sdk/apis/kubebind/v1alpha2" + bindclientset "github.com/kube-bind/kube-bind/sdk/client/clientset/versioned" + bindlisters "github.com/kube-bind/kube-bind/sdk/client/listers/kubebind/v1alpha2" +) + +const ( + controllerName = "kube-bind-konnector-claimed-object" +) + +// NewController returns a new controller reconciling downstream objects to upstream. +func NewController( + // scope kubebindv1alpha2.InformerScope, + gvr schema.GroupVersionResource, + claim kubebindv1alpha2.PermissionClaim, + apiServiceExport *kubebindv1alpha2.APIServiceExport, + providerNamespace string, + consumerConfig, providerConfig *rest.Config, + consumerDynamicInformer informers.GenericInformer, + providerDynamicInformer multinsinformer.GetterInformer, + serviceNamespaceInformer dynamic.Informer[bindlisters.APIServiceNamespaceLister], + namespaceCreationNotifyChan <-chan string, +) (*controller, error) { + queue := workqueue.NewTypedRateLimitingQueueWithConfig(workqueue.DefaultTypedControllerRateLimiter[string](), workqueue.TypedRateLimitingQueueConfig[string]{Name: controllerName}) + + logger := klog.Background().WithValues("controller", controllerName, "gvr", gvr) + + providerConfig = rest.CopyConfig(providerConfig) + providerConfig = rest.AddUserAgent(providerConfig, controllerName) + + consumerConfig = rest.CopyConfig(consumerConfig) + consumerConfig = rest.AddUserAgent(consumerConfig, controllerName) + + providerClient, err := dynamicclient.NewForConfig(providerConfig) + if err != nil { + return nil, err + } + consumerClient, err := dynamicclient.NewForConfig(consumerConfig) + if err != nil { + return nil, err + } + + bindClient, err := bindclientset.NewForConfig(providerConfig) + if err != nil { + return nil, err + } + + dynamicConsumerLister := dynamiclister.New(consumerDynamicInformer.Informer().GetIndexer(), gvr) + c := &controller{ + queue: queue, + + claim: claim, + apiServiceExport: apiServiceExport, // used to establish owner references when create happens on the provider side. TODO: Do we really need this? + + consumerClient: consumerClient, + providerClient: providerClient, + providerBindClient: bindClient, + + consumerDynamicLister: dynamicConsumerLister, + consumerDynamicIndexer: consumerDynamicInformer.Informer().GetIndexer(), + + providerDynamicInformer: providerDynamicInformer, + + serviceNamespaceInformer: serviceNamespaceInformer, + + providerNamespace: providerNamespace, + + namespaceCreationNotifyChan: namespaceCreationNotifyChan, + + readReconciler: readReconciler{ + getServiceNamespace: func(upstreamNamespace string) (*kubebindv1alpha2.APIServiceNamespace, error) { + sns, err := serviceNamespaceInformer.Informer().GetIndexer().ByIndex(indexers.ServiceNamespaceByNamespace, upstreamNamespace) + if err != nil { + return nil, err + } + if len(sns) == 0 { + return nil, errors.NewNotFound(kubebindv1alpha2.SchemeGroupVersion.WithResource("APIServiceNamespace").GroupResource(), upstreamNamespace) + } + return sns[0].(*kubebindv1alpha2.APIServiceNamespace), nil + }, + getConsumerObject: func(ctx context.Context, ns, name string) (*unstructured.Unstructured, error) { + return consumerClient.Resource(gvr).Namespace(ns).Get(ctx, name, metav1.GetOptions{}) + }, + getProviderObject: func(ns, name string) (*unstructured.Unstructured, error) { + obj, err := providerDynamicInformer.Get(ns, name) + if err != nil { + return nil, err + } + return obj.(*unstructured.Unstructured), nil + }, + createProviderObject: func(ctx context.Context, obj *unstructured.Unstructured) error { + _, err := providerClient.Resource(gvr).Namespace(obj.GetNamespace()).Create(ctx, obj, metav1.CreateOptions{}) + return err + }, + updateProviderObject: func(ctx context.Context, obj *unstructured.Unstructured) error { + _, err := providerClient.Resource(gvr).Namespace(obj.GetNamespace()).Update(ctx, obj, metav1.UpdateOptions{}) + return err + }, + deleteProviderObject: func(ctx context.Context, ns, name string) error { + return providerClient.Resource(gvr).Namespace(ns).Delete(ctx, name, metav1.DeleteOptions{}) + }, + deleteConsumerObject: func(ctx context.Context, ns, name string) error { + return consumerClient.Resource(gvr).Namespace(ns).Delete(ctx, name, metav1.DeleteOptions{}) + }, + updateConsumerObject: func(ctx context.Context, obj *unstructured.Unstructured) (*unstructured.Unstructured, error) { + return consumerClient.Resource(gvr).Namespace(obj.GetNamespace()).Update(ctx, obj, metav1.UpdateOptions{}) + }, + createConsumerObject: func(ctx context.Context, obj *unstructured.Unstructured) (*unstructured.Unstructured, error) { + return consumerClient.Resource(gvr).Namespace(obj.GetNamespace()).Create(ctx, obj, metav1.CreateOptions{}) + }, + }, + } + + if _, err = consumerDynamicInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + c.enqueueConsumer(logger, obj) + }, + UpdateFunc: func(_, newObj interface{}) { + c.enqueueConsumer(logger, newObj) + }, + DeleteFunc: func(obj interface{}) { + c.enqueueConsumer(logger, obj) + }, + }); err != nil { + return nil, err + } + + if err := providerDynamicInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + c.enqueueProvider(logger, obj) + }, + UpdateFunc: func(_, newObj interface{}) { + c.enqueueProvider(logger, newObj) + }, + DeleteFunc: func(obj interface{}) { + c.enqueueProvider(logger, obj) + }, + }); err != nil { + return nil, err + } + + return c, nil +} + +// controller reconciles upstream objects to downstream. +type controller struct { + queue workqueue.TypedRateLimitingInterface[string] + + claim kubebindv1alpha2.PermissionClaim + apiServiceExport *kubebindv1alpha2.APIServiceExport // used to establish owner references when create happens from the consumer side. + + consumerClient dynamicclient.Interface + providerClient dynamicclient.Interface + providerBindClient bindclientset.Interface + + consumerDynamicLister dynamiclister.Lister + consumerDynamicIndexer cache.Indexer + + providerDynamicInformer multinsinformer.GetterInformer + + serviceNamespaceInformer dynamic.Informer[bindlisters.APIServiceNamespaceLister] + + providerNamespace string + + // Channel to receive notifications when new namespaces are created + namespaceCreationNotifyChan <-chan string + + readReconciler +} + +func (c *controller) isClaimed(obj *unstructured.Unstructured) bool { + return resources.IsClaimed(c.claim.Selector, obj) +} + +func (c *controller) enqueueConsumer(logger klog.Logger, obj interface{}) { + // handle tombstones + if tombstone, ok := obj.(cache.DeletedFinalStateUnknown); ok { + obj = tombstone.Obj + } + o, ok := obj.(*unstructured.Unstructured) + if !ok { + runtime.HandleError(fmt.Errorf("unexpected type %T in enqueueConsumer", obj)) + return + } + if !c.isClaimed(o) { + return + } + logger.V(2).Info("queueing consumer object", "gvr", o.GroupVersionKind().String(), "key", fmt.Sprintf("%s/%s", o.GetNamespace(), o.GetName())) + + key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) + if err != nil { + runtime.HandleError(err) + return + } + + c.enqueueConsumerByKey(logger, key) +} + +// enqueueConsumerByKey will enqueue the given namespaced object. This is split from enqueueConsumer +// so we can accept channel requests from namespace creation watcher. +func (c *controller) enqueueConsumerByKey(logger klog.Logger, key string) { + ns, name, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + runtime.HandleError(err) + return + } + if ns != "" { // Namespaced object. + sn, err := c.serviceNamespaceInformer.Lister().APIServiceNamespaces(c.providerNamespace).Get(ns) + if err != nil { + if errors.IsNotFound(err) { + // No APIServiceNamespace - means claimedresourcesnamespaces controller should create it + logger.V(2).Info("no APIServiceNamespace found for claimed object. Will retry", "namespace", ns, "object", name) + return + } + runtime.HandleError(fmt.Errorf("failed to get APIServiceNamespace %q: %w", ns, err)) + return + } + if sn.Status.Namespace == "" { + return // not ready yet + } + if sn.Namespace == c.providerNamespace && sn.Status.Namespace != "" { + key := fmt.Sprintf("%s/%s", sn.Status.Namespace, name) + logger.V(2).Info("queueing Unstructured", "key", key, "reason", "Consumer") + c.queue.Add(key) + return + } + return + } + + logger.V(2).Info("queueing Unstructured", "key", key, "reason", "Consumer") + c.queue.Add(key) +} + +func (c *controller) enqueueProvider(logger klog.Logger, obj interface{}) { + // handle tombstones + if tombstone, ok := obj.(cache.DeletedFinalStateUnknown); ok { + obj = tombstone.Obj + } + o, ok := obj.(*unstructured.Unstructured) + if !ok { + runtime.HandleError(fmt.Errorf("unexpected type %T in enqueueProvider", obj)) + return + } + if !c.isClaimed(o) { + return + } + + providerKey, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) + if err != nil { + runtime.HandleError(err) + return + } + ns, _, err := cache.SplitMetaNamespaceKey(providerKey) + if err != nil { + runtime.HandleError(err) + return + } + + if ns != "" { + sns, err := c.serviceNamespaceInformer.Informer().GetIndexer().ByIndex(indexers.ServiceNamespaceByNamespace, ns) + if err != nil { + if !errors.IsNotFound(err) { + runtime.HandleError(err) + } + return + } + for _, obj := range sns { + sn := obj.(*kubebindv1alpha2.APIServiceNamespace) + if sn.Namespace == c.providerNamespace { + logger.V(2).Info("queueing Unstructured", "key", providerKey) + c.queue.Add(providerKey) + return + } + } + return + } + + logger.V(2).Info("queueing Unstructured", "key", providerKey) + c.queue.Add(providerKey) +} + +func (c *controller) enqueueServiceNamespace(logger klog.Logger, obj interface{}) { + snKey, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) + if err != nil { + runtime.HandleError(err) + return + } + ns, name, err := cache.SplitMetaNamespaceKey(snKey) + if err != nil { + runtime.HandleError(err) + return + } + if ns != c.providerNamespace { + return // not for us + } + + sn, err := c.serviceNamespaceInformer.Lister().APIServiceNamespaces(ns).Get(name) + if err != nil { + runtime.HandleError(err) + return + } + + if sn.Status.Namespace == "" { + return // not ready + } + + logger.Info("enqueueing service namespace", "upstreamNamespace", sn.Status.Namespace) + objs, err := c.providerDynamicInformer.List(sn.Status.Namespace) + if err != nil { + runtime.HandleError(err) + return + } + for _, obj := range objs { + unstructuredObj, ok := obj.(*unstructured.Unstructured) + if !ok { + continue + } + + // Check if the provider object is actually claimed using the full selector logic. + if !c.isClaimed(unstructuredObj) { + continue + } + + key, err := cache.MetaNamespaceKeyFunc(obj) + if err != nil { + runtime.HandleError(err) + continue + } + logger.V(2).Info("queueing Unstructured", "key", key, "reason", "APIServiceNamespace", "ServiceNamespaceKey", key) + c.queue.Add(key) + } + + // We need to list all the object which might not got synced at consumer side too: + var sel labels.Selector + switch v := c.claim.Selector; { + case v.LabelSelector == nil && len(v.NamedResource) == 0: + sel = labels.Everything() + case v.LabelSelector != nil: + var err error + sel, err = metav1.LabelSelectorAsSelector(v.LabelSelector) + if err != nil { + runtime.HandleError(err) + return + } + case len(v.NamedResource) > 0: + // namedResource-only: fetch specific objects from cache and enqueue + for _, nr := range v.NamedResource { + // Build consumer cache key; empty namespace implies cluster-scoped + key := nr.Name + if nr.Namespace != "" { + key = fmt.Sprintf("%s/%s", nr.Namespace, nr.Name) + } + // only process objects in this consumer namespace + if nr.Namespace != "" && nr.Namespace != sn.Name { + continue + } + obj, exists, err := c.consumerDynamicIndexer.GetByKey(key) + if err != nil { + runtime.HandleError(err) + continue + } + if !exists { + continue + } + u, ok := obj.(*unstructured.Unstructured) + if !ok || !c.isClaimed(u) { + continue + } + // Re-map to provider namespace key + provKey := fmt.Sprintf("%s/%s", sn.Status.Namespace, nr.Name) + logger.V(2).Info("queueing Unstructured", "key", provKey, "reason", "APIServiceNamespace", "ConsumerObject", key) + c.queue.Add(provKey) + } + return + default: + return // nothing is selected + } + objects, err := c.consumerDynamicLister.List(sel) + if err != nil { + runtime.HandleError(err) + return + } + for _, obj := range objects { + // Check if the object is actually claimed using the full selector logic + if !c.isClaimed(obj) { + continue + } + objKey, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) + if err != nil { + runtime.HandleError(err) + continue + } + objNamespace, name, err := cache.SplitMetaNamespaceKey(objKey) + if err != nil { + runtime.HandleError(err) + continue + } + // only process objects in this consumer namespace + if objNamespace != "" && objNamespace != sn.Name { + continue + } + provKey := fmt.Sprintf("%s/%s", sn.Status.Namespace, name) + logger.V(2).Info("queueing Unstructured", "key", provKey, "reason", "APIServiceNamespace", "ConsumerObject", objKey) + c.queue.Add(provKey) + } +} + +// Start starts the controller, which stops when ctx.Done() is closed. +func (c *controller) Start(ctx context.Context, numThreads int) { + defer runtime.HandleCrash() + defer c.queue.ShutDown() + + logger := klog.FromContext(ctx).WithValues("controller", controllerName) + + logger.Info("Starting controller") + defer logger.Info("Shutting down controller") + + c.serviceNamespaceInformer.Informer().AddDynamicEventHandler(ctx, controllerName, cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + c.enqueueServiceNamespace(logger, obj) + }, + UpdateFunc: func(_, newObj interface{}) { + c.enqueueServiceNamespace(logger, newObj) + }, + DeleteFunc: func(obj interface{}) { + c.enqueueServiceNamespace(logger, obj) + }, + }) + + // Start a goroutine to listen for namespace creation notifications. + go func() { + for key := range c.namespaceCreationNotifyChan { + logger.Info("received namespace creation notification", "namespace", key) + c.enqueueConsumerByKey(logger, key) + } + }() + + for i := 0; i < numThreads; i++ { + go wait.UntilWithContext(ctx, c.startWorker, time.Second) + } + + <-ctx.Done() +} + +func (c *controller) startWorker(ctx context.Context) { + defer runtime.HandleCrash() + + for c.processNextWorkItem(ctx) { + } +} + +func (c *controller) processNextWorkItem(ctx context.Context) bool { + // Wait until there is a new item in the working queue + key, quit := c.queue.Get() + if quit { + return false + } + + logger := klog.FromContext(ctx).WithValues("key", key) + ctx = klog.NewContext(ctx, logger) + logger.Info("processing key") + + // No matter what, tell the queue we're done with this key, to unblock + // other workers. + defer c.queue.Done(key) + + if err := c.process(ctx, key); err != nil { + runtime.HandleError(fmt.Errorf("%q controller failed to sync %q, err: %w", controllerName, key, err)) + c.queue.AddRateLimited(key) + return true + } + c.queue.Forget(key) + return true +} + +func (c *controller) process(ctx context.Context, key string) error { + namespace, name, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + runtime.HandleError(err) + return nil // we cannot do anything + } + + return c.reconcile(ctx, namespace, name) +} diff --git a/pkg/konnector/controllers/cluster/claimedresources/claimedresources_reconciler.go b/pkg/konnector/controllers/cluster/claimedresources/claimedresources_reconciler.go new file mode 100644 index 000000000..fc549305c --- /dev/null +++ b/pkg/konnector/controllers/cluster/claimedresources/claimedresources_reconciler.go @@ -0,0 +1,265 @@ +/* +Copyright 2023 The Kube Bind Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package claimedresources + +import ( + "context" + "fmt" + "strings" + + "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/api/errors" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/klog/v2" + + kubebindv1alpha2 "github.com/kube-bind/kube-bind/sdk/apis/kubebind/v1alpha2" +) + +const label = "kube-bind.io/owner" + +type readReconciler struct { + getServiceNamespace func(upstreamNamespace string) (*kubebindv1alpha2.APIServiceNamespace, error) + getProviderObject func(ns, name string) (*unstructured.Unstructured, error) + createProviderObject func(ctx context.Context, obj *unstructured.Unstructured) error + updateProviderObject func(ctx context.Context, obj *unstructured.Unstructured) error + deleteProviderObject func(ctx context.Context, ns, name string) error + + getConsumerObject func(ctx context.Context, ns, name string) (*unstructured.Unstructured, error) + updateConsumerObject func(ctx context.Context, obj *unstructured.Unstructured) (*unstructured.Unstructured, error) + createConsumerObject func(ctx context.Context, ob *unstructured.Unstructured) (*unstructured.Unstructured, error) + deleteConsumerObject func(ctx context.Context, ns, name string) error +} + +// reconcile syncs consumer claimed resources to provider. +func (r *readReconciler) reconcile(ctx context.Context, providerNamespace, name string) error { + logger := klog.FromContext(ctx) + logger = logger.WithValues("name", name, "providerNamespace", providerNamespace) + + logger.Info("reconciling object") + consumerNS := "" + if providerNamespace != "" { + sn, err := r.getServiceNamespace(providerNamespace) + if err != nil && !errors.IsNotFound(err) { + return err + } else if errors.IsNotFound(err) { + runtime.HandleError(err) + return err // hoping the APIServiceNamespace will be created soon. Otherwise, this item goes into backoff. + } + if sn.Status.Namespace == "" { + e := fmt.Errorf("APIServiceNamespace %q has empty status.namespace", providerNamespace) + runtime.HandleError(e) + return e // hoping the status is set soon. + } + + logger = logger.WithValues("providerNamespace", sn.Status.Namespace) + consumerNS = sn.Name + logger = logger.WithValues("consumerNamespace", consumerNS) + + ctx = klog.NewContext(ctx, logger) + } + + providerObj, providerErr := r.getProviderObject(providerNamespace, name) + if providerErr != nil && !errors.IsNotFound(providerErr) { + return providerErr + } + consumerObj, consumerErr := r.getConsumerObject(ctx, consumerNS, name) + if consumerErr != nil && !errors.IsNotFound(consumerErr) { + return consumerErr + } + + if errors.IsNotFound(providerErr) && errors.IsNotFound(consumerErr) { + // Nothing to do + return nil + } + + // Determine owner + owner, err := determineOwner(providerObj, consumerObj) + if err != nil { // nothing we can do + logger.Error(err, "could not determine owner") + return nil + } + logger = logger.WithValues("owner", owner) + + switch owner { + case kubebindv1alpha2.OwnerProvider: + if errors.IsNotFound(providerErr) { + err := r.deleteConsumerObject(ctx, consumerNS, name) + if errors.IsNotFound(err) { + return nil + } + return err + } + ownerCandidate := providerObj.DeepCopy() + + // Set owner label if needed + r.makeProviderOwner(ownerCandidate) + if !equality.Semantic.DeepEqual(providerObj, ownerCandidate) { + if err := r.updateProviderObject(ctx, ownerCandidate); err != nil { + return err + } + } + + if errors.IsNotFound(consumerErr) { + logger.Info("Creating missing consumer object", "consumerNamespace", consumerNS, "consumerName", providerObj.GetName()) + + candidate := candidateFromOwnerObj(consumerNS, providerObj) + r.makeProviderOwner(candidate) + + if _, err := r.createConsumerObject(ctx, candidate); err != nil { + return err + } + + return nil + } + + if !providerObj.GetDeletionTimestamp().IsZero() { + logger.Info("Deleting consumer object because it has been deleted upstream", "consumerNamespace", consumerNS, "consumerName", providerObj.GetName()) + return r.deleteConsumerObject(ctx, consumerNS, providerObj.GetName()) + } + + candidate := candidateFromOwnerObj(consumerNS, providerObj) + current := candidateFromOwnerObj(consumerNS, consumerObj) + if !equality.Semantic.DeepEqual(candidate, current) { + logger.Info("Updating consumer object data", "consumerNamespace", consumerNS, "consumerName", consumerObj.GetName()) + if _, err := r.updateConsumerObject(ctx, candidate); err != nil { + logger.Error(err, "error updating consumer object") + return err + } + } + + case kubebindv1alpha2.OwnerConsumer: + if errors.IsNotFound(consumerErr) { + logger.Info("Owner copy of the object is gone, deleting downstream object", "name", name, "namespace", providerNamespace) + err := r.deleteProviderObject(ctx, providerNamespace, name) + if errors.IsNotFound(err) { + return nil + } + return err + } + + ownerCandidate := consumerObj.DeepCopy() + r.makeConsumerOwner(ownerCandidate) + if !equality.Semantic.DeepEqual(consumerObj, ownerCandidate) { + logger.Info("setting owner annotation for Consumer object") + if _, err := r.updateConsumerObject(ctx, ownerCandidate); err != nil { + return err + } + } + + candidate := candidateFromOwnerObj(providerNamespace, ownerCandidate) + r.makeConsumerOwner(candidate) + + if errors.IsNotFound(providerErr) { + logger.Info("creating consumer owned object at provider") + return r.createProviderObject(ctx, candidate) + } + + providerObj := candidateFromOwnerObj(providerNamespace, providerObj) + if !equality.Semantic.DeepEqual(providerObj, candidate) { + logger.Info("updating consumer owned object at provider") + return r.updateProviderObject(ctx, candidate) + } + } + + return nil +} + +func (r readReconciler) makeConsumerOwner(obj *unstructured.Unstructured) { + a := obj.GetLabels() + if a == nil { + a = map[string]string{} + } + a[label] = string(kubebindv1alpha2.OwnerConsumer) + obj.SetLabels(a) +} + +func (r readReconciler) makeProviderOwner(obj *unstructured.Unstructured) { + a := obj.GetLabels() + if a == nil { + a = map[string]string{} + } + a[label] = string(kubebindv1alpha2.OwnerProvider) + obj.SetLabels(a) +} + +func candidateFromOwnerObj(downstreamNS string, obj *unstructured.Unstructured) *unstructured.Unstructured { + // clean up object + candidate := obj.DeepCopy() + candidate.SetUID("") + candidate.SetResourceVersion("") + candidate.SetNamespace(downstreamNS) + candidate.SetManagedFields(nil) + candidate.SetDeletionTimestamp(nil) + candidate.SetDeletionGracePeriodSeconds(nil) + candidate.SetOwnerReferences(nil) + candidate.SetFinalizers(nil) + candidate.SetNamespace(downstreamNS) + candidate.SetCreationTimestamp(v1.Time{}) + + labels := map[string]string{} + for key, label := range obj.GetLabels() { + if strings.Contains(key, "claimed.internal.apis.kcp.io") { + continue + } + labels[key] = label + } + candidate.SetLabels(labels) + + annotations := map[string]string{} + for key, annotation := range obj.GetAnnotations() { + if strings.Contains(key, "kcp.io/cluster") { + continue + } + annotations[key] = annotation + } + candidate.SetAnnotations(annotations) + + return candidate +} + +// determineOwner determines the owner of a resource given at least one object exists either on the +// consumer or provider side. +func determineOwner(providerObj, consumerObj *unstructured.Unstructured) (kubebindv1alpha2.Owner, error) { + if providerObj != nil { + ownerLabel := providerObj.GetLabels()[label] + switch ownerLabel { + case kubebindv1alpha2.OwnerProvider.String(): + return kubebindv1alpha2.OwnerProvider, nil + case kubebindv1alpha2.OwnerConsumer.String(): + return kubebindv1alpha2.OwnerConsumer, nil + } + if ownerLabel == "" && consumerObj == nil { + return kubebindv1alpha2.OwnerProvider, nil + } + } + + if consumerObj != nil { + ownerLabel := consumerObj.GetLabels()[label] + switch ownerLabel { + case kubebindv1alpha2.OwnerProvider.String(): + return kubebindv1alpha2.OwnerProvider, nil + case kubebindv1alpha2.OwnerConsumer.String(): + return kubebindv1alpha2.OwnerConsumer, nil + } + if ownerLabel == "" && providerObj == nil { + return kubebindv1alpha2.OwnerConsumer, nil + } + } + return "", fmt.Errorf("unable to determine owner") +} diff --git a/pkg/konnector/controllers/cluster/claimedresourcesnamespaces/README.md b/pkg/konnector/controllers/cluster/claimedresourcesnamespaces/README.md new file mode 100644 index 000000000..960cd02e0 --- /dev/null +++ b/pkg/konnector/controllers/cluster/claimedresourcesnamespaces/README.md @@ -0,0 +1,13 @@ +# claimedresourcesnamespaces + +When using claimed resources, it can be situations where resource does not yet exists in the consumer cluster OR +we are operating in cluster scoped mode, and hence no resource namespace is created. In scenarios like this, +`APIServiceNamespace` object is never creates, and hence backend will not create appropriate RBAC for the consumer to be able to +access the resources. + +It will watch same GVR resources, as claimed resources (hence the name of the controller), and will create `APIServiceNamespace` objects, +if object is created in the consumer cluster. + +We can't add this logic into `claimedresource` controller, as it never gets to healthy state until provider has right rbac configured and +consumer can access the resources and hence start the informers. Alternative to this is create `APIServiceNamespace` objects outside reconile +loop, but that would be against the controller pattern. \ No newline at end of file diff --git a/pkg/konnector/controllers/cluster/claimedresourcesnamespaces/claimedresourcesnamespaces_controller.go b/pkg/konnector/controllers/cluster/claimedresourcesnamespaces/claimedresourcesnamespaces_controller.go new file mode 100644 index 000000000..f2ff0948a --- /dev/null +++ b/pkg/konnector/controllers/cluster/claimedresourcesnamespaces/claimedresourcesnamespaces_controller.go @@ -0,0 +1,288 @@ +/* +Copyright 2025 The Kube Bind Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package claimedresourcesnamespaces + +import ( + "context" + "fmt" + "time" + + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/informers" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + + "github.com/kube-bind/kube-bind/pkg/konnector/controllers/dynamic" + "github.com/kube-bind/kube-bind/pkg/resources" + kubebindv1alpha2 "github.com/kube-bind/kube-bind/sdk/apis/kubebind/v1alpha2" + bindclientset "github.com/kube-bind/kube-bind/sdk/client/clientset/versioned" + bindlisters "github.com/kube-bind/kube-bind/sdk/client/listers/kubebind/v1alpha2" +) + +const ( + controllerName = "kube-bind-konnector-claimedresourcesnamespaces" +) + +// NewController returns a new controller ensuring APIServiceNamespaces exist for claimed objects. +func NewController( + gvr schema.GroupVersionResource, + claim kubebindv1alpha2.PermissionClaim, + apiServiceExport *kubebindv1alpha2.APIServiceExport, + providerNamespace string, + providerConfig *rest.Config, + consumerDynamicInformer informers.GenericInformer, + serviceNamespaceInformer dynamic.Informer[bindlisters.APIServiceNamespaceLister], + claimedResourcesEnqueueChan chan<- string, +) (*controller, error) { + queue := workqueue.NewTypedRateLimitingQueueWithConfig(workqueue.DefaultTypedControllerRateLimiter[string](), workqueue.TypedRateLimitingQueueConfig[string]{Name: controllerName}) + + logger := klog.Background().WithValues("controller", controllerName, "gvr", gvr) + + providerConfig = rest.CopyConfig(providerConfig) + providerConfig = rest.AddUserAgent(providerConfig, controllerName) + + bindClient, err := bindclientset.NewForConfig(providerConfig) + if err != nil { + return nil, err + } + c := &controller{ + queue: queue, + claim: claim, + apiServiceExport: apiServiceExport, + + providerBindClient: bindClient, + + consumerDynamicInformer: consumerDynamicInformer, + + serviceNamespaceInformer: serviceNamespaceInformer, + + providerNamespace: providerNamespace, + + claimedResourcesEnqueueChan: claimedResourcesEnqueueChan, + } + + // Only watch for ADD events on consumer side to detect when claimed objects are created + if _, err = consumerDynamicInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + c.enqueueConsumer(logger, obj) + }, + }); err != nil { + return nil, err + } + + return c, nil +} + +// controller ensures APIServiceNamespaces exist for claimed objects. +type controller struct { + queue workqueue.TypedRateLimitingInterface[string] + + claim kubebindv1alpha2.PermissionClaim + apiServiceExport *kubebindv1alpha2.APIServiceExport + + providerBindClient bindclientset.Interface + + consumerDynamicInformer informers.GenericInformer + + serviceNamespaceInformer dynamic.Informer[bindlisters.APIServiceNamespaceLister] + + providerNamespace string + + // Channel to notify claimedresources controller when new namespaces are created + // so we don't have to wait for the next resync period. + claimedResourcesEnqueueChan chan<- string +} + +func (c *controller) isClaimed(obj *unstructured.Unstructured) bool { + return resources.IsClaimed(c.claim.Selector, obj) +} + +func (c *controller) enqueueConsumer(logger klog.Logger, obj interface{}) { + // handle tombstones + if tombstone, ok := obj.(cache.DeletedFinalStateUnknown); ok { + obj = tombstone.Obj + } + o, ok := obj.(*unstructured.Unstructured) + if !ok { + runtime.HandleError(fmt.Errorf("unexpected type %T in enqueueConsumer", obj)) + return + } + if !c.isClaimed(o) { + return + } + + ns := o.GetNamespace() + if ns == "" { + return // cluster-scoped objects don't need namespaces + } + + logger.V(2).Info("checking APIServiceNamespace for claimed object", "namespace", ns, "object", o.GetName()) + + // Check if APIServiceNamespace already exists + _, err := c.serviceNamespaceInformer.Lister().APIServiceNamespaces(c.providerNamespace).Get(ns) + if err != nil { + if errors.IsNotFound(err) { + // No namespace - queue ServiceNamespace creation + key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) + if err != nil { + runtime.HandleError(err) + return + } + logger.V(2).Info("queueing ServiceNamespace creation", "key", key) + c.queue.Add(key) + return + } + runtime.HandleError(fmt.Errorf("failed to get APIServiceNamespace %q: %w", ns, err)) + return + } + // APIServiceNamespace already exists, nothing to do +} + +// Start starts the controller, which stops when ctx.Done() is closed. +func (c *controller) Start(ctx context.Context, numThreads int) { + defer runtime.HandleCrash() + defer c.queue.ShutDown() + + logger := klog.FromContext(ctx).WithValues("controller", controllerName) + + logger.Info("Starting controller") + defer logger.Info("Shutting down controller") + + for i := 0; i < numThreads; i++ { + go wait.UntilWithContext(ctx, c.startWorker, time.Second) + } + + <-ctx.Done() +} + +func (c *controller) startWorker(ctx context.Context) { + defer runtime.HandleCrash() + + for c.processNextWorkItem(ctx) { + } +} + +func (c *controller) processNextWorkItem(ctx context.Context) bool { + // Wait until there is a new item in the working queue + key, quit := c.queue.Get() + if quit { + return false + } + + logger := klog.FromContext(ctx).WithValues("key", key) + ctx = klog.NewContext(ctx, logger) + logger.Info("processing key") + + // No matter what, tell the queue we're done with this key, to unblock + // other workers. + defer c.queue.Done(key) + + if err := c.process(ctx, key); err != nil { + runtime.HandleError(fmt.Errorf("%q controller failed to sync %q, err: %w", controllerName, key, err)) + c.queue.AddRateLimited(key) + return true + } + c.queue.Forget(key) + return true +} + +func (c *controller) process(ctx context.Context, key string) error { + ns, _, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + runtime.HandleError(err) + return nil + } + if ns == "" { + return nil // we only handle namespaced objects + } + + created, err := c.ensureServiceNamespace(ctx, ns) + if err != nil { + return err + } + if created { + // Notify claimedresources controller that a new namespace was created + select { + case c.claimedResourcesEnqueueChan <- key: + default: + // Don't block if the channel is full + } + } + return nil +} + +func (c *controller) createServiceNamespace(ctx context.Context, sns *kubebindv1alpha2.APIServiceNamespace) (*kubebindv1alpha2.APIServiceNamespace, error) { + return c.providerBindClient.KubeBindV1alpha2().APIServiceNamespaces(sns.Namespace).Create(ctx, sns, metav1.CreateOptions{}) +} + +// ensureServiceNamespace ensures that the APIServiceNamespace for the given object namespace exists. +func (c *controller) ensureServiceNamespace(ctx context.Context, ns string) (created bool, err error) { + logger := klog.FromContext(ctx).WithValues("namespace", ns) + logger.Info("ensuring APIServiceNamespace exists", "providerNamespace", c.providerNamespace, "namespace", ns) + + _, err = c.serviceNamespaceInformer.Lister().APIServiceNamespaces(c.providerNamespace).Get(ns) + if err == nil { + logger.V(2).Info("APIServiceNamespace already exists") + return false, nil // already exists + } + if !errors.IsNotFound(err) { + return false, err + } + + logger.Info("creating APIServiceNamespace") + _, err = c.createServiceNamespace(ctx, &kubebindv1alpha2.APIServiceNamespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: ns, + Namespace: c.providerNamespace, + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(c.apiServiceExport, kubebindv1alpha2.SchemeGroupVersion.WithKind("APIServiceExport")), + }, + }, + }) + if err != nil && !errors.IsAlreadyExists(err) { + return false, fmt.Errorf("failed to create APIServiceNamespace %q: %w", ns, err) + } + + // Wait until the servicenamespace status has been updated. + // As this controller does not do anything else, we are ok to block here. + err = wait.PollUntilContextCancel(ctx, 500*time.Millisecond, true, func(ctx context.Context) (done bool, err error) { + sns, err := c.serviceNamespaceInformer.Lister().APIServiceNamespaces(c.providerNamespace).Get(ns) + if err != nil { + if errors.IsNotFound(err) { + return false, nil // keep polling + } + return false, err + } + if sns.Status.Namespace != "" { + return true, nil + } + return false, nil + }) + if err != nil { + return false, fmt.Errorf("failed to wait for APIServiceNamespace %q to be ready: %w", ns, err) + } + + logger.Info("APIServiceNamespace created successfully") + return true, nil +} diff --git a/pkg/konnector/controllers/cluster/servicebinding/servicebinding_reconcile.go b/pkg/konnector/controllers/cluster/servicebinding/servicebinding_reconcile.go index b94c64f43..8a842124f 100644 --- a/pkg/konnector/controllers/cluster/servicebinding/servicebinding_reconcile.go +++ b/pkg/konnector/controllers/cluster/servicebinding/servicebinding_reconcile.go @@ -116,7 +116,7 @@ func (r *reconciler) ensureValidServiceExport(_ context.Context, binding *kubebi func (r *reconciler) ensureCRDs(ctx context.Context, binding *kubebindv1alpha2.APIServiceBinding) error { var errs []error - export, err := r.getServiceExport(binding.Name) + exportOriginal, err := r.getServiceExport(binding.Name) if err != nil && !errors.IsNotFound(err) { return err } else if errors.IsNotFound(err) { @@ -131,6 +131,8 @@ func (r *reconciler) ensureCRDs(ctx context.Context, binding *kubebindv1alpha2.A return nil // nothing we can do here } + export := exportOriginal.DeepCopy() + // Get all BoundSchema objects referenced by the export schemas, err := r.getSchemasFromExport(ctx, export) if err != nil { @@ -152,6 +154,10 @@ func (r *reconciler) ensureCRDs(ctx context.Context, binding *kubebindv1alpha2.A errs = append(errs, err) } + if err := r.referencePermissionClaims(ctx, binding, export); err != nil { + errs = append(errs, err) + } + if err := r.ensureCRDsFromBoundSchema(ctx, binding, schema); err != nil { errs = append(errs, err) } @@ -202,6 +208,11 @@ func (r *reconciler) referenceBoundSchema(ctx context.Context, binding *kubebind return nil } +func (r *reconciler) referencePermissionClaims(ctx context.Context, binding *kubebindv1alpha2.APIServiceBinding, export *kubebindv1alpha2.APIServiceExport) error { + binding.Status.PermissionClaims = export.Spec.PermissionClaims + return nil +} + func (r *reconciler) ensureCRDsFromBoundSchema(ctx context.Context, binding *kubebindv1alpha2.APIServiceBinding, schema *kubebindv1alpha2.BoundSchema) error { var errs []error crd := kubebindhelpers.BoundSchemaToCRD(schema) @@ -283,11 +294,10 @@ func (r *reconciler) ensurePrettyName(ctx context.Context, binding *kubebindv1al func (r *reconciler) getSchemasFromExport(ctx context.Context, export *kubebindv1alpha2.APIServiceExport) ([]*kubebindv1alpha2.BoundSchema, error) { schemas := make([]*kubebindv1alpha2.BoundSchema, 0, len(export.Spec.Resources)) - for _, res := range export.Spec.Resources { - schema, err := r.getBoundSchema(ctx, res.ResourceGroupName()) + for _, ref := range export.Spec.Resources { + schema, err := r.getBoundSchema(ctx, ref.ResourceGroupName()) if err != nil { - return nil, fmt.Errorf("failed to get Schema %s: %w", - res.ResourceGroupName(), err) + return nil, fmt.Errorf("failed to get Schema %s: %w", ref.ResourceGroupName(), err) } schemas = append(schemas, schema) diff --git a/pkg/konnector/controllers/cluster/serviceexport/multinsinformer/informer.go b/pkg/konnector/controllers/cluster/serviceexport/multinsinformer/informer.go index 3a2d92d6a..c41587f75 100644 --- a/pkg/konnector/controllers/cluster/serviceexport/multinsinformer/informer.go +++ b/pkg/konnector/controllers/cluster/serviceexport/multinsinformer/informer.go @@ -23,6 +23,7 @@ import ( "time" "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -60,6 +61,7 @@ var _ GetterInformer = &DynamicMultiNamespaceInformer{} // by starting individual informers per namespace and aggregating all of these. type DynamicMultiNamespaceInformer struct { gvr schema.GroupVersionResource + labelSelector *metav1.LabelSelector providerNamespace string providerDynamicClient dynamicclient.Interface @@ -77,6 +79,7 @@ func NewDynamicMultiNamespaceInformer( providerNamespace string, providerConfig *rest.Config, serviceNamespaceInformer dynamic.Informer[bindlisters.APIServiceNamespaceLister], + labelSelector *metav1.LabelSelector, ) (*DynamicMultiNamespaceInformer, error) { providerConfig = rest.CopyConfig(providerConfig) providerConfig = rest.AddUserAgent(providerConfig, controllerName) @@ -88,6 +91,7 @@ func NewDynamicMultiNamespaceInformer( inf := DynamicMultiNamespaceInformer{ gvr: gvr, + labelSelector: labelSelector, providerNamespace: providerNamespace, providerDynamicClient: providerDynamicClient, serviceNamespaceInformer: serviceNamespaceInformer, @@ -184,7 +188,20 @@ func (inf *DynamicMultiNamespaceInformer) enqueueServiceNamespace(obj any) { logger.V(1).Info("starting dynamic informer", "namespace", sns.Status.Namespace) ctx, cancel := context.WithCancel(context.Background()) - factory := dynamicinformer.NewFilteredDynamicSharedInformerFactory(inf.providerDynamicClient, time.Minute*30, sns.Status.Namespace, nil) + + var tweakListOptions func(options *metav1.ListOptions) + if inf.labelSelector != nil { + tweakListOptions = func(options *metav1.ListOptions) { + selector, err := metav1.LabelSelectorAsSelector(inf.labelSelector) + if err != nil { + utilruntime.HandleError(fmt.Errorf("failed to convert label selector: %w", err)) + return + } + options.LabelSelector = selector.String() + } + } + + factory := dynamicinformer.NewFilteredDynamicSharedInformerFactory(inf.providerDynamicClient, time.Minute*30, sns.Status.Namespace, tweakListOptions) gvrInf := factory.ForResource(inf.gvr) gvrInf.Lister() // to wire the GVR up in the informer factory inf.namespaceCancel[name] = cancel diff --git a/pkg/konnector/controllers/cluster/serviceexport/serviceexport_controller.go b/pkg/konnector/controllers/cluster/serviceexport/serviceexport_controller.go index a5be2edd6..d83470dc5 100644 --- a/pkg/konnector/controllers/cluster/serviceexport/serviceexport_controller.go +++ b/pkg/konnector/controllers/cluster/serviceexport/serviceexport_controller.go @@ -35,6 +35,7 @@ import ( "github.com/kube-bind/kube-bind/pkg/committer" "github.com/kube-bind/kube-bind/pkg/indexers" + "github.com/kube-bind/kube-bind/pkg/konnector/controllers/contextstore" "github.com/kube-bind/kube-bind/pkg/konnector/controllers/dynamic" kubebindv1alpha2 "github.com/kube-bind/kube-bind/sdk/apis/kubebind/v1alpha2" bindclient "github.com/kube-bind/kube-bind/sdk/client/clientset/versioned" @@ -75,15 +76,17 @@ func NewController( if err != nil { return nil, err } + + indexers.AddIfNotPresentOrDie(dynamicServiceNamespaceInformer.Informer().GetIndexer(), cache.Indexers{ + indexers.ServiceNamespaceByNamespace: indexers.IndexServiceNamespaceByNamespace, + }) + c := &controller{ queue: queue, serviceExportLister: serviceExportInformer.Lister(), serviceExportIndexer: serviceExportInformer.Informer().GetIndexer(), - serviceNamespaceLister: serviceNamespaceInformer.Lister(), - serviceNamespaceIndexer: serviceNamespaceInformer.Informer().GetIndexer(), - serviceBindingInformer: serviceBindingInformer, crdInformer: crdInformer, @@ -94,7 +97,7 @@ func NewController( consumerConfig: consumerConfig, providerConfig: providerConfig, - syncContext: map[string]syncContext{}, + syncStore: contextstore.New(), getServiceBinding: func(name string) (*kubebindv1alpha2.APIServiceBinding, error) { return serviceBindingInformer.Lister().Get(name) @@ -118,8 +121,8 @@ func NewController( ), } - indexers.AddIfNotPresentOrDie(serviceNamespaceInformer.Informer().GetIndexer(), cache.Indexers{ - indexers.ServiceNamespaceByNamespace: indexers.IndexServiceNamespaceByNamespace, + indexers.AddIfNotPresentOrDie(serviceExportInformer.Informer().GetIndexer(), cache.Indexers{ + indexers.ServiceExportByBoundSchema: indexers.IndexServiceExportByBoundSchema, }) if _, err := serviceExportInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ @@ -149,9 +152,6 @@ type controller struct { serviceExportLister bindlisters.APIServiceExportLister serviceExportIndexer cache.Indexer - serviceNamespaceLister bindlisters.APIServiceNamespaceLister - serviceNamespaceIndexer cache.Indexer - serviceBindingInformer dynamic.Informer[bindlisters.APIServiceBindingLister] crdInformer dynamic.Informer[apiextensionslisters.CustomResourceDefinitionLister] @@ -184,15 +184,28 @@ func (c *controller) enqueueServiceBinding(logger klog.Logger, obj any) { } func (c *controller) enqueueCRD(logger klog.Logger, obj any) { - crdKey, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) + name, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) if err != nil { runtime.HandleError(err) return } - key := c.providerNamespace + "/" + crdKey - logger.V(2).Info("queueing APIServiceExport", "key", key, "reason", "APIServiceExport", "APIServiceExportKey", crdKey) - c.queue.Add(key) + exports, err := c.serviceExportIndexer.ByIndex(indexers.ServiceExportByBoundSchema, name) + if err != nil { + runtime.HandleError(err) + return + } + + for _, obj := range exports { + export := obj.(*kubebindv1alpha2.APIServiceExport) + key, err := cache.MetaNamespaceKeyFunc(export) + if err != nil { + runtime.HandleError(err) + return + } + logger.V(2).Info("queueing APIServiceExport", "key", key, "reason", "CustomResourceDefinition", "name", name) + c.queue.Add(key) + } } // Start starts the controller, which stops when ctx.Done() is closed. @@ -268,7 +281,7 @@ func (c *controller) processNextWorkItem(ctx context.Context) bool { } func (c *controller) process(ctx context.Context, key string) error { - ns, name, err := cache.SplitMetaNamespaceKey(key) + namespace, name, err := cache.SplitMetaNamespaceKey(key) if err != nil { runtime.HandleError(err) return nil // we cannot do anything @@ -276,19 +289,19 @@ func (c *controller) process(ctx context.Context, key string) error { logger := klog.FromContext(ctx) - obj, err := c.serviceExportLister.APIServiceExports(ns).Get(name) + obj, err := c.serviceExportLister.APIServiceExports(namespace).Get(name) if err != nil && !errors.IsNotFound(err) { return err } else if errors.IsNotFound(err) { logger.Error(err, "APIServiceExport disappeared") - return c.reconcile(ctx, name, nil) + return c.reconcile(ctx, namespace, name, nil) } old := obj obj = obj.DeepCopy() var errs []error - if err := c.reconcile(ctx, name, obj); err != nil { + if err := c.reconcile(ctx, namespace, name, obj); err != nil { errs = append(errs, err) } diff --git a/pkg/konnector/controllers/cluster/serviceexport/serviceexport_reconcile.go b/pkg/konnector/controllers/cluster/serviceexport/serviceexport_reconcile.go index c76301565..849883af4 100644 --- a/pkg/konnector/controllers/cluster/serviceexport/serviceexport_reconcile.go +++ b/pkg/konnector/controllers/cluster/serviceexport/serviceexport_reconcile.go @@ -19,8 +19,9 @@ package serviceexport import ( "context" "fmt" + "maps" + "slices" "strings" - "sync" "time" corev1 "k8s.io/api/core/v1" @@ -35,9 +36,12 @@ import ( "k8s.io/client-go/rest" "k8s.io/klog/v2" + "github.com/kube-bind/kube-bind/pkg/konnector/controllers/cluster/claimedresources" + "github.com/kube-bind/kube-bind/pkg/konnector/controllers/cluster/claimedresourcesnamespaces" "github.com/kube-bind/kube-bind/pkg/konnector/controllers/cluster/serviceexport/multinsinformer" "github.com/kube-bind/kube-bind/pkg/konnector/controllers/cluster/serviceexport/spec" "github.com/kube-bind/kube-bind/pkg/konnector/controllers/cluster/serviceexport/status" + "github.com/kube-bind/kube-bind/pkg/konnector/controllers/contextstore" "github.com/kube-bind/kube-bind/pkg/konnector/controllers/dynamic" kubebindv1alpha2 "github.com/kube-bind/kube-bind/sdk/apis/kubebind/v1alpha2" conditionsapi "github.com/kube-bind/kube-bind/sdk/apis/third_party/conditions/apis/conditions/v1alpha1" @@ -53,8 +57,7 @@ type reconciler struct { consumerConfig, providerConfig *rest.Config - lock sync.Mutex - syncContext map[string]syncContext // by CRD name + syncStore contextstore.Store // by APIServiceExport name. This includes same ctx for resourc claims and crds. getCRD func(name string) (*apiextensionsv1.CustomResourceDefinition, error) getServiceBinding func(name string) (*kubebindv1alpha2.APIServiceBinding, error) @@ -63,15 +66,10 @@ type reconciler struct { updateRemoteBoundSchema func(ctx context.Context, boundSchema *kubebindv1alpha2.BoundSchema) error } -type syncContext struct { - generation int64 - cancel func() -} - -func (r *reconciler) reconcile(ctx context.Context, name string, export *kubebindv1alpha2.APIServiceExport) error { +func (r *reconciler) reconcile(ctx context.Context, namespace, name string, export *kubebindv1alpha2.APIServiceExport) error { errs := []error{} - if err := r.ensureControllers(ctx, name, export); err != nil { + if err := r.ensureControllers(ctx, namespace, name, export); err != nil { errs = append(errs, err) } @@ -87,22 +85,17 @@ func (r *reconciler) reconcile(ctx context.Context, name string, export *kubebin return utilerrors.NewAggregate(errs) } -func (r *reconciler) ensureControllers(ctx context.Context, name string, export *kubebindv1alpha2.APIServiceExport) error { +func (r *reconciler) ensureControllers(ctx context.Context, namespace, name string, export *kubebindv1alpha2.APIServiceExport) error { logger := klog.FromContext(ctx) + exportKey := contextstore.Key(namespace + "." + name) // Key for the export if export == nil { - // stop dangling syncers on delete - r.lock.Lock() - defer r.lock.Unlock() - // Clean up any controllers associated with this export - for key, c := range r.syncContext { - if strings.HasSuffix(key, "."+name) { - logger.V(1).Info("Stopping APIServiceExport sync", "key", key, "reason", "APIServiceExport deleted") - c.cancel() - delete(r.syncContext, key) - } + deleted := r.syncStore.BulkDeletePrefixed(exportKey) + for _, c := range deleted { + logger.V(1).Info("Stopping APIServiceExport sync", "key", c.Key(), "reason", "NoAPIServiceExport") } + return nil } @@ -113,14 +106,9 @@ func (r *reconciler) ensureControllers(ctx context.Context, name string, export } if binding == nil { // Stop all controllers for this export - r.lock.Lock() - defer r.lock.Unlock() - for key, c := range r.syncContext { - if strings.HasSuffix(key, "."+export.Name) { - logger.V(1).Info("Stopping APIServiceExport sync", "key", key, "reason", "NoAPIServiceBinding") - c.cancel() - delete(r.syncContext, key) - } + deleted := r.syncStore.BulkDeletePrefixed(exportKey) + for _, k := range deleted { + logger.V(1).Info("Stopping APIServiceExport sync", "key", k.Key(), "reason", "NoAPIServiceBinding") } return nil } @@ -128,6 +116,7 @@ func (r *reconciler) ensureControllers(ctx context.Context, name string, export // Process each resource referenced by the export var errs []error processedSchemas := make(map[string]bool) + var isClusterScoped bool for _, res := range export.Spec.Resources { name := res.ResourceGroupName() @@ -135,15 +124,12 @@ func (r *reconciler) ensureControllers(ctx context.Context, name string, export schema, err := r.getRemoteBoundSchema(ctx, name) if err != nil { if errors.IsNotFound(err) { - // Stop the controller for this schema if it exists - r.lock.Lock() - key := name + "." + export.Name - if c, found := r.syncContext[key]; found { - logger.V(1).Info("Stopping APIServiceExport resource sync", "key", key, "reason", "BoundSchema not found") - c.cancel() - delete(r.syncContext, key) + // Stop the controller for this schema if it does not exists. + key := contextstore.NewKey(namespace, name, name) + deleted := r.syncStore.BulkDeletePrefixed(key) + for _, k := range deleted { + logger.V(1).Info("Stopping APIServiceExport sync", "key", k.Key(), "reason", "BoundSchema not found") } - r.lock.Unlock() continue } errs = append(errs, err) @@ -155,44 +141,48 @@ func (r *reconciler) ensureControllers(ctx context.Context, name string, export errs = append(errs, err) } - processedSchemas[name] = true + processedSchemas[name] = true // This is only schemas names (suffix) + isClusterScoped = schema.Spec.Scope == apiextensionsv1.ClusterScoped || schema.Spec.InformerScope == kubebindv1alpha2.ClusterScope } - // Stop controllers for schemas that are no longer referenced - r.lock.Lock() - for key, c := range r.syncContext { - parts := strings.Split(key, ".") - if len(parts) == 2 && parts[1] == export.Name { - schemaName := parts[0] - if !processedSchemas[schemaName] { - logger.V(1).Info("Stopping APIServiceExport resource sync", "key", key, "reason", "Schema no longer referenced") - c.cancel() - delete(r.syncContext, key) - } + // Ensure controller for permission claims + if err := r.ensureControllersForPermissionClaims(ctx, export, binding, isClusterScoped); err != nil { + errs = append(errs, err) + } + + // Stop controllers for schemas that are no longer referenced. + // This will be `exportNamespace.exportName.` + contexts := r.syncStore.ListPrefixed(exportKey) + for _, c := range contexts { + schemaName := strings.TrimPrefix(string(c.Key()), string(exportKey)+".") // schemaName will be processed schema names. + // Skip permission claim controllers - they are handled separately in ensureControllersForPermissionClaims + if strings.HasPrefix(schemaName, "claim.") { + continue + } + if !processedSchemas[schemaName] { + logger.V(1).Info("Stopping APIServiceExport resource sync", "key", c.Key(), "reason", "Schema no longer referenced") + r.syncStore.Delete(c.Key()) } } - r.lock.Unlock() return utilerrors.NewAggregate(errs) } func (r *reconciler) ensureControllerForSchema(ctx context.Context, export *kubebindv1alpha2.APIServiceExport, schema *kubebindv1alpha2.BoundSchema) error { logger := klog.FromContext(ctx) - key := schema.Name + "." + export.Name + key := contextstore.NewKey(export.Namespace, export.Name, schema.Name) - r.lock.Lock() - c, found := r.syncContext[key] + c, found := r.syncStore.Get(key) if found { - if c.generation == export.Generation { - r.lock.Unlock() + if c.Generation == export.Generation { return nil // all as expected } logger.V(1).Info("Stopping APIServiceExport resource sync", "key", key, "reason", "GenerationChanged", "generation", schema.Generation) - c.cancel() - delete(r.syncContext, key) + r.syncStore.Delete(key) } - r.lock.Unlock() + + // At this point we dont have a controller for this schema. // start a new syncer var syncVersion string @@ -243,6 +233,7 @@ func (r *reconciler) ensureControllerForSchema(ctx context.Context, export *kube r.providerNamespace, r.providerConfig, r.serviceNamespaceInformer, + nil, // no label selector ) if err != nil { return err @@ -250,6 +241,7 @@ func (r *reconciler) ensureControllerForSchema(ctx context.Context, export *kube } specCtrl, err := spec.NewController( + export, // pass the export to establish owner references on ServiceNamespace creation gvr, r.providerNamespace, providerNamespaceUID, @@ -288,25 +280,226 @@ func (r *reconciler) ensureControllerForSchema(ctx context.Context, export *kube go func() { // to not block the main thread consumerSynced := consumerInf.WaitForCacheSync(ctxWithCancel.Done()) - logger.V(2).Info("Synced informers", "key", key, "consumer", consumerSynced) + logger.V(2).Info("Synced informers", "key", key, "consumer", slices.Collect(maps.Keys(consumerSynced))) providerSynced := providerInf.WaitForCacheSync(ctxWithCancel.Done()) - logger.V(2).Info("Synced informers", "key", key, "provider", providerSynced) + logger.V(2).Info("Synced informers", "key", key, "provider", slices.Collect(maps.Keys(providerSynced))) go specCtrl.Start(ctxWithCancel, 1) go statusCtrl.Start(ctxWithCancel, 1) }() - r.lock.Lock() - defer r.lock.Unlock() - if c, found := r.syncContext[key]; found { - c.cancel() + r.syncStore.Set(key, contextstore.SyncContext{ + Generation: export.Generation, + Cancel: cancel, + }) + + return nil +} + +func (r *reconciler) ensureControllersForPermissionClaims( + ctx context.Context, + export *kubebindv1alpha2.APIServiceExport, + binding *kubebindv1alpha2.APIServiceBinding, + isClusterScoped bool, // schema.Spec.Scope == apiextensionsv1.ClusterScoped || schema.Spec.InformerScope == kubebindv1alpha2.ClusterScope +) error { + logger := klog.FromContext(ctx) + + // Track processed claims for cleanup + processedClaims := make(map[string]bool) + var errs []error + + // Process each permission claim + for _, claim := range binding.Status.PermissionClaims { + claimGVR, err := kubebindv1alpha2.ResolveClaimableAPI(claim) + if err != nil { + logger.Info("skipping unsupported claim", "claim", claim, "error", err) + continue + } + + // Create unique key for this claim controller + claimKey := contextstore.NewKey(export.Namespace, export.Name, "claim", claimGVR.String()) + processedClaims[claimKey.String()] = true + + // Check if controller already exists with correct generation + c, found := r.syncStore.Get(claimKey) + if found { + if c.Generation == binding.Generation { + continue // controller is up to date + } + // Generation changed, stop old controller + logger.V(1).Info("Stopping permission claim controller", "key", claimKey, "reason", "GenerationChanged") + r.syncStore.Delete(claimKey) + } + + // Start new controller for this claim + if err := r.ensureControllerForPermissionClaim(ctx, binding, claim, export, claimGVR, isClusterScoped, claimKey); err != nil { + errs = append(errs, err) + } + } + + // Cleanup controllers for claims that are no longer present + claimPrefix := contextstore.NewKey(export.Namespace, export.Name, "claim") + contexts := r.syncStore.ListPrefixed(claimPrefix) + for _, c := range contexts { + if !processedClaims[c.Key().String()] { + logger.V(1).Info("Stopping permission claim controller", "key", c.Key(), "reason", "Claim no longer present") + r.syncStore.Delete(c.Key()) + } + } + + return utilerrors.NewAggregate(errs) +} + +func (r *reconciler) ensureControllerForPermissionClaim( + ctx context.Context, + binding *kubebindv1alpha2.APIServiceBinding, + claim kubebindv1alpha2.PermissionClaim, + export *kubebindv1alpha2.APIServiceExport, + claimGVR runtimeschema.GroupVersionResource, + isClusterScoped bool, + claimKey contextstore.Key, +) error { + logger := klog.FromContext(ctx) + + dynamicProviderClient := dynamicclient.NewForConfigOrDie(r.providerConfig) + dynamicConsumerClient := dynamicclient.NewForConfigOrDie(r.consumerConfig) + + // Precompute label selector string if present + var selectorStr string + if claim.Selector.LabelSelector != nil { + selector, err := metav1.LabelSelectorAsSelector(claim.Selector.LabelSelector) + if err != nil { + return fmt.Errorf("failed to convert label selector: %w", err) + } + selectorStr = selector.String() + logger.V(2).Info("Using label selector for informer", "selector", selectorStr, "resource", claim.Resource) + } + + tweakListOptions := func(options *metav1.ListOptions) { + if selectorStr != "" { + options.LabelSelector = selectorStr + } } - r.syncContext[key] = syncContext{ - generation: schema.Generation, - cancel: cancel, + + // Create consumer informer factory + var defaultConsumerInf dynamicinformer.DynamicSharedInformerFactory + if claim.Selector.LabelSelector != nil { + defaultConsumerInf = dynamicinformer.NewFilteredDynamicSharedInformerFactory(dynamicConsumerClient, time.Minute*30, metav1.NamespaceAll, tweakListOptions) + } else { + defaultConsumerInf = dynamicinformer.NewDynamicSharedInformerFactory(dynamicConsumerClient, time.Minute*30) } + // Now if you reading this and thinking "What is happening here...", this is why this comment exists :) + // Provider informers can operate in 2 modes: + // 1. Cluster-scoped - in this case we create a single informer factory that watches all namespaces + // for the given GVR. + // 2. Namespace-scoped - in this case we create a multi-namespace informer that watches only the + // provider namespace and any service namespaces created for the binding. + // But in additon to that, for permission claims, we also need to consider label selectors + // If one is present, we need to filter the informers using that selector. So we have 4 possible + // combinations: + // 1A Cluster-scoped, no label selector - single informer factory watching all namespaces + // 1B Cluster-scoped, with label selector - single filtered informer factory watching all namespaces + // B1 Namespace-scoped, no label selector - multi-namespace informer watching provider + service namespaces + // B2 Namespace-scoped, with label selector - multi-namespace filtered informer watching provider + service namespaces + // The code below implements this logic. Follow the letters. + var defaultProviderInf multinsinformer.GetterInformer // will hold the default provider informer (A or B) + if isClusterScoped { + var factory dynamicinformer.DynamicSharedInformerFactory // Placeholder for A. + if claim.Selector.LabelSelector != nil { // A2 + factory = dynamicinformer.NewFilteredDynamicSharedInformerFactory(dynamicProviderClient, time.Minute*30, metav1.NamespaceAll, tweakListOptions) + } else { // A1 + factory = dynamicinformer.NewDynamicSharedInformerFactory(dynamicProviderClient, time.Minute*30) + } + factory.ForResource(claimGVR).Lister() // wire the GVR up in the informer factory + defaultProviderInf = multinsinformer.GetterInformerWrapper{ + GVR: claimGVR, + Delegate: factory, + } + } else { + // B case for labels is implemented inside multinsinformer.NewDynamicMultiNamespaceInformer + // Logic in there is same as here - if label selector is present, create filtered informer factory + var err error + defaultProviderInf, err = multinsinformer.NewDynamicMultiNamespaceInformer( + claimGVR, + r.providerNamespace, + r.providerConfig, + r.serviceNamespaceInformer, + claim.Selector.LabelSelector, + ) + + if err != nil { + logger.Info("aborting", "error", err) + return err + } + } + + serviceNamespaceChannel := make(chan string, 100) + + claimedCtrl, err := claimedresources.NewController( + claimGVR, + claim, + export, + r.providerNamespace, + r.consumerConfig, + r.providerConfig, + defaultConsumerInf.ForResource(claimGVR), + defaultProviderInf, + r.serviceNamespaceInformer, + serviceNamespaceChannel, + ) + if err != nil { + return err + } + + claimedNamespacesCtrl, err := claimedresourcesnamespaces.NewController( + claimGVR, + claim, + export, + r.providerNamespace, + r.providerConfig, + defaultConsumerInf.ForResource(claimGVR), + r.serviceNamespaceInformer, + serviceNamespaceChannel, + ) + if err != nil { + return err + } + + logger.Info("creating claim reconciler", "gvr", claimGVR, "key", claimKey) + + ctxWithCancel, cancel := context.WithCancel(ctx) + // Start the informers and controllers in a goroutine + go func() { + defer r.syncStore.Delete(claimKey) + defaultConsumerInf.Start(ctxWithCancel.Done()) + + // Wait for consumer informers to sync + consumerSynced := defaultConsumerInf.WaitForCacheSync(ctxWithCancel.Done()) + logger.V(2).Info("Synced consumer informers", "consumer", slices.Collect(maps.Keys(consumerSynced)), "key", claimKey) + + // IMPORTANT: WE need to start namespaces controller before the claimed resources controller, + // as the later depends on the former to ensure service namespaces are present. + // Else provider informers will never sync and the controller will not start. + // check: claimedresourcesnamespaces/README.md for more details. + go claimedNamespacesCtrl.Start(ctxWithCancel, 1) + + // Start provider informer and wait for sync + defaultProviderInf.Start(ctxWithCancel) + providerSynced := defaultProviderInf.WaitForCacheSync(ctxWithCancel.Done()) + logger.V(2).Info("Synced provider informers", "provider", slices.Collect(maps.Keys(providerSynced)), "key", claimKey) + + // Start the claimed resources controller + claimedCtrl.Start(ctxWithCancel, 1) + }() + + // Store the controller context for tracking + r.syncStore.Set(claimKey, contextstore.SyncContext{ + Generation: binding.Generation, + Cancel: cancel, + }) + return nil } @@ -371,11 +564,11 @@ func (r *reconciler) ensureCRDConditionsCopiedToBoundSchema(ctx context.Context, if err := r.updateRemoteBoundSchema(ctx, boundSchema); err != nil { errs = append(errs, err) - allValid = false // at least one BoundAPIResourceSchema is not valid + allValid = false // at least one BoundSchemas is not valid } } - // Set APIServiceExport Ready condition based on all BoundAPIResourceSchemas + // Set APIServiceExport Ready condition based on all BoundSchemas if allValid { conditions.MarkTrue( export, @@ -385,9 +578,9 @@ func (r *reconciler) ensureCRDConditionsCopiedToBoundSchema(ctx context.Context, conditions.MarkFalse( export, kubebindv1alpha2.APIServiceExportConditionConnected, - "BoundAPIResourceSchemasNotValid", + "BoundSchemasNotValid", conditionsapi.ConditionSeverityWarning, - "One or more BoundAPIResourceSchemas are not valid", + "One or more BoundSchemas are not valid", ) } diff --git a/pkg/konnector/controllers/cluster/serviceexport/spec/spec_controller.go b/pkg/konnector/controllers/cluster/serviceexport/spec/spec_controller.go index 884e1ec4b..db794909b 100644 --- a/pkg/konnector/controllers/cluster/serviceexport/spec/spec_controller.go +++ b/pkg/konnector/controllers/cluster/serviceexport/spec/spec_controller.go @@ -55,6 +55,7 @@ const ( // NewController returns a new controller reconciling downstream objects to upstream. func NewController( + apiServiceExport *kubebindv1alpha2.APIServiceExport, // used to establish owner references when create happens from the consumer side. gvr schema.GroupVersionResource, providerNamespace string, providerNamespaceUID string, @@ -99,6 +100,8 @@ func NewController( reconciler: reconciler{ providerNamespace: providerNamespace, + apiServiceExport: apiServiceExport, + getServiceNamespace: func(name string) (*kubebindv1alpha2.APIServiceNamespace, error) { return serviceNamespaceInformer.Lister().APIServiceNamespaces(providerNamespace).Get(name) }, diff --git a/pkg/konnector/controllers/cluster/serviceexport/spec/spec_reconcile.go b/pkg/konnector/controllers/cluster/serviceexport/spec/spec_reconcile.go index 0eae960ad..7acc317c9 100644 --- a/pkg/konnector/controllers/cluster/serviceexport/spec/spec_reconcile.go +++ b/pkg/konnector/controllers/cluster/serviceexport/spec/spec_reconcile.go @@ -33,6 +33,7 @@ import ( type reconciler struct { providerNamespace string + apiServiceExport *kubebindv1alpha2.APIServiceExport // used to establish owner references when create happens from the consumer side. getServiceNamespace func(name string) (*kubebindv1alpha2.APIServiceNamespace, error) createServiceNamespace func(ctx context.Context, sn *kubebindv1alpha2.APIServiceNamespace) (*kubebindv1alpha2.APIServiceNamespace, error) @@ -50,6 +51,9 @@ type reconciler struct { // reconcile syncs downstream objects (metadata and spec) with upstream objects. func (r *reconciler) reconcile(ctx context.Context, obj *unstructured.Unstructured) error { logger := klog.FromContext(ctx) + if r.apiServiceExport == nil { // Should never happen, but we check to make sure we dont regress in the future. + return fmt.Errorf("internal error: apiServiceExport is nil") + } ns := obj.GetNamespace() if ns != "" { @@ -62,6 +66,9 @@ func (r *reconciler) reconcile(ctx context.Context, obj *unstructured.Unstructur ObjectMeta: metav1.ObjectMeta{ Name: ns, Namespace: r.providerNamespace, + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(r.apiServiceExport, kubebindv1alpha2.SchemeGroupVersion.WithKind("APIServiceExport")), + }, }, }) if err != nil { diff --git a/pkg/konnector/controllers/contextstore/contextstore.go b/pkg/konnector/controllers/contextstore/contextstore.go new file mode 100644 index 000000000..cd610d228 --- /dev/null +++ b/pkg/konnector/controllers/contextstore/contextstore.go @@ -0,0 +1,126 @@ +/* +Copyright 2022 The Kube Bind Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// contextstore allows to manage and track context per controllers, stored at the different levels of the controller hierarchy: +// APIServiceExport - schemas and permissionClaims - each schema runs its own gvr controller and each permissionClaim runs its own controller. +// Context are stored at the APIServiceExport level. +package contextstore + +import ( + "strings" + "sync" +) + +type Key string + +func (k Key) String() string { + return string(k) +} + +func NewKey(exportNamespace, exportName string, suffix ...string) Key { + if len(suffix) == 0 { + return Key(exportNamespace + "." + exportName) + } + return Key(exportNamespace + "." + exportName + "." + strings.Join(suffix, ".")) +} + +type Store interface { + Get(key Key) (SyncContext, bool) + ListPrefixed(prefix Key) []SyncContext + Set(key Key, value SyncContext) + Delete(key Key) + BulkDeletePrefixed(prefix Key) []SyncContext +} + +type contextStore struct { + lock sync.Mutex + store map[Key]SyncContext +} + +type SyncContext struct { + key Key // reference key for the context for logging + Generation int64 + Cancel func() +} + +func New() Store { + return &contextStore{ + store: make(map[Key]SyncContext), + } +} + +func (c *SyncContext) Key() Key { + return c.key +} + +func (c *contextStore) Get(key Key) (SyncContext, bool) { + c.lock.Lock() + defer c.lock.Unlock() + val, ok := c.store[key] + return val, ok +} + +func (c *contextStore) ListPrefixed(prefix Key) []SyncContext { + c.lock.Lock() + defer c.lock.Unlock() + + var results []SyncContext + for k, v := range c.store { + if strings.HasPrefix(k.String(), prefix.String()) { + results = append(results, v) + } + } + return results +} + +func (c *contextStore) Set(key Key, value SyncContext) { + c.lock.Lock() + defer c.lock.Unlock() + value.key = key + c.store[key] = value +} + +func (c *contextStore) Delete(key Key) { + c.lock.Lock() + if ctx, ok := c.store[key]; ok { + defer ctx.Cancel() + delete(c.store, key) + } + c.lock.Unlock() +} + +func (c *contextStore) BulkDeletePrefixed(prefix Key) []SyncContext { + c.lock.Lock() + defer c.lock.Unlock() + + var deleted []SyncContext + var keysToDelete []Key + + for k, v := range c.store { + if strings.HasPrefix(k.String(), prefix.String()) { + keysToDelete = append(keysToDelete, k) + deleted = append(deleted, v) + } + } + + for _, k := range keysToDelete { + ctx := c.store[k] + ctx.Cancel() + delete(c.store, k) + } + + return deleted +} diff --git a/pkg/resources/resources.go b/pkg/resources/resources.go new file mode 100644 index 000000000..d37e524c3 --- /dev/null +++ b/pkg/resources/resources.go @@ -0,0 +1,69 @@ +/* +Copyright 2025 The Kube Bind Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resources + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" + + kubebindv1alpha2 "github.com/kube-bind/kube-bind/sdk/apis/kubebind/v1alpha2" +) + +// IsClaimed returns true if the given object matches the given selector and named resources. +func IsClaimed(selector kubebindv1alpha2.Selector, obj *unstructured.Unstructured) bool { + if obj == nil { + return false + } + // Empty selector selects everything + if selector.LabelSelector == nil && len(selector.NamedResource) == 0 { + return true + } + + // Both label selector and named resources must match if both are specified + labelSelectorMatches := true + namedResourceMatches := true + + // Check label selector if specified + if selector.LabelSelector != nil { + selector, err := metav1.LabelSelectorAsSelector(selector.LabelSelector) + if err != nil { + return false + } + l := obj.GetLabels() + if l == nil { + l = make(map[string]string) + } + labelSelectorMatches = selector.Matches(labels.Set(l)) + } + + // Check named resources if specified + if len(selector.NamedResource) > 0 { + namedResourceMatches = false // Default to false, must match at least one + for _, nr := range selector.NamedResource { + if nr.Namespace != "" && nr.Namespace != obj.GetNamespace() { + continue + } + if nr.Name == obj.GetName() { + namedResourceMatches = true + break + } + } + } + + return labelSelectorMatches && namedResourceMatches +} diff --git a/pkg/resources/resources_test.go b/pkg/resources/resources_test.go new file mode 100644 index 000000000..a5d0a465c --- /dev/null +++ b/pkg/resources/resources_test.go @@ -0,0 +1,348 @@ +/* +Copyright 2025 The Kube Bind Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resources + +import ( + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + kubebindv1alpha2 "github.com/kube-bind/kube-bind/sdk/apis/kubebind/v1alpha2" +) + +func TestSelector_IsClaimed(t *testing.T) { + tests := []struct { + name string + selector kubebindv1alpha2.Selector + obj *unstructured.Unstructured + want bool + }{ + { + name: "empty selector should select all", + selector: kubebindv1alpha2.Selector{}, + obj: &unstructured.Unstructured{ + Object: map[string]any{ + "metadata": map[string]any{ + "name": "test-obj", + "namespace": "test-ns", + "labels": map[string]any{ + "app": "test", + }, + }, + }, + }, + want: true, + }, + { + name: "label selector should match labels", + selector: kubebindv1alpha2.Selector{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "test", + }, + }, + }, + obj: &unstructured.Unstructured{ + Object: map[string]any{ + "metadata": map[string]any{ + "name": "test-obj", + "namespace": "test-ns", + "labels": map[string]any{ + "app": "test", + }, + }, + }, + }, + want: true, + }, + { + name: "label selector should not match different labels", + selector: kubebindv1alpha2.Selector{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "different", + }, + }, + }, + obj: &unstructured.Unstructured{ + Object: map[string]any{ + "metadata": map[string]any{ + "name": "test-obj", + "namespace": "test-ns", + "labels": map[string]any{ + "app": "test", + }, + }, + }, + }, + want: false, + }, + { + name: "named resource selector should match exact name and namespace", + selector: kubebindv1alpha2.Selector{ + NamedResource: []kubebindv1alpha2.NamedResource{ + { + Name: "test-obj", + Namespace: "test-ns", + }, + }, + }, + obj: &unstructured.Unstructured{ + Object: map[string]any{ + "metadata": map[string]any{ + "name": "test-obj", + "namespace": "test-ns", + }, + }, + }, + want: true, + }, + { + name: "named resource selector should match name when namespace is empty", + selector: kubebindv1alpha2.Selector{ + NamedResource: []kubebindv1alpha2.NamedResource{ + { + Name: "test-obj", + Namespace: "", + }, + }, + }, + obj: &unstructured.Unstructured{ + Object: map[string]any{ + "metadata": map[string]any{ + "name": "test-obj", + "namespace": "test-ns", + }, + }, + }, + want: true, + }, + { + name: "named resource selector should not match different name", + selector: kubebindv1alpha2.Selector{ + NamedResource: []kubebindv1alpha2.NamedResource{ + { + Name: "other-obj", + Namespace: "test-ns", + }, + }, + }, + obj: &unstructured.Unstructured{ + Object: map[string]any{ + "metadata": map[string]any{ + "name": "test-obj", + "namespace": "test-ns", + }, + }, + }, + want: false, + }, + { + name: "named resource selector should not match different namespace", + selector: kubebindv1alpha2.Selector{ + NamedResource: []kubebindv1alpha2.NamedResource{ + { + Name: "test-obj", + Namespace: "other-ns", + }, + }, + }, + obj: &unstructured.Unstructured{ + Object: map[string]any{ + "metadata": map[string]any{ + "name": "test-obj", + "namespace": "test-ns", + }, + }, + }, + want: false, + }, + { + name: "named resource selector should match one of multiple resources", + selector: kubebindv1alpha2.Selector{ + NamedResource: []kubebindv1alpha2.NamedResource{ + { + Name: "other-obj", + Namespace: "test-ns", + }, + { + Name: "test-obj", + Namespace: "test-ns", + }, + }, + }, + obj: &unstructured.Unstructured{ + Object: map[string]any{ + "metadata": map[string]any{ + "name": "test-obj", + "namespace": "test-ns", + }, + }, + }, + want: true, + }, + { + name: "label selector with object having no labels should not match", + selector: kubebindv1alpha2.Selector{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "test", + }, + }, + }, + obj: &unstructured.Unstructured{ + Object: map[string]any{ + "metadata": map[string]any{ + "name": "test-obj", + "namespace": "test-ns", + }, + }, + }, + want: false, + }, + { + name: "combination of label selector and named resource should match when both match", + selector: kubebindv1alpha2.Selector{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "test", + }, + }, + NamedResource: []kubebindv1alpha2.NamedResource{ + { + Name: "test-obj", + Namespace: "test-ns", + }, + }, + }, + obj: &unstructured.Unstructured{ + Object: map[string]any{ + "metadata": map[string]any{ + "name": "test-obj", + "namespace": "test-ns", + "labels": map[string]any{ + "app": "test", + }, + }, + }, + }, + want: true, + }, + { + name: "combination of label selector and named resource should not match when label doesn't match", + selector: kubebindv1alpha2.Selector{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "different", + }, + }, + NamedResource: []kubebindv1alpha2.NamedResource{ + { + Name: "test-obj", + Namespace: "test-ns", + }, + }, + }, + obj: &unstructured.Unstructured{ + Object: map[string]any{ + "metadata": map[string]any{ + "name": "test-obj", + "namespace": "test-ns", + "labels": map[string]any{ + "app": "test", + }, + }, + }, + }, + want: false, + }, + { + name: "combination of label selector and named resource should not match when name doesn't match", + selector: kubebindv1alpha2.Selector{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "test", + }, + }, + NamedResource: []kubebindv1alpha2.NamedResource{ + { + Name: "other-obj", + Namespace: "test-ns", + }, + }, + }, + obj: &unstructured.Unstructured{ + Object: map[string]any{ + "metadata": map[string]any{ + "name": "test-obj", + "namespace": "test-ns", + "labels": map[string]any{ + "app": "test", + }, + }, + }, + }, + want: false, + }, + { + name: "label-only-secret test case - should not sync when has label but not in named resource list", + selector: kubebindv1alpha2.Selector{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "secrets", + }, + }, + NamedResource: []kubebindv1alpha2.NamedResource{ + { + Name: "test-secret", + Namespace: "default", + }, + { + Name: "named-secret-1", + Namespace: "default", + }, + { + Name: "named-secret-2", + Namespace: "default", + }, + }, + }, + obj: &unstructured.Unstructured{ + Object: map[string]any{ + "metadata": map[string]any{ + "name": "label-only-secret", + "namespace": "default", + "labels": map[string]any{ + "app": "secrets", + }, + }, + }, + }, + want: false, // Should NOT match because name is not in named resource list + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := IsClaimed(tt.selector, tt.obj) + if got != tt.want { + t.Errorf("IsClaimed() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/sdk/apis/kubebind/v1alpha2/apiservicebinding_types.go b/sdk/apis/kubebind/v1alpha2/apiservicebinding_types.go index 97cfc0450..174f9dc38 100644 --- a/sdk/apis/kubebind/v1alpha2/apiservicebinding_types.go +++ b/sdk/apis/kubebind/v1alpha2/apiservicebinding_types.go @@ -103,6 +103,10 @@ type APIServiceBindingStatus struct { // +optional // +kubebuilder:validation:MinItems=1 BoundSchemas []BoundSchemaReference `json:"boundSchemas,omitempty"` + + // PermissionClaims records decisions about permission claims requested by the service provider. + // Access is granted per GroupResource. + PermissionClaims []PermissionClaim `json:"permissionClaims,omitempty"` } // BoundSchemaReference contains a reference to a BoundAPIResourceSchema with status information. diff --git a/sdk/apis/kubebind/v1alpha2/apiserviceexport_types.go b/sdk/apis/kubebind/v1alpha2/apiserviceexport_types.go index f51f8eff1..2237c787b 100644 --- a/sdk/apis/kubebind/v1alpha2/apiserviceexport_types.go +++ b/sdk/apis/kubebind/v1alpha2/apiserviceexport_types.go @@ -17,7 +17,6 @@ limitations under the License. package v1alpha2 import ( - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" conditionsapi "github.com/kube-bind/kube-bind/sdk/apis/third_party/conditions/apis/conditions/v1alpha1" @@ -38,6 +37,9 @@ const ( // APIServiceExportConditionConsumerInSync is set to true when the APIServiceExport's // schema is applied to the consumer cluster. APIServiceExportConditionConsumerInSync conditionsapi.ConditionType = "ConsumerInSync" + + // APIServiceExportConditionPermissionClaim describes status of the permission claim, requested in the APIServiceExport and APIServiceExportRequest. + APIServiceExportConditionPermissionClaim conditionsapi.ConditionType = "PermissionClaim" ) // APIServiceExport specifies the resource to be exported. It is mostly a CRD: @@ -86,6 +88,11 @@ type APIServiceExportSpec struct { // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="resources are immutable" Resources []APIServiceExportResource `json:"resources"` + // PermissionClaims records decisions about permission claims requested by the service provider. + // Access is granted per GroupResource. + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="permissionClaims are immutable" + PermissionClaims []PermissionClaim `json:"permissionClaims,omitempty"` + // informerScope is the scope of the APIServiceExport. It can be either Cluster or Namespace. // // Cluster: The konnector has permission to watch all namespaces at once and cluster-scoped resources. @@ -122,20 +129,6 @@ const ( // APIServiceExportStatus stores status information about a APIServiceExport. It // reflects the status of the CRD of the consumer cluster. type APIServiceExportStatus struct { - // acceptedNames are the names that are actually being used to serve discovery. - // They may be different than the names in spec. - // +optional - AcceptedNames apiextensionsv1.CustomResourceDefinitionNames `json:"acceptedNames"` - - // storedVersions lists all versions of CustomResources that were ever persisted. Tracking these - // versions allows a migration path for stored versions in etcd. The field is mutable - // so a migration controller can finish a migration to another version (ensuring - // no old objects are left in storage), and then remove the rest of the - // versions from this list. - // Versions may not be removed from `spec.versions` while they exist in this list. - // +optional - StoredVersions []string `json:"storedVersions"` - // conditions is a list of conditions that apply to the APIServiceExport. It is // updated by the konnector on the consumer cluster. Conditions conditionsapi.Conditions `json:"conditions,omitempty"` diff --git a/sdk/apis/kubebind/v1alpha2/apiserviceexportrequest_types.go b/sdk/apis/kubebind/v1alpha2/apiserviceexportrequest_types.go index 938ecb9a1..5c3ad11b2 100644 --- a/sdk/apis/kubebind/v1alpha2/apiserviceexportrequest_types.go +++ b/sdk/apis/kubebind/v1alpha2/apiserviceexportrequest_types.go @@ -104,8 +104,17 @@ type APIServiceExportRequestSpec struct { // +required // +kubebuilder:validation:Required // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:XListType=map + // +kubebuilder:validation:XListMapKey=resource + // +kubebuilder:validation:XListMapKey=group // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="resources are immutable" Resources []APIServiceExportRequestResource `json:"resources"` + + // PermissionClaims records decisions about permission claims requested by the service provider. + // Access is granted per GroupResource. + // + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="permissionClaims are immutable" + PermissionClaims []PermissionClaim `json:"permissionClaims,omitempty"` } type APIServiceExportRequestResource struct { @@ -123,6 +132,35 @@ func (r APIServiceExportRequestResource) ResourceGroupName() string { return fmt.Sprintf("%s.%s", r.Resource, r.Group) } +// Selector is a resource selector that selects objects of a GVR. +// Selectors are ANDed together if multiple are specified. +type Selector struct { + // NamedResource is a shorthand for selecting a single resource by name and namespace. + // +optional + NamedResource []NamedResource `json:"namedResource,omitempty"` + + // LabelSelector is a label selector that selects objects of a GVR. + // +optional + LabelSelector *metav1.LabelSelector `json:"labelSelector,omitempty"` +} + +// NamedResource selects a specific resource by name and namespace. +type NamedResource struct { + // Name is the name of the resource. + // Name matches the metadata.name field of the underlying object. + // + // +required + // +kubebuilder:validation:Required + Name string `json:"name,omitempty"` + + // Namespace represents namespace where an object of the given group/resource may be managed. + // Namespaces matches against the metadata.namespace field. If not provided, the object is assumed to be cluster-scoped. + // Namespaces field is ignored for namespaced isolation mode. + // + // +optional + Namespace string `json:"namespace,omitempty"` +} + // GroupResource identifies a resource. type GroupResource struct { // group is the name of an API group. @@ -142,6 +180,35 @@ type GroupResource struct { Resource string `json:"resource"` } +func (r GroupResource) String() string { + return fmt.Sprintf("%s.%s", r.Resource, r.Group) +} + +// PermissionClaim selects objects of a GVR that a service provider may +// request and that a consumer may accept and allow the service provider access to. +type PermissionClaim struct { + GroupResource `json:",inline"` + + // Selector is a resource selector that selects objects of a GVR. + // +required + // +kubebuilder:validation:Required + Selector Selector `json:"selector,omitempty"` +} + +// Owner is the owner of the resource. +type Owner string + +const ( + // OwnerProvider indicates that the resource is owned by the provider. + OwnerProvider Owner = "provider" + // OwnerConsumer indicates that the resource is owned by the consumer. + OwnerConsumer Owner = "consumer" +) + +func (o Owner) String() string { + return string(o) +} + // APIServiceExportRequestPhase describes the phase of a binding request. type APIServiceExportRequestPhase string diff --git a/sdk/apis/kubebind/v1alpha2/boundchema_types.go b/sdk/apis/kubebind/v1alpha2/boundchema_types.go index a3d2193d4..ed39fc3c5 100644 --- a/sdk/apis/kubebind/v1alpha2/boundchema_types.go +++ b/sdk/apis/kubebind/v1alpha2/boundchema_types.go @@ -18,6 +18,7 @@ package v1alpha2 import ( "encoding/json" + "fmt" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -26,6 +27,10 @@ import ( conditionsapi "github.com/kube-bind/kube-bind/sdk/apis/third_party/conditions/apis/conditions/v1alpha1" ) +// ExportedSchemas are the schemas exported by the current backend. +// Keys are "resource.group" for quick resolve (version is not part of the key). +type ExportedSchemas map[string]*BoundSchema + // BoundSchema // +crd // +genclient @@ -41,6 +46,13 @@ type BoundSchema struct { Status BoundSchemaStatus `json:"status,omitempty"` } +// ResourceGroupName returns the group name of the resource. +// +// Important: If you change this, change one for APIServiceExportRequestResource too. +func (b *BoundSchema) ResourceGroupName() string { + return fmt.Sprintf("%s.%s", b.Spec.Names.Plural, b.Spec.Group) +} + // InformerScope is the scope of the Api. // // +kubebuilder:validation:Enum=Cluster;Namespaced @@ -246,12 +258,12 @@ const ( BoundSchemaDriftDetected BoundSchemaConditionReason = "DriftDetected" ) -func (in *BoundSchema) GetConditions() conditionsapi.Conditions { - return in.Status.Conditions +func (b *BoundSchema) GetConditions() conditionsapi.Conditions { + return b.Status.Conditions } -func (in *BoundSchema) SetConditions(conditions conditionsapi.Conditions) { - in.Status.Conditions = conditions +func (b *BoundSchema) SetConditions(conditions conditionsapi.Conditions) { + b.Status.Conditions = conditions } // BoundSchemaStatus defines the observed state of the BoundSchema. diff --git a/sdk/apis/kubebind/v1alpha2/claimable_apis.go b/sdk/apis/kubebind/v1alpha2/claimable_apis.go new file mode 100644 index 000000000..6e76f37f0 --- /dev/null +++ b/sdk/apis/kubebind/v1alpha2/claimable_apis.go @@ -0,0 +1,90 @@ +/* +Copyright 2025 The Kube Bind Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha2 + +import ( + "fmt" + + corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// InternalAPI describes an API to be imported from some schemes and generated OpenAPI V2 definitions. +type InternalAPI struct { + Names apiextensionsv1.CustomResourceDefinitionNames + GroupVersionResource schema.GroupVersionResource + Instance runtime.Object + ResourceScope apiextensionsv1.ResourceScope + HasStatus bool +} + +// ClaimableAPIs is a list of APIs that can be claimed by a user. +var ClaimableAPIs = []InternalAPI{ + { + Names: apiextensionsv1.CustomResourceDefinitionNames{ + Plural: "configmaps", + Singular: "configmap", + Kind: "ConfigMap", + }, + GroupVersionResource: schema.GroupVersionResource{ + Group: "", + Version: "v1", + Resource: "configmaps", + }, + Instance: &corev1.ConfigMap{}, + ResourceScope: apiextensionsv1.NamespaceScoped, + }, + { + Names: apiextensionsv1.CustomResourceDefinitionNames{ + Plural: "secrets", + Singular: "secret", + Kind: "Secret", + }, + GroupVersionResource: schema.GroupVersionResource{ + Group: "", + Version: "v1", + Resource: "secrets", + }, + Instance: &corev1.Secret{}, + ResourceScope: apiextensionsv1.NamespaceScoped, + }, + { + Names: apiextensionsv1.CustomResourceDefinitionNames{ + Plural: "serviceaccounts", + Singular: "serviceaccount", + Kind: "ServiceAccount", + }, + GroupVersionResource: schema.GroupVersionResource{ + Group: "", + Version: "v1", + Resource: "serviceaccounts", + }, + Instance: &corev1.ServiceAccount{}, + ResourceScope: apiextensionsv1.NamespaceScoped, + }, +} + +func ResolveClaimableAPI(claim PermissionClaim) (schema.GroupVersionResource, error) { + for _, api := range ClaimableAPIs { + if api.Names.Plural == claim.Resource && api.GroupVersionResource.Group == claim.Group { + return api.GroupVersionResource, nil + } + } + return schema.GroupVersionResource{}, fmt.Errorf("no matching API found") +} diff --git a/sdk/apis/kubebind/v1alpha2/zz_generated.deepcopy.go b/sdk/apis/kubebind/v1alpha2/zz_generated.deepcopy.go index 9792299d4..f952ef835 100644 --- a/sdk/apis/kubebind/v1alpha2/zz_generated.deepcopy.go +++ b/sdk/apis/kubebind/v1alpha2/zz_generated.deepcopy.go @@ -24,6 +24,7 @@ package v1alpha2 import ( v1alpha1 "github.com/kube-bind/kube-bind/sdk/apis/third_party/conditions/apis/conditions/v1alpha1" v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -177,6 +178,13 @@ func (in *APIServiceBindingStatus) DeepCopyInto(out *APIServiceBindingStatus) { *out = make([]BoundSchemaReference, len(*in)) copy(*out, *in) } + if in.PermissionClaims != nil { + in, out := &in.PermissionClaims, &out.PermissionClaims + *out = make([]PermissionClaim, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } return } @@ -377,6 +385,13 @@ func (in *APIServiceExportRequestSpec) DeepCopyInto(out *APIServiceExportRequest (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.PermissionClaims != nil { + in, out := &in.PermissionClaims, &out.PermissionClaims + *out = make([]PermissionClaim, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } return } @@ -423,6 +438,13 @@ func (in *APIServiceExportSpec) DeepCopyInto(out *APIServiceExportSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.PermissionClaims != nil { + in, out := &in.PermissionClaims, &out.PermissionClaims + *out = make([]PermissionClaim, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } return } @@ -439,12 +461,6 @@ func (in *APIServiceExportSpec) DeepCopy() *APIServiceExportSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *APIServiceExportStatus) DeepCopyInto(out *APIServiceExportStatus) { *out = *in - in.AcceptedNames.DeepCopyInto(&out.AcceptedNames) - if in.StoredVersions != nil { - in, out := &in.StoredVersions, &out.StoredVersions - *out = make([]string, len(*in)) - copy(*out, *in) - } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions *out = make(v1alpha1.Conditions, len(*in)) @@ -952,6 +968,36 @@ func (in *CustomResourceConversion) DeepCopy() *CustomResourceConversion { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in ExportedSchemas) DeepCopyInto(out *ExportedSchemas) { + { + in := &in + *out = make(ExportedSchemas, len(*in)) + for key, val := range *in { + var outVal *BoundSchema + if val == nil { + (*out)[key] = nil + } else { + in, out := &val, &outVal + *out = new(BoundSchema) + (*in).DeepCopyInto(*out) + } + (*out)[key] = outVal + } + return + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExportedSchemas. +func (in ExportedSchemas) DeepCopy() ExportedSchemas { + if in == nil { + return nil + } + out := new(ExportedSchemas) + in.DeepCopyInto(out) + return *out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GroupResource) DeepCopyInto(out *GroupResource) { *out = *in @@ -968,6 +1014,27 @@ func (in *GroupResource) DeepCopy() *GroupResource { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InternalAPI) DeepCopyInto(out *InternalAPI) { + *out = *in + in.Names.DeepCopyInto(&out.Names) + out.GroupVersionResource = in.GroupVersionResource + if in.Instance != nil { + out.Instance = in.Instance.DeepCopyObject() + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InternalAPI. +func (in *InternalAPI) DeepCopy() *InternalAPI { + if in == nil { + return nil + } + out := new(InternalAPI) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LocalSecretKeyRef) DeepCopyInto(out *LocalSecretKeyRef) { *out = *in @@ -1000,6 +1067,22 @@ func (in *NameObjectMeta) DeepCopy() *NameObjectMeta { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NamedResource) DeepCopyInto(out *NamedResource) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NamedResource. +func (in *NamedResource) DeepCopy() *NamedResource { + if in == nil { + return nil + } + out := new(NamedResource) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OAuth2CodeGrant) DeepCopyInto(out *OAuth2CodeGrant) { *out = *in @@ -1016,6 +1099,50 @@ func (in *OAuth2CodeGrant) DeepCopy() *OAuth2CodeGrant { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PermissionClaim) DeepCopyInto(out *PermissionClaim) { + *out = *in + out.GroupResource = in.GroupResource + in.Selector.DeepCopyInto(&out.Selector) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PermissionClaim. +func (in *PermissionClaim) DeepCopy() *PermissionClaim { + if in == nil { + return nil + } + out := new(PermissionClaim) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Selector) DeepCopyInto(out *Selector) { + *out = *in + if in.NamedResource != nil { + in, out := &in.NamedResource, &out.NamedResource + *out = make([]NamedResource, len(*in)) + copy(*out, *in) + } + if in.LabelSelector != nil { + in, out := &in.LabelSelector, &out.LabelSelector + *out = new(metav1.LabelSelector) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Selector. +func (in *Selector) DeepCopy() *Selector { + if in == nil { + return nil + } + out := new(Selector) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *WebhookClientConfig) DeepCopyInto(out *WebhookClientConfig) { *out = *in diff --git a/test/e2e/bind/happy-case_test.go b/test/e2e/bind/happy-case_test.go index 2bcebe56b..4bfb2d6f4 100644 --- a/test/e2e/bind/happy-case_test.go +++ b/test/e2e/bind/happy-case_test.go @@ -18,6 +18,7 @@ package bind import ( "context" + "encoding/json" "fmt" "strings" "testing" @@ -25,6 +26,7 @@ import ( "github.com/headzoo/surf" "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -43,19 +45,25 @@ import ( func TestClusterScoped(t *testing.T) { t.Parallel() - // cluster scoped resource, with cluster scoped informers - testHappyCase(t, apiextensionsv1.ClusterScoped, kubebindv1alpha2.ClusterScope) + testHappyCase(t, apiextensionsv1.ClusterScoped, kubebindv1alpha2.ClusterScope, false) + testHappyCase(t, apiextensionsv1.ClusterScoped, kubebindv1alpha2.ClusterScope, true) } func TestNamespacedScoped(t *testing.T) { t.Parallel() - // namespaced resource, with namespace scoped informers - testHappyCase(t, apiextensionsv1.NamespaceScoped, kubebindv1alpha2.NamespacedScope) - // namespaced resource, but with cluster scoped informers - testHappyCase(t, apiextensionsv1.NamespaceScoped, kubebindv1alpha2.ClusterScope) + + testHappyCase(t, apiextensionsv1.NamespaceScoped, kubebindv1alpha2.NamespacedScope, false) + testHappyCase(t, apiextensionsv1.NamespaceScoped, kubebindv1alpha2.NamespacedScope, true) + testHappyCase(t, apiextensionsv1.NamespaceScoped, kubebindv1alpha2.ClusterScope, false) + testHappyCase(t, apiextensionsv1.NamespaceScoped, kubebindv1alpha2.ClusterScope, true) } -func testHappyCase(t *testing.T, resourceScope apiextensionsv1.ResourceScope, informerScope kubebindv1alpha2.InformerScope) { +func testHappyCase( + t *testing.T, + resourceScope apiextensionsv1.ResourceScope, + informerScope kubebindv1alpha2.InformerScope, + withPermissionClaims bool, +) { ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) @@ -85,6 +93,10 @@ func testHappyCase(t *testing.T, resourceScope apiextensionsv1.ResourceScope, in consumerClient := framework.DynamicClient(t, consumerConfig).Resource(serviceGVR) providerClient := framework.DynamicClient(t, providerConfig).Resource(serviceGVR) + consumerCoreClient := framework.KubeClient(t, consumerConfig).CoreV1() + providerCoreClient := framework.KubeClient(t, providerConfig).CoreV1() + providerBindClient := framework.BindClient(t, providerConfig) + mangodbInstance := ` apiVersion: mangodb.com/v1alpha1 kind: MangoDB @@ -102,7 +114,7 @@ spec: deploymentName: test-foo replicas: 2 ` - downstreamNs, upstreamNS := "default", "unknown" + consumerNS, providerNS := "default", "unknown" clusterNs, clusterScopedUpInsName := "unknown", "unknown" for _, tc := range []struct { @@ -130,6 +142,60 @@ spec: framework.Bind(t, iostreams, authURLCh, invocations, fmt.Sprintf("http://%s/exports", addr.String()), "--kubeconfig", consumerKubeconfig, "--skip-konnector") inv := <-invocations requireEqualSlicePattern(t, []string{"apiservice", "--remote-kubeconfig-namespace", "*", "--remote-kubeconfig-name", "*", "-f", "-", "--kubeconfig=" + consumerKubeconfig, "--skip-konnector=true", "--no-banner"}, inv.Args) + + // If we are in permissions claims mode - add configmaps & secrets + if withPermissionClaims { + var request kubebindv1alpha2.APIServiceExportRequest + err := json.Unmarshal(inv.Stdin, &request) + require.NoError(t, err) + request.Spec.PermissionClaims = []kubebindv1alpha2.PermissionClaim{ + { + GroupResource: kubebindv1alpha2.GroupResource{ + Group: "", + Resource: "configmaps", + }, + Selector: kubebindv1alpha2.Selector{ + NamedResource: []kubebindv1alpha2.NamedResource{ + { + Name: "named-configmap-only", + Namespace: consumerNS, + }, + }, + }, + }, + { + GroupResource: kubebindv1alpha2.GroupResource{ + Group: "", + Resource: "secrets", + }, + Selector: kubebindv1alpha2.Selector{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "secrets", + }, + }, + NamedResource: []kubebindv1alpha2.NamedResource{ + { + Name: "test-secret", + Namespace: consumerNS, + }, + { + Name: "named-secret-1", + Namespace: consumerNS, + }, + { + Name: "named-secret-2", + Namespace: consumerNS, + }, + }, + }, + }, + } + payload, err := json.Marshal(request) + require.NoError(t, err) + inv.Stdin = payload + } + framework.BindAPIService(t, inv.Stdin, "", inv.Args...) t.Logf("Waiting for %s CRD to be created on consumer side", serviceGVR.Resource) @@ -151,7 +217,7 @@ spec: require.Eventually(t, func() bool { var err error if resourceScope == apiextensionsv1.NamespaceScoped { - _, err = consumerClient.Namespace(downstreamNs).Create(ctx, toUnstructured(t, mangodbInstance), metav1.CreateOptions{}) + _, err = consumerClient.Namespace(consumerNS).Create(ctx, toUnstructured(t, mangodbInstance), metav1.CreateOptions{}) } else { _, err = consumerClient.Create(ctx, toUnstructured(t, fooInstance), metav1.CreateOptions{}) } @@ -167,19 +233,269 @@ spec: }, wait.ForeverTestTimeout, time.Millisecond*100, "waiting for the %s instance to be created on provider side", serviceGVR.Resource) // these are used everywhere further down - upstreamNS = instances.Items[0].GetNamespace() + providerNS = instances.Items[0].GetNamespace() if resourceScope == apiextensionsv1.ClusterScoped { clusterNs, _ = clusterscoped.ExtractClusterNs(&instances.Items[0]) clusterScopedUpInsName = clusterscoped.Prepend("test", clusterNs) } }, }, + { + name: "create secrets and configmaps if permission claims enabled", + step: func(t *testing.T) { + if !withPermissionClaims { + t.Skip("Skipping permission claims test when permission claims are disabled") + return + } + + t.Logf("Creating named-only configmap on consumer side") + namedConfigMapData := map[string]string{ + "named-config.yaml": "named: value", + } + namedConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "named-configmap-only", + Namespace: consumerNS, + // Note: No "app" label - this should only be captured by NamedResource + }, + Data: namedConfigMapData, + } + _, err := consumerCoreClient.ConfigMaps(consumerNS).Create(ctx, namedConfigMap, metav1.CreateOptions{}) + require.NoError(t, err) + + t.Logf("Creating secret on consumer side") + secretData := map[string][]byte{ + "password": []byte("secret-password"), + } + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-secret", + Namespace: consumerNS, + Labels: map[string]string{ + "app": "secrets", + }, + }, + Data: secretData, + } + _, err = consumerCoreClient.Secrets(consumerNS).Create(ctx, secret, metav1.CreateOptions{}) + require.NoError(t, err) + + t.Logf("Creating named secrets on consumer side") + + // Create the first named secret (matches BOTH label AND named resource) + namedSecret1 := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "named-secret-1", + Namespace: consumerNS, + Labels: map[string]string{ + "app": "secrets", // Matches the label selector + }, + }, + Data: map[string][]byte{ + "key1": []byte("value1"), + }, + } + _, err = consumerCoreClient.Secrets(consumerNS).Create(ctx, namedSecret1, metav1.CreateOptions{}) + require.NoError(t, err) + + // Create the second named secret (matches BOTH label AND named resource) + namedSecret2 := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "named-secret-2", + Namespace: consumerNS, + Labels: map[string]string{ + "app": "secrets", // Matches the label selector + }, + }, + Data: map[string][]byte{ + "key2": []byte("value2"), + }, + } + _, err = consumerCoreClient.Secrets(consumerNS).Create(ctx, namedSecret2, metav1.CreateOptions{}) + require.NoError(t, err) + + // Create a secret that has the correct label BUT is not in the named resource list (should NOT sync) + labelOnlySecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "label-only-secret", + Namespace: consumerNS, + Labels: map[string]string{ + "app": "secrets", // Matches label but not in named list + }, + }, + Data: map[string][]byte{ + "labelonly": []byte("should-not-sync"), + }, + } + _, err = consumerCoreClient.Secrets(consumerNS).Create(ctx, labelOnlySecret, metav1.CreateOptions{}) + require.NoError(t, err) + + // Create a secret that should NOT be synced (neither label nor named resource match) + nonSyncedSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "other-secret", + Namespace: consumerNS, + }, + Data: map[string][]byte{ + "key3": []byte("value3"), + }, + } + _, err = consumerCoreClient.Secrets(consumerNS).Create(ctx, nonSyncedSecret, metav1.CreateOptions{}) + require.NoError(t, err) + }, + }, + { + name: "establish permission claims namespace", + step: func(t *testing.T) { + // We need to establish namespace only in cluster scope for cluster scoped resources. + // Else we can trust sync object namespace as it will be the same. + if withPermissionClaims && + informerScope == kubebindv1alpha2.ClusterScope && + resourceScope == apiextensionsv1.ClusterScoped { + if providerNS == "unknown" { + t.Fatal("providerNS is not set. Programming error in the test.") + } + + var namespaces *kubebindv1alpha2.APIServiceNamespaceList + t.Logf("Waiting for APIServiceNamespace to be created on provider side") + require.Eventually(t, func() bool { + var err error + namespaces, err = providerBindClient.KubeBindV1alpha2().APIServiceNamespaces(providerNS).List(ctx, metav1.ListOptions{}) + if err != nil { + return false + } + + return len(namespaces.Items) == 1 && namespaces.Items[0].Status.Namespace != "" + }, wait.ForeverTestTimeout, time.Millisecond*100, "waiting for APIServiceNamespace to be created on provider side") + + providerNS = namespaces.Items[0].Status.Namespace + require.NotEmpty(t, providerNS, "No cluster namespaces found") + } + }, + }, + { + name: "verify secrets and configmaps are synced to provider", + step: func(t *testing.T) { + if !withPermissionClaims { + t.Skip("Skipping permission claims test when permission claims are disabled") + return + } + + t.Logf("Waiting for named-only configmap to be synced to provider side") + require.Eventually(t, func() bool { + _, err := providerCoreClient.ConfigMaps(providerNS).Get(ctx, "named-configmap-only", metav1.GetOptions{}) + return err == nil + }, wait.ForeverTestTimeout, time.Millisecond*100, "waiting for named-only configmap to be synced to provider side") + + t.Logf("Waiting for secret to be synced to provider side") + require.Eventually(t, func() bool { + _, err := providerCoreClient.Secrets(providerNS).Get(ctx, "test-secret", metav1.GetOptions{}) + return err == nil + }, wait.ForeverTestTimeout, time.Millisecond*100, "waiting for secret to be synced to provider side") + + t.Logf("Verifying named-only configmap data is correct") + providerNamedConfigMap, err := providerCoreClient.ConfigMaps(providerNS).Get(ctx, "named-configmap-only", metav1.GetOptions{}) + require.NoError(t, err) + require.Equal(t, "named: value", providerNamedConfigMap.Data["named-config.yaml"]) + + t.Logf("Verifying secret data is correct") + providerSecret, err := providerCoreClient.Secrets(providerNS).Get(ctx, "test-secret", metav1.GetOptions{}) + require.NoError(t, err) + require.Equal(t, []byte("secret-password"), providerSecret.Data["password"]) + + t.Logf("Waiting for named secrets to be synced to provider side") + + // Verify first named secret is synced + require.Eventually(t, func() bool { + _, err := providerCoreClient.Secrets(providerNS).Get(ctx, "named-secret-1", metav1.GetOptions{}) + return err == nil + }, wait.ForeverTestTimeout, time.Millisecond*100, "waiting for named-secret-1 to be synced to provider side") + + // Verify second named secret is synced + require.Eventually(t, func() bool { + _, err := providerCoreClient.Secrets(providerNS).Get(ctx, "named-secret-2", metav1.GetOptions{}) + return err == nil + }, wait.ForeverTestTimeout, time.Millisecond*100, "waiting for named-secret-2 to be synced to provider side") + + // Verify data integrity of named secrets + providerNamedSecret1, err := providerCoreClient.Secrets(providerNS).Get(ctx, "named-secret-1", metav1.GetOptions{}) + require.NoError(t, err) + require.Equal(t, []byte("value1"), providerNamedSecret1.Data["key1"]) + + providerNamedSecret2, err := providerCoreClient.Secrets(providerNS).Get(ctx, "named-secret-2", metav1.GetOptions{}) + require.NoError(t, err) + require.Equal(t, []byte("value2"), providerNamedSecret2.Data["key2"]) + + // Verify that secrets that don't match BOTH conditions are NOT synced + _, err = providerCoreClient.Secrets(providerNS).Get(ctx, "label-only-secret", metav1.GetOptions{}) + require.True(t, errors.IsNotFound(err), "label-only-secret should not be synced (not in named resource list)") + + _, err = providerCoreClient.Secrets(providerNS).Get(ctx, "other-secret", metav1.GetOptions{}) + require.True(t, errors.IsNotFound(err), "other-secret should not be synced (neither label nor named resource match)") + }, + }, + { + name: "verify secrets and configmaps are deleted when removed from consumer", + step: func(t *testing.T) { + if !withPermissionClaims { + t.Skip("Skipping permission claims test when permission claims are disabled") + return + } + + t.Logf("Deleting named-only configmap from consumer side") + err := consumerCoreClient.ConfigMaps(consumerNS).Delete(ctx, "named-configmap-only", metav1.DeleteOptions{}) + require.NoError(t, err) + + t.Logf("Deleting secret from consumer side") + err = consumerCoreClient.Secrets(consumerNS).Delete(ctx, "test-secret", metav1.DeleteOptions{}) + require.NoError(t, err) + + t.Logf("Waiting for named-only configmap to be deleted from provider side") + require.Eventually(t, func() bool { + _, err := providerCoreClient.ConfigMaps(providerNS).Get(ctx, "named-configmap-only", metav1.GetOptions{}) + return errors.IsNotFound(err) + }, wait.ForeverTestTimeout, time.Millisecond*100, "waiting for named-only configmap to be deleted from provider side") + + t.Logf("Waiting for secret to be deleted from provider side") + require.Eventually(t, func() bool { + _, err := providerCoreClient.Secrets(providerNS).Get(ctx, "test-secret", metav1.GetOptions{}) + return errors.IsNotFound(err) + }, wait.ForeverTestTimeout, time.Millisecond*100, "waiting for secret to be deleted from provider side") + + t.Logf("Deleting named secrets from consumer side") + + err = consumerCoreClient.Secrets(consumerNS).Delete(ctx, "named-secret-1", metav1.DeleteOptions{}) + require.NoError(t, err) + + err = consumerCoreClient.Secrets(consumerNS).Delete(ctx, "named-secret-2", metav1.DeleteOptions{}) + require.NoError(t, err) + + t.Logf("Cleaning up additional test secrets from consumer side") + err = consumerCoreClient.Secrets(consumerNS).Delete(ctx, "label-only-secret", metav1.DeleteOptions{}) + require.NoError(t, err) + + err = consumerCoreClient.Secrets(consumerNS).Delete(ctx, "other-secret", metav1.DeleteOptions{}) + require.NoError(t, err) + + t.Logf("Waiting for named secrets to be deleted from provider side") + + require.Eventually(t, func() bool { + _, err := providerCoreClient.Secrets(providerNS).Get(ctx, "named-secret-1", metav1.GetOptions{}) + return errors.IsNotFound(err) + }, wait.ForeverTestTimeout, time.Millisecond*100, "waiting for named-secret-1 to be deleted from provider side") + + require.Eventually(t, func() bool { + _, err := providerCoreClient.Secrets(providerNS).Get(ctx, "named-secret-2", metav1.GetOptions{}) + return errors.IsNotFound(err) + }, wait.ForeverTestTimeout, time.Millisecond*100, "waiting for named-secret-2 to be deleted from provider side") + }, + }, { name: "instance deleted upstream is recreated", step: func(t *testing.T) { var err error if resourceScope == apiextensionsv1.NamespaceScoped { - err = providerClient.Namespace(upstreamNS).Delete(ctx, "test", metav1.DeleteOptions{}) + err = providerClient.Namespace(providerNS).Delete(ctx, "test", metav1.DeleteOptions{}) } else { err = providerClient.Delete(ctx, clusterScopedUpInsName, metav1.DeleteOptions{}) } @@ -188,7 +504,7 @@ spec: require.Eventually(t, func() bool { var err error if resourceScope == apiextensionsv1.NamespaceScoped { - _, err = providerClient.Namespace(upstreamNS).Get(ctx, "test", metav1.GetOptions{}) + _, err = providerClient.Namespace(providerNS).Get(ctx, "test", metav1.GetOptions{}) } else { _, err = providerClient.Get(ctx, clusterScopedUpInsName, metav1.GetOptions{}) } @@ -203,14 +519,14 @@ spec: var obj *unstructured.Unstructured var err error if resourceScope == apiextensionsv1.NamespaceScoped { - obj, err = consumerClient.Namespace(downstreamNs).Get(ctx, "test", metav1.GetOptions{}) + obj, err = consumerClient.Namespace(consumerNS).Get(ctx, "test", metav1.GetOptions{}) } else { obj, err = consumerClient.Get(ctx, "test", metav1.GetOptions{}) } require.NoError(t, err) if resourceScope == apiextensionsv1.NamespaceScoped { unstructured.SetNestedField(obj.Object, "Dedicated", "spec", "tier") //nolint:errcheck - _, err = consumerClient.Namespace(downstreamNs).Update(ctx, obj, metav1.UpdateOptions{}) + _, err = consumerClient.Namespace(consumerNS).Update(ctx, obj, metav1.UpdateOptions{}) } else { unstructured.SetNestedField(obj.Object, "tested", "spec", "deploymentName") //nolint:errcheck _, err = consumerClient.Update(ctx, obj, metav1.UpdateOptions{}) @@ -223,7 +539,7 @@ spec: var obj *unstructured.Unstructured var err error if resourceScope == apiextensionsv1.NamespaceScoped { - obj, err = providerClient.Namespace(upstreamNS).Get(ctx, "test", metav1.GetOptions{}) + obj, err = providerClient.Namespace(providerNS).Get(ctx, "test", metav1.GetOptions{}) } else { obj, err = providerClient.Get(ctx, clusterScopedUpInsName, metav1.GetOptions{}) } @@ -250,14 +566,14 @@ spec: var obj *unstructured.Unstructured var err error if resourceScope == apiextensionsv1.NamespaceScoped { - obj, err = providerClient.Namespace(upstreamNS).Get(ctx, "test", metav1.GetOptions{}) + obj, err = providerClient.Namespace(providerNS).Get(ctx, "test", metav1.GetOptions{}) } else { obj, err = providerClient.Get(ctx, clusterScopedUpInsName, metav1.GetOptions{}) } require.NoError(t, err) unstructured.SetNestedField(obj.Object, "Running", "status", "phase") //nolint:errcheck if resourceScope == apiextensionsv1.NamespaceScoped { - _, err = providerClient.Namespace(upstreamNS).UpdateStatus(ctx, obj, metav1.UpdateOptions{}) + _, err = providerClient.Namespace(providerNS).UpdateStatus(ctx, obj, metav1.UpdateOptions{}) } else { _, err = providerClient.UpdateStatus(ctx, obj, metav1.UpdateOptions{}) } @@ -269,7 +585,7 @@ spec: var obj *unstructured.Unstructured var err error if resourceScope == apiextensionsv1.NamespaceScoped { - obj, err = consumerClient.Namespace(downstreamNs).Get(ctx, "test", metav1.GetOptions{}) + obj, err = consumerClient.Namespace(consumerNS).Get(ctx, "test", metav1.GetOptions{}) } else { obj, err = consumerClient.Get(ctx, "test", metav1.GetOptions{}) } @@ -287,14 +603,14 @@ spec: var obj *unstructured.Unstructured var err error if resourceScope == apiextensionsv1.NamespaceScoped { - obj, err = providerClient.Namespace(upstreamNS).Get(ctx, "test", metav1.GetOptions{}) + obj, err = providerClient.Namespace(providerNS).Get(ctx, "test", metav1.GetOptions{}) } else { obj, err = providerClient.Get(ctx, clusterScopedUpInsName, metav1.GetOptions{}) } require.NoError(t, err) if resourceScope == apiextensionsv1.NamespaceScoped { unstructured.SetNestedField(obj.Object, "Shared", "spec", "tier") //nolint:errcheck - _, err = providerClient.Namespace(upstreamNS).Update(ctx, obj, metav1.UpdateOptions{}) + _, err = providerClient.Namespace(providerNS).Update(ctx, obj, metav1.UpdateOptions{}) } else { unstructured.SetNestedField(obj.Object, "drifting", "spec", "deploymentName") //nolint:errcheck _, err = providerClient.Update(ctx, obj, metav1.UpdateOptions{}) @@ -307,7 +623,7 @@ spec: var obj *unstructured.Unstructured var err error if resourceScope == apiextensionsv1.NamespaceScoped { - obj, err = providerClient.Namespace(upstreamNS).Get(ctx, "test", metav1.GetOptions{}) + obj, err = providerClient.Namespace(providerNS).Get(ctx, "test", metav1.GetOptions{}) } else { obj, err = providerClient.Get(ctx, clusterScopedUpInsName, metav1.GetOptions{}) } @@ -332,7 +648,7 @@ spec: step: func(t *testing.T) { var err error if resourceScope == apiextensionsv1.NamespaceScoped { - err = consumerClient.Namespace(downstreamNs).Delete(ctx, "test", metav1.DeleteOptions{}) + err = consumerClient.Namespace(consumerNS).Delete(ctx, "test", metav1.DeleteOptions{}) } else { err = consumerClient.Delete(ctx, "test", metav1.DeleteOptions{}) } @@ -341,7 +657,7 @@ spec: require.Eventually(t, func() bool { var err error if resourceScope == apiextensionsv1.NamespaceScoped { - _, err = providerClient.Namespace(upstreamNS).Get(ctx, "test", metav1.GetOptions{}) + _, err = providerClient.Namespace(providerNS).Get(ctx, "test", metav1.GetOptions{}) } else { _, err = providerClient.Get(ctx, clusterScopedUpInsName, metav1.GetOptions{}) } diff --git a/test/e2e/framework/clients.go b/test/e2e/framework/clients.go index 8ec5cbb2b..088ea53f3 100644 --- a/test/e2e/framework/clients.go +++ b/test/e2e/framework/clients.go @@ -26,6 +26,8 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" + + bindclientset "github.com/kube-bind/kube-bind/sdk/client/clientset/versioned" ) func DynamicClient(t testing.TB, config *rest.Config) dynamic.Interface { @@ -52,7 +54,13 @@ func DiscoveryClient(t testing.TB, config *rest.Config) discovery.DiscoveryInter return c } -func NewRESTConfig(t testing.TB, kubeconfig string) *rest.Config { +func BindClient(t *testing.T, config *rest.Config) bindclientset.Interface { + c, err := bindclientset.NewForConfig(config) + require.NoError(t, err) + return c +} + +func NewRESTConfig(t *testing.T, kubeconfig string) *rest.Config { config, err := clientcmd.BuildConfigFromFlags("", kubeconfig) require.NoError(t, err, "Failed to build config from kubeconfig file") return config