diff options
| author | David Bond <davidsbond93@gmail.com> | 2026-03-27 16:34:20 +0000 |
|---|---|---|
| committer | David Bond <davidsbond93@gmail.com> | 2026-03-27 16:34:20 +0000 |
| commit | 9227aaa8899729e57aa03731e1a23215d8840743 (patch) | |
| tree | c6775ca275fb0a6d1385898317978100ad3dcfda | |
| parent | 9ab73b3d3a77759a060db3efa614371551f5a1c8 (diff) | |
| download | tailscale-davidb/reconciler-moving.tar.xz tailscale-davidb/reconciler-moving.zip | |
cmd/k8s-operator: move dnsrecords and nameserver into their own packagesdavidb/reconciler-moving
This commit moves the reconcilers for both the DNS nameserver and
DNSConfig custom resource into their own packages within
`k8s-operator/reconciler`
Closes: https://github.com/tailscale/corp/issues/37088
Signed-off-by: David Bond <davidsbond93@gmail.com>
| -rw-r--r-- | cmd/k8s-operator/depaware.txt | 7 | ||||
| -rw-r--r-- | cmd/k8s-operator/operator.go | 154 | ||||
| -rw-r--r-- | k8s-operator/reconciler/dnsrecords/dnsrecords.go (renamed from cmd/k8s-operator/dnsrecords.go) | 310 | ||||
| -rw-r--r-- | k8s-operator/reconciler/dnsrecords/dnsrecords_test.go (renamed from cmd/k8s-operator/dnsrecords_test.go) | 141 | ||||
| -rw-r--r-- | k8s-operator/reconciler/nameserver/manifests/cm.yaml | 4 | ||||
| -rw-r--r-- | k8s-operator/reconciler/nameserver/manifests/deploy.yaml | 37 | ||||
| -rw-r--r-- | k8s-operator/reconciler/nameserver/manifests/sa.yaml | 4 | ||||
| -rw-r--r-- | k8s-operator/reconciler/nameserver/manifests/svc.yaml | 16 | ||||
| -rw-r--r-- | k8s-operator/reconciler/nameserver/nameserver.go (renamed from cmd/k8s-operator/nameserver.go) | 227 | ||||
| -rw-r--r-- | k8s-operator/reconciler/nameserver/nameserver_test.go (renamed from cmd/k8s-operator/nameserver_test.go) | 80 | ||||
| -rw-r--r-- | k8s-operator/reconciler/proxyclass/proxyclass.go | 8 | ||||
| -rw-r--r-- | k8s-operator/reconciler/reconciler.go | 53 |
12 files changed, 740 insertions, 301 deletions
diff --git a/cmd/k8s-operator/depaware.txt b/cmd/k8s-operator/depaware.txt index a3340d03b..15830d78f 100644 --- a/cmd/k8s-operator/depaware.txt +++ b/cmd/k8s-operator/depaware.txt @@ -91,7 +91,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ W 💣 github.com/dblohm7/wingoes/com/automation from tailscale.com/util/osdiag/internal/wsc W github.com/dblohm7/wingoes/internal from github.com/dblohm7/wingoes/com W 💣 github.com/dblohm7/wingoes/pe from tailscale.com/util/osdiag+ - github.com/distribution/reference from tailscale.com/cmd/k8s-operator + github.com/distribution/reference from tailscale.com/cmd/k8s-operator+ github.com/emicklei/go-restful/v3 from k8s.io/kube-openapi/pkg/common github.com/emicklei/go-restful/v3/log from github.com/emicklei/go-restful/v3 github.com/evanphx/json-patch/v5 from sigs.k8s.io/controller-runtime/pkg/client @@ -832,7 +832,10 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ tailscale.com/k8s-operator/api-proxy from tailscale.com/cmd/k8s-operator tailscale.com/k8s-operator/apis from tailscale.com/k8s-operator/apis/v1alpha1 tailscale.com/k8s-operator/apis/v1alpha1 from tailscale.com/cmd/k8s-operator+ - tailscale.com/k8s-operator/reconciler from tailscale.com/k8s-operator/reconciler/tailnet + tailscale.com/k8s-operator/reconciler from tailscale.com/k8s-operator/reconciler/tailnet+ + tailscale.com/k8s-operator/reconciler/dnsrecords from tailscale.com/cmd/k8s-operator + tailscale.com/k8s-operator/reconciler/nameserver from tailscale.com/cmd/k8s-operator + tailscale.com/k8s-operator/reconciler/proxyclass from tailscale.com/cmd/k8s-operator tailscale.com/k8s-operator/reconciler/proxygrouppolicy from tailscale.com/cmd/k8s-operator tailscale.com/k8s-operator/reconciler/tailnet from tailscale.com/cmd/k8s-operator tailscale.com/k8s-operator/sessionrecording from tailscale.com/k8s-operator/api-proxy diff --git a/cmd/k8s-operator/operator.go b/cmd/k8s-operator/operator.go index 13748503c..15efc0c65 100644 --- a/cmd/k8s-operator/operator.go +++ b/cmd/k8s-operator/operator.go @@ -55,6 +55,8 @@ import ( "tailscale.com/ipn/store/kubestore" apiproxy "tailscale.com/k8s-operator/api-proxy" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler/dnsrecords" + "tailscale.com/k8s-operator/reconciler/nameserver" "tailscale.com/k8s-operator/reconciler/proxyclass" "tailscale.com/k8s-operator/reconciler/proxygrouppolicy" "tailscale.com/k8s-operator/reconciler/tailnet" @@ -525,22 +527,14 @@ func runReconcilers(opts reconcilerOpts) { // TODO (irbekrm): switch to metadata-only watches for resources whose // spec we don't need to inspect to reduce memory consumption. // https://github.com/kubernetes-sigs/controller-runtime/issues/1159 - nameserverFilter := handler.EnqueueRequestsFromMapFunc(managedResourceHandlerForType("nameserver")) - err = builder.ControllerManagedBy(mgr). - For(&tsapi.DNSConfig{}). - Named("nameserver-reconciler"). - Watches(&appsv1.Deployment{}, nameserverFilter). - Watches(&corev1.ConfigMap{}, nameserverFilter). - Watches(&corev1.Service{}, nameserverFilter). - Watches(&corev1.ServiceAccount{}, nameserverFilter). - Complete(&NameserverReconciler{ - recorder: eventRecorder, - tsNamespace: opts.tailscaleNamespace, - Client: mgr.GetClient(), - logger: opts.log.Named("nameserver-reconciler"), - clock: tstime.DefaultClock{}, - }) - if err != nil { + nameserverOptions := nameserver.ReconcilerOptions{ + Client: mgr.GetClient(), + Recorder: eventRecorder, + TailscaleNamespace: opts.tailscaleNamespace, + Logger: opts.log, + Clock: tstime.DefaultClock{}, + } + if err = nameserver.NewReconciler(nameserverOptions).Register(mgr); err != nil { startlog.Fatalf("could not create nameserver reconciler: %v", err) } @@ -621,43 +615,24 @@ func runReconcilers(opts reconcilerOpts) { } proxyClassOptions := proxyclass.ReconcilerOptions{ - Client: mgr.GetClient(), - Recorder: eventRecorder, - TsNamespace: opts.tailscaleNamespace, - Logger: opts.log, - Clock: tstime.DefaultClock{}, + Client: mgr.GetClient(), + Recorder: eventRecorder, + TailscaleNamespace: opts.tailscaleNamespace, + Logger: opts.log, + Clock: tstime.DefaultClock{}, } if err = proxyclass.NewReconciler(proxyClassOptions).Register(mgr); err != nil { startlog.Fatalf("could not create proxyclass reconciler: %v", err) } - logger := startlog.Named("dns-records-reconciler-event-handlers") - // On EndpointSlice events, if it is an EndpointSlice for an - // ingress/egress proxy headless Service, reconcile the headless - // Service. - dnsRREpsOpts := handler.EnqueueRequestsFromMapFunc(dnsRecordsReconcilerEndpointSliceHandler) - // On DNSConfig changes, reconcile all headless Services for - // ingress/egress proxies in operator namespace. - dnsRRDNSConfigOpts := handler.EnqueueRequestsFromMapFunc(enqueueAllIngressEgressProxySvcsInNS(opts.tailscaleNamespace, mgr.GetClient(), logger)) - // On Service events, if it is an ingress/egress proxy headless Service, reconcile it. - dnsRRServiceOpts := handler.EnqueueRequestsFromMapFunc(dnsRecordsReconcilerServiceHandler) - // On Ingress events, if it is a tailscale Ingress or if tailscale is the default ingress controller, reconcile the proxy - // headless Service. - dnsRRIngressOpts := handler.EnqueueRequestsFromMapFunc(dnsRecordsReconcilerIngressHandler(opts.tailscaleNamespace, opts.proxyActAsDefaultLoadBalancer, mgr.GetClient(), logger)) - err = builder.ControllerManagedBy(mgr). - Named("dns-records-reconciler"). - Watches(&corev1.Service{}, dnsRRServiceOpts). - Watches(&networkingv1.Ingress{}, dnsRRIngressOpts). - Watches(&discoveryv1.EndpointSlice{}, dnsRREpsOpts). - Watches(&tsapi.DNSConfig{}, dnsRRDNSConfigOpts). - Complete(&dnsRecordsReconciler{ - Client: mgr.GetClient(), - tsNamespace: opts.tailscaleNamespace, - logger: opts.log.Named("dns-records-reconciler"), - isDefaultLoadBalancer: opts.proxyActAsDefaultLoadBalancer, - }) - if err != nil { + dnsRecordsOptions := dnsrecords.ReconcilerOptions{ + Client: mgr.GetClient(), + TailscaleNamespace: opts.tailscaleNamespace, + Logger: opts.log, + IsDefaultLoadBalancer: opts.proxyActAsDefaultLoadBalancer, + } + if err = dnsrecords.NewReconciler(dnsRecordsOptions).Register(mgr); err != nil { startlog.Fatalf("could not create DNS records reconciler: %v", err) } @@ -801,91 +776,6 @@ type reconcilerOpts struct { // enqueueAllIngressEgressProxySvcsinNS returns a reconcile request for each // ingress/egress proxy headless Service found in the provided namespace. -func enqueueAllIngressEgressProxySvcsInNS(ns string, cl client.Client, logger *zap.SugaredLogger) handler.MapFunc { - return func(ctx context.Context, _ client.Object) []reconcile.Request { - reqs := make([]reconcile.Request, 0) - - // Get all headless Services for proxies configured using Service. - svcProxyLabels := map[string]string{ - kubetypes.LabelManaged: "true", - LabelParentType: "svc", - } - svcHeadlessSvcList := &corev1.ServiceList{} - if err := cl.List(ctx, svcHeadlessSvcList, client.InNamespace(ns), client.MatchingLabels(svcProxyLabels)); err != nil { - logger.Errorf("error listing headless Services for tailscale ingress/egress Services in operator namespace: %v", err) - return nil - } - for _, svc := range svcHeadlessSvcList.Items { - reqs = append(reqs, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: svc.Namespace, Name: svc.Name}}) - } - - // Get all headless Services for proxies configured using Ingress. - ingProxyLabels := map[string]string{ - kubetypes.LabelManaged: "true", - LabelParentType: "ingress", - } - ingHeadlessSvcList := &corev1.ServiceList{} - if err := cl.List(ctx, ingHeadlessSvcList, client.InNamespace(ns), client.MatchingLabels(ingProxyLabels)); err != nil { - logger.Errorf("error listing headless Services for tailscale Ingresses in operator namespace: %v", err) - return nil - } - for _, svc := range ingHeadlessSvcList.Items { - reqs = append(reqs, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: svc.Namespace, Name: svc.Name}}) - } - return reqs - } -} - -// dnsRecordsReconciler filters EndpointSlice events for which -// dns-records-reconciler should reconcile a headless Service. The only events -// it should reconcile are those for EndpointSlices associated with proxy -// headless Services. -func dnsRecordsReconcilerEndpointSliceHandler(ctx context.Context, o client.Object) []reconcile.Request { - if !isManagedByType(o, "svc") && !isManagedByType(o, "ingress") { - return nil - } - headlessSvcName, ok := o.GetLabels()[discoveryv1.LabelServiceName] // https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/#ownership - if !ok { - return nil - } - return []reconcile.Request{{NamespacedName: types.NamespacedName{Namespace: o.GetNamespace(), Name: headlessSvcName}}} -} - -// dnsRecordsReconcilerServiceHandler filters Service events for which -// dns-records-reconciler should reconcile. If the event is for a cluster -// ingress/cluster egress proxy's headless Service, returns the Service for -// reconcile. -func dnsRecordsReconcilerServiceHandler(ctx context.Context, o client.Object) []reconcile.Request { - if isManagedByType(o, "svc") || isManagedByType(o, "ingress") { - return []reconcile.Request{{NamespacedName: types.NamespacedName{Namespace: o.GetNamespace(), Name: o.GetName()}}} - } - return nil -} - -// dnsRecordsReconcilerIngressHandler filters Ingress events to ensure that -// dns-records-reconciler only reconciles on tailscale Ingress events. When an -// event is observed on a tailscale Ingress, reconcile the proxy headless Service. -func dnsRecordsReconcilerIngressHandler(ns string, isDefaultLoadBalancer bool, cl client.Client, logger *zap.SugaredLogger) handler.MapFunc { - return func(ctx context.Context, o client.Object) []reconcile.Request { - ing, ok := o.(*networkingv1.Ingress) - if !ok { - return nil - } - if !isDefaultLoadBalancer && (ing.Spec.IngressClassName == nil || *ing.Spec.IngressClassName != "tailscale") { - return nil - } - proxyResourceLabels := childResourceLabels(ing.Name, ing.Namespace, "ingress") - headlessSvc, err := getSingleObject[corev1.Service](ctx, cl, ns, proxyResourceLabels) - if err != nil { - logger.Errorf("error getting headless Service from parent labels: %v", err) - return nil - } - if headlessSvc == nil { - return nil - } - return []reconcile.Request{{NamespacedName: types.NamespacedName{Namespace: headlessSvc.Namespace, Name: headlessSvc.Name}}} - } -} func isManagedResource(o client.Object) bool { ls := o.GetLabels() diff --git a/cmd/k8s-operator/dnsrecords.go b/k8s-operator/reconciler/dnsrecords/dnsrecords.go index e75bcd4c2..63d28a089 100644 --- a/cmd/k8s-operator/dnsrecords.go +++ b/k8s-operator/reconciler/dnsrecords/dnsrecords.go @@ -3,7 +3,9 @@ //go:build !plan9 -package main +// Package dnsrecords provides reconciliation logic for keeping the dnsrecords +// ConfigMap up to date with DNS records for tailscale ingress and egress proxies. +package dnsrecords import ( "context" @@ -18,27 +20,52 @@ import ( networkingv1 "k8s.io/api/networking/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/net" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" + operatorutils "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" + "tailscale.com/kube/kubetypes" "tailscale.com/util/mak" "tailscale.com/util/set" ) const ( + reconcilerName = "dns-records-reconciler" + dnsRecordsRecocilerFinalizer = "tailscale.com/dns-records-reconciler" annotationTSMagicDNSName = "tailscale.com/magic-dnsname" - // Service types for consistent string usage + // Service types for consistent string usage. serviceTypeIngress = "ingress" serviceTypeSvc = "svc" + + optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again" + + // AnnotationTailnetTargetFQDN is the annotation used to configure an egress proxy's tailnet target FQDN. + AnnotationTailnetTargetFQDN = "tailscale.com/tailnet-fqdn" + + labelProxyGroup = "tailscale.com/proxy-group" + labelSvcType = "tailscale.com/svc-type" + typeEgress = "egress" ) -// dnsRecordsReconciler knows how to update dnsrecords ConfigMap with DNS -// records. +// ReconcilerOptions contains the options for creating a new Reconciler. +type ReconcilerOptions struct { + Client client.Client + TailscaleNamespace string + Logger *zap.SugaredLogger + IsDefaultLoadBalancer bool // true if operator is the default ingress controller in this cluster +} + +// Reconciler knows how to update dnsrecords ConfigMap with DNS records. // The records that it creates are: // - For tailscale Ingress, a mapping of the Ingress's MagicDNSName to the IP addresses // (both IPv4 and IPv6) of the ingress proxy Pod. @@ -48,23 +75,49 @@ const ( // Records will only be created if there is exactly one ready // tailscale.com/v1alpha1.DNSConfig instance in the cluster (so that we know // that there is a ts.net nameserver deployed in the cluster). -type dnsRecordsReconciler struct { +type Reconciler struct { client.Client - tsNamespace string // namespace in which we provision tailscale resources + tsNamespace string logger *zap.SugaredLogger - isDefaultLoadBalancer bool // true if operator is the default ingress controller in this cluster + isDefaultLoadBalancer bool +} + +// NewReconciler creates a new Reconciler. +func NewReconciler(options ReconcilerOptions) *Reconciler { + return &Reconciler{ + Client: options.Client, + tsNamespace: options.TailscaleNamespace, + logger: options.Logger.Named(reconcilerName), + isDefaultLoadBalancer: options.IsDefaultLoadBalancer, + } +} + +// Register registers the dnsrecords reconciler with the controller manager. +func (r *Reconciler) Register(mgr manager.Manager) error { + logger := r.logger.Named("event-handlers") + epsHandler := handler.EnqueueRequestsFromMapFunc(endpointSliceHandler) + dnsCfgHandler := handler.EnqueueRequestsFromMapFunc(enqueueAllIngressEgressProxySvcsInNS(r.tsNamespace, r.Client, logger)) + svcHandler := handler.EnqueueRequestsFromMapFunc(serviceHandler) + ingressHandler := handler.EnqueueRequestsFromMapFunc(ingressHandlerForNamespace(r.tsNamespace, r.isDefaultLoadBalancer, r.Client, logger)) + return builder.ControllerManagedBy(mgr). + Named(reconcilerName). + Watches(&corev1.Service{}, svcHandler). + Watches(&networkingv1.Ingress{}, ingressHandler). + Watches(&discoveryv1.EndpointSlice{}, epsHandler). + Watches(&tsapi.DNSConfig{}, dnsCfgHandler). + Complete(r) } // Reconcile takes a reconcile.Request for a Service fronting a // tailscale proxy and updates DNS Records in dnsrecords ConfigMap for the // in-cluster ts.net nameserver if required. -func (dnsRR *dnsRecordsReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) { - logger := dnsRR.logger.With("Service", req.NamespacedName) +func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) { + logger := r.logger.With("Service", req.NamespacedName) logger.Debugf("starting reconcile") defer logger.Debugf("reconcile finished") proxySvc := new(corev1.Service) - err = dnsRR.Client.Get(ctx, req.NamespacedName, proxySvc) + err = r.Client.Get(ctx, req.NamespacedName, proxySvc) if apierrors.IsNotFound(err) { logger.Debugf("Service not found") return reconcile.Result{}, nil @@ -72,21 +125,21 @@ func (dnsRR *dnsRecordsReconciler) Reconcile(ctx context.Context, req reconcile. if err != nil { return reconcile.Result{}, fmt.Errorf("failed to get Service: %w", err) } - if !(isManagedByType(proxySvc, serviceTypeSvc) || isManagedByType(proxySvc, serviceTypeIngress)) { + if !(reconciler.IsManagedByType(proxySvc, serviceTypeSvc) || reconciler.IsManagedByType(proxySvc, serviceTypeIngress)) { logger.Debugf("Service is not a proxy Service for a tailscale ingress or egress proxy; do nothing") return reconcile.Result{}, nil } if !proxySvc.DeletionTimestamp.IsZero() { logger.Debug("Service is being deleted, clean up resources") - return reconcile.Result{}, dnsRR.maybeCleanup(ctx, proxySvc, logger) + return reconcile.Result{}, r.maybeCleanup(ctx, proxySvc, logger) } // Check that there is a ts.net nameserver deployed to the cluster by // checking that there is tailscale.com/v1alpha1.DNSConfig resource in a // Ready state. dnsCfgLst := new(tsapi.DNSConfigList) - if err = dnsRR.List(ctx, dnsCfgLst); err != nil { + if err = r.List(ctx, dnsCfgLst); err != nil { return reconcile.Result{}, fmt.Errorf("error listing DNSConfigs: %w", err) } if len(dnsCfgLst.Items) == 0 { @@ -103,7 +156,7 @@ func (dnsRR *dnsRecordsReconciler) Reconcile(ctx context.Context, req reconcile. return reconcile.Result{}, nil } - if err := dnsRR.maybeProvision(ctx, proxySvc, logger); err != nil { + if err := r.maybeProvision(ctx, proxySvc, logger); err != nil { if strings.Contains(err.Error(), optimisticLockErrorMsg) { logger.Infof("optimistic lock error, retrying: %s", err) } else { @@ -136,12 +189,12 @@ func (dnsRR *dnsRecordsReconciler) Reconcile(ctx context.Context, req reconcile. // If records need to be created for this proxy, maybeProvision will also: // - update the Service with a tailscale.com/magic-dnsname annotation // - update the Service with a finalizer -func (dnsRR *dnsRecordsReconciler) maybeProvision(ctx context.Context, proxySvc *corev1.Service, logger *zap.SugaredLogger) error { - if !dnsRR.isInterestingService(ctx, proxySvc) { +func (r *Reconciler) maybeProvision(ctx context.Context, proxySvc *corev1.Service, logger *zap.SugaredLogger) error { + if !r.isInterestingService(ctx, proxySvc) { logger.Debug("Service is not fronting a proxy that we create DNS records for; do nothing") return nil } - fqdn, err := dnsRR.fqdnForDNSRecord(ctx, proxySvc, logger) + fqdn, err := r.fqdnForDNSRecord(ctx, proxySvc, logger) if err != nil { return fmt.Errorf("error determining DNS name for record: %w", err) } @@ -165,20 +218,20 @@ func (dnsRR *dnsRecordsReconciler) maybeProvision(ctx context.Context, proxySvc updateFunc := func(rec *operatorutils.Records) { delete(rec.IP4, oldFqdn) } - if err = dnsRR.updateDNSConfig(ctx, updateFunc); err != nil { + if err = r.updateDNSConfig(ctx, updateFunc); err != nil { return fmt.Errorf("error removing record for %s: %w", oldFqdn, err) } } mak.Set(&proxySvc.Annotations, annotationTSMagicDNSName, fqdn) if !apiequality.Semantic.DeepEqual(oldProxySvc, proxySvc) { logger.Infof("provisioning DNS record for MagicDNS name: %s", fqdn) // this will be printed exactly once - if err := dnsRR.Update(ctx, proxySvc); err != nil { + if err := r.Update(ctx, proxySvc); err != nil { return fmt.Errorf("error updating proxy Service metadata: %w", err) } } // Get the IP addresses for the DNS record - ip4s, ip6s, err := dnsRR.getTargetIPs(ctx, proxySvc, logger) + ip4s, ip6s, err := r.getTargetIPs(ctx, proxySvc, logger) if err != nil { return fmt.Errorf("error getting target IPs: %w", err) } @@ -195,7 +248,7 @@ func (dnsRR *dnsRecordsReconciler) maybeProvision(ctx context.Context, proxySvc mak.Set(&rec.IP6, fqdn, ip6s) } } - if err = dnsRR.updateDNSConfig(ctx, updateFunc); err != nil { + if err = r.updateDNSConfig(ctx, updateFunc); err != nil { return fmt.Errorf("error updating DNS records: %w", err) } return nil @@ -217,33 +270,33 @@ func epIsReady(ep *discoveryv1.Endpoint) bool { // has been removed from the Service. If the record is not found in the // ConfigMap, the ConfigMap does not exist, or the Service does not have // tailscale.com/magic-dnsname annotation, just remove the finalizer. -func (dnsRR *dnsRecordsReconciler) maybeCleanup(ctx context.Context, proxySvc *corev1.Service, logger *zap.SugaredLogger) error { +func (r *Reconciler) maybeCleanup(ctx context.Context, proxySvc *corev1.Service, logger *zap.SugaredLogger) error { ix := slices.Index(proxySvc.Finalizers, dnsRecordsRecocilerFinalizer) if ix == -1 { logger.Debugf("no finalizer, nothing to do") return nil } cm := &corev1.ConfigMap{} - err := dnsRR.Client.Get(ctx, types.NamespacedName{Name: operatorutils.DNSRecordsCMName, Namespace: dnsRR.tsNamespace}, cm) + err := r.Client.Get(ctx, types.NamespacedName{Name: operatorutils.DNSRecordsCMName, Namespace: r.tsNamespace}, cm) if apierrors.IsNotFound(err) { logger.Debug("'dnsrecords' ConfigMap not found") - return dnsRR.removeProxySvcFinalizer(ctx, proxySvc) + return r.removeProxySvcFinalizer(ctx, proxySvc) } if err != nil { return fmt.Errorf("error retrieving 'dnsrecords' ConfigMap: %w", err) } if cm.Data == nil { logger.Debug("'dnsrecords' ConfigMap contains no records") - return dnsRR.removeProxySvcFinalizer(ctx, proxySvc) + return r.removeProxySvcFinalizer(ctx, proxySvc) } _, ok := cm.Data[operatorutils.DNSRecordsCMKey] if !ok { logger.Debug("'dnsrecords' ConfigMap contains no records") - return dnsRR.removeProxySvcFinalizer(ctx, proxySvc) + return r.removeProxySvcFinalizer(ctx, proxySvc) } fqdn := proxySvc.GetAnnotations()[annotationTSMagicDNSName] if fqdn == "" { - return dnsRR.removeProxySvcFinalizer(ctx, proxySvc) + return r.removeProxySvcFinalizer(ctx, proxySvc) } logger.Infof("removing DNS record for MagicDNS name %s", fqdn) updateFunc := func(rec *operatorutils.Records) { @@ -252,33 +305,33 @@ func (dnsRR *dnsRecordsReconciler) maybeCleanup(ctx context.Context, proxySvc *c delete(rec.IP6, fqdn) } } - if err = dnsRR.updateDNSConfig(ctx, updateFunc); err != nil { + if err = r.updateDNSConfig(ctx, updateFunc); err != nil { return fmt.Errorf("error updating DNS config: %w", err) } - return dnsRR.removeProxySvcFinalizer(ctx, proxySvc) + return r.removeProxySvcFinalizer(ctx, proxySvc) } -func (dnsRR *dnsRecordsReconciler) removeProxySvcFinalizer(ctx context.Context, proxySvc *corev1.Service) error { +func (r *Reconciler) removeProxySvcFinalizer(ctx context.Context, proxySvc *corev1.Service) error { idx := slices.Index(proxySvc.Finalizers, dnsRecordsRecocilerFinalizer) if idx == -1 { return nil } proxySvc.Finalizers = slices.Delete(proxySvc.Finalizers, idx, idx+1) - return dnsRR.Update(ctx, proxySvc) + return r.Update(ctx, proxySvc) } // fqdnForDNSRecord returns MagicDNS name associated with a given proxy Service. // If the proxy Service is for a tailscale Ingress proxy, returns ingress.status.loadBalancer.ingress.hostname. -// If the proxy Service is for an tailscale egress proxy configured via tailscale.com/tailnet-fqdn annotation, returns the annotation value. +// If the proxy Service is for a tailscale egress proxy configured via tailscale.com/tailnet-fqdn annotation, returns the annotation value. // For ProxyGroup egress Services, returns the tailnet-fqdn annotation from the parent Service. // This function is not expected to be called with proxy Services for other // proxy types, or any other Services, but it just returns an empty string if // that happens. -func (dnsRR *dnsRecordsReconciler) fqdnForDNSRecord(ctx context.Context, proxySvc *corev1.Service, logger *zap.SugaredLogger) (string, error) { - parentName := parentFromObjectLabels(proxySvc) - if isManagedByType(proxySvc, serviceTypeIngress) { +func (r *Reconciler) fqdnForDNSRecord(ctx context.Context, proxySvc *corev1.Service, logger *zap.SugaredLogger) (string, error) { + parentName := reconciler.ParentFromObjectLabels(proxySvc) + if reconciler.IsManagedByType(proxySvc, serviceTypeIngress) { ing := new(networkingv1.Ingress) - if err := dnsRR.Get(ctx, parentName, ing); err != nil { + if err := r.Get(ctx, parentName, ing); err != nil { return "", err } if len(ing.Status.LoadBalancer.Ingress) == 0 { @@ -286,9 +339,9 @@ func (dnsRR *dnsRecordsReconciler) fqdnForDNSRecord(ctx context.Context, proxySv } return ing.Status.LoadBalancer.Ingress[0].Hostname, nil } - if isManagedByType(proxySvc, serviceTypeSvc) { + if reconciler.IsManagedByType(proxySvc, serviceTypeSvc) { svc := new(corev1.Service) - if err := dnsRR.Get(ctx, parentName, svc); apierrors.IsNotFound(err) { + if err := r.Get(ctx, parentName, svc); apierrors.IsNotFound(err) { logger.Infof("[unexpected] parent Service for egress proxy %s not found", proxySvc.Name) return "", nil } else if err != nil { @@ -302,11 +355,11 @@ func (dnsRR *dnsRecordsReconciler) fqdnForDNSRecord(ctx context.Context, proxySv // updateDNSConfig runs the provided update function against dnsrecords // ConfigMap. At this point the in-cluster ts.net nameserver is expected to be // successfully created together with the ConfigMap. -func (dnsRR *dnsRecordsReconciler) updateDNSConfig(ctx context.Context, update func(*operatorutils.Records)) error { +func (r *Reconciler) updateDNSConfig(ctx context.Context, update func(*operatorutils.Records)) error { cm := &corev1.ConfigMap{} - err := dnsRR.Get(ctx, types.NamespacedName{Name: operatorutils.DNSRecordsCMName, Namespace: dnsRR.tsNamespace}, cm) + err := r.Get(ctx, types.NamespacedName{Name: operatorutils.DNSRecordsCMName, Namespace: r.tsNamespace}, cm) if apierrors.IsNotFound(err) { - dnsRR.logger.Info("[unexpected] dnsrecords ConfigMap not found in cluster. Not updating DNS records. Please open an issue and attach operator logs.") + r.logger.Info("[unexpected] dnsrecords ConfigMap not found in cluster. Not updating DNS records. Please open an issue and attach operator logs.") return nil } if err != nil { @@ -324,19 +377,19 @@ func (dnsRR *dnsRecordsReconciler) updateDNSConfig(ctx context.Context, update f return fmt.Errorf("error marshalling DNS records: %w", err) } mak.Set(&cm.Data, operatorutils.DNSRecordsCMKey, string(dnsRecordsBs)) - return dnsRR.Update(ctx, cm) + return r.Update(ctx, cm) } // isSvcForFQDNEgressProxy returns true if the Service is a headless Service // created for a proxy for a tailscale egress Service configured via // tailscale.com/tailnet-fqdn annotation. -func (dnsRR *dnsRecordsReconciler) isSvcForFQDNEgressProxy(ctx context.Context, svc *corev1.Service) (bool, error) { - if !isManagedByType(svc, "svc") { +func (r *Reconciler) isSvcForFQDNEgressProxy(ctx context.Context, svc *corev1.Service) (bool, error) { + if !reconciler.IsManagedByType(svc, "svc") { return false, nil } - parentName := parentFromObjectLabels(svc) + parentName := reconciler.ParentFromObjectLabels(svc) parentSvc := new(corev1.Service) - if err := dnsRR.Get(ctx, parentName, parentSvc); apierrors.IsNotFound(err) { + if err := r.Get(ctx, parentName, parentSvc); apierrors.IsNotFound(err) { return false, nil } else if err != nil { return false, err @@ -349,21 +402,21 @@ func (dnsRR *dnsRecordsReconciler) isSvcForFQDNEgressProxy(ctx context.Context, // created for ProxyGroup egress. For ProxyGroup egress, there are no headless // services. Instead, the DNS reconciler processes the ClusterIP Service // directly, which has portmapping and should use its own IP for DNS records. -func (dnsRR *dnsRecordsReconciler) isProxyGroupEgressService(svc *corev1.Service) bool { +func (r *Reconciler) isProxyGroupEgressService(svc *corev1.Service) bool { return svc.GetLabels()[labelProxyGroup] != "" && svc.GetLabels()[labelSvcType] == typeEgress && svc.Spec.Type == corev1.ServiceTypeClusterIP && - isManagedByType(svc, serviceTypeSvc) + reconciler.IsManagedByType(svc, serviceTypeSvc) } // isInterestingService reports whether the Service is one that we should create // DNS records for. -func (dnsRR *dnsRecordsReconciler) isInterestingService(ctx context.Context, svc *corev1.Service) bool { - if isManagedByType(svc, serviceTypeIngress) { +func (r *Reconciler) isInterestingService(ctx context.Context, svc *corev1.Service) bool { + if reconciler.IsManagedByType(svc, serviceTypeIngress) { return true } - isEgressFQDNSvc, err := dnsRR.isSvcForFQDNEgressProxy(ctx, svc) + isEgressFQDNSvc, err := r.isSvcForFQDNEgressProxy(ctx, svc) if err != nil { return false } @@ -371,8 +424,8 @@ func (dnsRR *dnsRecordsReconciler) isInterestingService(ctx context.Context, svc return true } - if dnsRR.isProxyGroupEgressService(svc) { - return dnsRR.parentSvcTargetsFQDN(ctx, svc) + if r.isProxyGroupEgressService(svc) { + return r.parentSvcTargetsFQDN(ctx, svc) } return false @@ -380,29 +433,27 @@ func (dnsRR *dnsRecordsReconciler) isInterestingService(ctx context.Context, svc // parentSvcTargetsFQDN reports whether the parent Service of a ProxyGroup // egress Service has an FQDN target (not an IP target). -func (dnsRR *dnsRecordsReconciler) parentSvcTargetsFQDN(ctx context.Context, svc *corev1.Service) bool { - - parentName := parentFromObjectLabels(svc) +func (r *Reconciler) parentSvcTargetsFQDN(ctx context.Context, svc *corev1.Service) bool { + parentName := reconciler.ParentFromObjectLabels(svc) parentSvc := new(corev1.Service) - if err := dnsRR.Get(ctx, parentName, parentSvc); err != nil { + if err := r.Get(ctx, parentName, parentSvc); err != nil { return false } - return parentSvc.Annotations[AnnotationTailnetTargetFQDN] != "" } // getTargetIPs returns the IPv4 and IPv6 addresses that should be used for DNS records // for the given proxy Service. -func (dnsRR *dnsRecordsReconciler) getTargetIPs(ctx context.Context, proxySvc *corev1.Service, logger *zap.SugaredLogger) ([]string, []string, error) { - if dnsRR.isProxyGroupEgressService(proxySvc) { - return dnsRR.getClusterIPServiceIPs(proxySvc, logger) +func (r *Reconciler) getTargetIPs(ctx context.Context, proxySvc *corev1.Service, logger *zap.SugaredLogger) ([]string, []string, error) { + if r.isProxyGroupEgressService(proxySvc) { + return r.getClusterIPServiceIPs(proxySvc, logger) } - return dnsRR.getPodIPs(ctx, proxySvc, logger) + return r.getPodIPs(ctx, proxySvc, logger) } // getClusterIPServiceIPs returns the ClusterIPs of a ProxyGroup egress Service. // It separates IPv4 and IPv6 addresses for dual-stack services. -func (dnsRR *dnsRecordsReconciler) getClusterIPServiceIPs(proxySvc *corev1.Service, logger *zap.SugaredLogger) ([]string, []string, error) { +func (r *Reconciler) getClusterIPServiceIPs(proxySvc *corev1.Service, logger *zap.SugaredLogger) ([]string, []string, error) { // Handle services with no ClusterIP if proxySvc.Spec.ClusterIP == "" || proxySvc.Spec.ClusterIP == "None" { logger.Debugf("ProxyGroup egress ClusterIP Service does not have a ClusterIP yet.") @@ -438,13 +489,13 @@ func (dnsRR *dnsRecordsReconciler) getClusterIPServiceIPs(proxySvc *corev1.Servi } // getPodIPs returns Pod IPv4 and IPv6 addresses from EndpointSlices for non-ProxyGroup Services. -func (dnsRR *dnsRecordsReconciler) getPodIPs(ctx context.Context, proxySvc *corev1.Service, logger *zap.SugaredLogger) ([]string, []string, error) { +func (r *Reconciler) getPodIPs(ctx context.Context, proxySvc *corev1.Service, logger *zap.SugaredLogger) ([]string, []string, error) { // Get the Pod IP addresses for the proxy from the EndpointSlices for // the headless Service. The Service can have multiple EndpointSlices // associated with it, for example in dual-stack clusters. labels := map[string]string{discoveryv1.LabelServiceName: proxySvc.Name} // https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/#ownership var eps = new(discoveryv1.EndpointSliceList) - if err := dnsRR.List(ctx, eps, client.InNamespace(dnsRR.tsNamespace), client.MatchingLabels(labels)); err != nil { + if err := r.List(ctx, eps, client.InNamespace(r.tsNamespace), client.MatchingLabels(labels)); err != nil { return nil, nil, fmt.Errorf("error listing EndpointSlices for the proxy's Service: %w", err) } if len(eps.Items) == 0 { @@ -494,3 +545,132 @@ func (dnsRR *dnsRecordsReconciler) getPodIPs(ctx context.Context, proxySvc *core } return ip4s.Slice(), ip6s.Slice(), nil } + +// endpointSliceHandler filters EndpointSlice events for which +// dns-records-reconciler should reconcile a headless Service. The only events +// it should reconcile are those for EndpointSlices associated with proxy +// headless Services. +func endpointSliceHandler(ctx context.Context, o client.Object) []reconcile.Request { + if !reconciler.IsManagedByType(o, "svc") && !reconciler.IsManagedByType(o, "ingress") { + return nil + } + headlessSvcName, ok := o.GetLabels()[discoveryv1.LabelServiceName] // https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/#ownership + if !ok { + return nil + } + return []reconcile.Request{{NamespacedName: types.NamespacedName{Namespace: o.GetNamespace(), Name: headlessSvcName}}} +} + +// serviceHandler filters Service events for which dns-records-reconciler +// should reconcile. If the event is for a cluster ingress/cluster egress +// proxy's headless Service, returns the Service for reconcile. +func serviceHandler(ctx context.Context, o client.Object) []reconcile.Request { + if reconciler.IsManagedByType(o, "svc") || reconciler.IsManagedByType(o, "ingress") { + return []reconcile.Request{{NamespacedName: types.NamespacedName{Namespace: o.GetNamespace(), Name: o.GetName()}}} + } + return nil +} + +// ingressHandlerForNamespace filters Ingress events to ensure that +// dns-records-reconciler only reconciles on tailscale Ingress events. When an +// event is observed on a tailscale Ingress, reconcile the proxy headless Service. +func ingressHandlerForNamespace(ns string, isDefaultLoadBalancer bool, cl client.Client, logger *zap.SugaredLogger) handler.MapFunc { + return func(ctx context.Context, o client.Object) []reconcile.Request { + ing, ok := o.(*networkingv1.Ingress) + if !ok { + return nil + } + if !isDefaultLoadBalancer && (ing.Spec.IngressClassName == nil || *ing.Spec.IngressClassName != "tailscale") { + return nil + } + proxyResourceLabels := reconciler.ChildResourceLabels(ing.Name, ing.Namespace, "ingress") + headlessSvc, err := getSingleObject[corev1.Service](ctx, cl, ns, proxyResourceLabels) + if err != nil { + logger.Errorf("error getting headless Service from parent labels: %v", err) + return nil + } + if headlessSvc == nil { + return nil + } + return []reconcile.Request{{NamespacedName: types.NamespacedName{Namespace: headlessSvc.Namespace, Name: headlessSvc.Name}}} + } +} + +// enqueueAllIngressEgressProxySvcsInNS returns a handler.MapFunc that on +// DNSConfig changes enqueues all headless Services for ingress/egress proxies +// in the operator namespace. +func enqueueAllIngressEgressProxySvcsInNS(ns string, cl client.Client, logger *zap.SugaredLogger) handler.MapFunc { + return func(ctx context.Context, _ client.Object) []reconcile.Request { + reqs := make([]reconcile.Request, 0) + + // Get all headless Services for proxies configured using Service. + svcProxyLabels := map[string]string{ + kubetypes.LabelManaged: "true", + reconciler.LabelParentType: "svc", + } + svcHeadlessSvcList := &corev1.ServiceList{} + if err := cl.List(ctx, svcHeadlessSvcList, client.InNamespace(ns), client.MatchingLabels(svcProxyLabels)); err != nil { + logger.Errorf("error listing headless Services for tailscale ingress/egress Services in operator namespace: %v", err) + return nil + } + for _, svc := range svcHeadlessSvcList.Items { + reqs = append(reqs, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: svc.Namespace, Name: svc.Name}}) + } + + // Get all headless Services for proxies configured using Ingress. + ingProxyLabels := map[string]string{ + kubetypes.LabelManaged: "true", + reconciler.LabelParentType: "ingress", + } + ingHeadlessSvcList := &corev1.ServiceList{} + if err := cl.List(ctx, ingHeadlessSvcList, client.InNamespace(ns), client.MatchingLabels(ingProxyLabels)); err != nil { + logger.Errorf("error listing headless Services for tailscale Ingresses in operator namespace: %v", err) + return nil + } + for _, svc := range ingHeadlessSvcList.Items { + reqs = append(reqs, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: svc.Namespace, Name: svc.Name}}) + } + return reqs + } +} + +type ptrObject[T any] interface { + client.Object + *T +} + +// getSingleObject searches for k8s objects of type T with the given labels, +// and returns it. Returns nil if no objects match the labels, and an error if +// more than one object matches. +func getSingleObject[T any, O ptrObject[T]](ctx context.Context, c client.Client, ns string, labels map[string]string) (O, error) { + ret := O(new(T)) + kinds, _, err := c.Scheme().ObjectKinds(ret) + if err != nil { + return nil, err + } + if len(kinds) != 1 { + return nil, fmt.Errorf("more than 1 GroupVersionKind for %T", ret) + } + + gvk := kinds[0] + gvk.Kind += "List" + lst := unstructured.UnstructuredList{} + lst.SetGroupVersionKind(gvk) + if err := c.List(ctx, &lst, client.InNamespace(ns), client.MatchingLabels(labels)); err != nil { + return nil, err + } + + if len(lst.Items) == 0 { + return nil, nil + } + if len(lst.Items) > 1 { + return nil, fmt.Errorf("found multiple matching %T objects", ret) + } + + item := lst.Items[0] + ret2 := O(new(T)) + if err := c.Scheme().Convert(&item, ret2, nil); err != nil { + return nil, err + } + return ret2, nil +} diff --git a/cmd/k8s-operator/dnsrecords_test.go b/k8s-operator/reconciler/dnsrecords/dnsrecords_test.go index c6c5ee029..9bc268fc9 100644 --- a/cmd/k8s-operator/dnsrecords_test.go +++ b/k8s-operator/reconciler/dnsrecords/dnsrecords_test.go @@ -3,7 +3,7 @@ //go:build !plan9 -package main +package dnsrecords import ( "context" @@ -21,8 +21,12 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + operatorutils "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" + "tailscale.com/k8s-operator/reconciler/nameserver" "tailscale.com/kube/kubetypes" "tailscale.com/tstest" ) @@ -67,9 +71,9 @@ func TestDNSRecordsReconciler(t *testing.T) { cl := tstest.NewClock(tstest.ClockOpts{}) // Set the ready condition of the DNSConfig mustUpdateStatus(t, fc, "", "test", func(c *tsapi.DNSConfig) { - operatorutils.SetDNSConfigCondition(c, tsapi.NameserverReady, metav1.ConditionTrue, reasonNameserverCreated, reasonNameserverCreated, 0, cl, zl.Sugar()) + operatorutils.SetDNSConfigCondition(c, tsapi.NameserverReady, metav1.ConditionTrue, nameserver.ReasonNameserverCreated, nameserver.ReasonNameserverCreated, 0, cl, zl.Sugar()) }) - dnsRR := &dnsRecordsReconciler{ + dnsRR := &Reconciler{ Client: fc, logger: zl.Sugar(), tsNamespace: "tailscale", @@ -182,12 +186,12 @@ func TestDNSRecordsReconciler(t *testing.T) { Name: "ts-proxygroup-egress-abcd1", Namespace: "tailscale", Labels: map[string]string{ - kubetypes.LabelManaged: "true", - LabelParentName: "external-service", - LabelParentNamespace: "default", - LabelParentType: "svc", - labelProxyGroup: "test-proxy-group", - labelSvcType: typeEgress, + kubetypes.LabelManaged: "true", + reconciler.LabelParentName: "external-service", + reconciler.LabelParentNamespace: "default", + reconciler.LabelParentType: "svc", + labelProxyGroup: "test-proxy-group", + labelSvcType: typeEgress, }, }, Spec: corev1.ServiceSpec{ @@ -206,13 +210,13 @@ func TestDNSRecordsReconciler(t *testing.T) { Name: "ts-proxygroup-egress-abcd1-ipv4", Namespace: "tailscale", Labels: map[string]string{ - discoveryv1.LabelServiceName: "ts-proxygroup-egress-abcd1", - kubetypes.LabelManaged: "true", - LabelParentName: "external-service", - LabelParentNamespace: "default", - LabelParentType: "svc", - labelProxyGroup: "test-proxy-group", - labelSvcType: typeEgress, + discoveryv1.LabelServiceName: "ts-proxygroup-egress-abcd1", + kubetypes.LabelManaged: "true", + reconciler.LabelParentName: "external-service", + reconciler.LabelParentNamespace: "default", + reconciler.LabelParentType: "svc", + labelProxyGroup: "test-proxy-group", + labelSvcType: typeEgress, }, }, AddressType: discoveryv1.AddressTypeIPv4, @@ -260,7 +264,7 @@ func TestDNSRecordsReconcilerErrorCases(t *testing.T) { t.Fatal(err) } - dnsRR := &dnsRecordsReconciler{ + r := &Reconciler{ logger: zl.Sugar(), } @@ -271,14 +275,14 @@ func TestDNSRecordsReconcilerErrorCases(t *testing.T) { // Test invalid IP format testSvc.Spec.ClusterIP = "invalid-ip" - _, _, err = dnsRR.getClusterIPServiceIPs(testSvc, zl.Sugar()) + _, _, err = r.getClusterIPServiceIPs(testSvc, zl.Sugar()) if err == nil { t.Error("expected error for invalid IP format") } // Test valid IP testSvc.Spec.ClusterIP = "10.0.100.50" - ip4s, ip6s, err := dnsRR.getClusterIPServiceIPs(testSvc, zl.Sugar()) + ip4s, ip6s, err := r.getClusterIPServiceIPs(testSvc, zl.Sugar()) if err != nil { t.Errorf("unexpected error for valid IP: %v", err) } @@ -329,21 +333,16 @@ func TestDNSRecordsReconcilerDualStack(t *testing.T) { headlessSvc := headlessSvcForParent(ing, "ingress") headlessSvc.Name = "ts-dual-stack-ingress" headlessSvc.SetLabels(map[string]string{ - kubetypes.LabelManaged: "true", - LabelParentName: "dual-stack-ingress", - LabelParentNamespace: "test", - LabelParentType: "ingress", + kubetypes.LabelManaged: "true", + reconciler.LabelParentName: "dual-stack-ingress", + reconciler.LabelParentNamespace: "test", + reconciler.LabelParentType: "ingress", }) // Create both IPv4 and IPv6 endpoints epv4 := endpointSliceForService(headlessSvc, "10.1.2.3", discoveryv1.AddressTypeIPv4) epv6 := endpointSliceForService(headlessSvc, "2001:db8::1", discoveryv1.AddressTypeIPv6) - dnsRRDualStack := &dnsRecordsReconciler{ - tsNamespace: "tailscale", - logger: zl.Sugar(), - } - // Create the dnsrecords ConfigMap cm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ @@ -358,7 +357,11 @@ func TestDNSRecordsReconcilerDualStack(t *testing.T) { WithStatusSubresource(dnsCfg). Build() - dnsRRDualStack.Client = fc + dnsRRDualStack := &Reconciler{ + Client: fc, + tsNamespace: "tailscale", + logger: zl.Sugar(), + } // Test dual-stack service records expectReconciled(t, dnsRRDualStack, "tailscale", "ts-dual-stack-ingress") @@ -388,12 +391,12 @@ func TestDNSRecordsReconcilerDualStack(t *testing.T) { Name: "ts-proxygroup-dualstack", Namespace: "tailscale", Labels: map[string]string{ - kubetypes.LabelManaged: "true", - labelProxyGroup: "test-pg", - labelSvcType: typeEgress, - LabelParentName: "pg-service", - LabelParentNamespace: "tailscale", - LabelParentType: "svc", + kubetypes.LabelManaged: "true", + labelProxyGroup: "test-pg", + labelSvcType: typeEgress, + reconciler.LabelParentName: "pg-service", + reconciler.LabelParentNamespace: "tailscale", + reconciler.LabelParentType: "svc", }, Annotations: map[string]string{ annotationTSMagicDNSName: "pg-service.example.ts.net", @@ -421,10 +424,10 @@ func headlessSvcForParent(o client.Object, typ string) *corev1.Service { Name: o.GetName(), Namespace: "tailscale", Labels: map[string]string{ - kubetypes.LabelManaged: "true", - LabelParentName: o.GetName(), - LabelParentNamespace: o.GetNamespace(), - LabelParentType: typ, + kubetypes.LabelManaged: "true", + reconciler.LabelParentName: o.GetName(), + reconciler.LabelParentNamespace: o.GetNamespace(), + reconciler.LabelParentType: typ, }, }, Spec: corev1.ServiceSpec{ @@ -500,3 +503,63 @@ func expectHostsRecordsWithIPv6(t *testing.T, cl client.Client, wantsHostsIPv4, t.Fatalf("unexpected IPv6 dns config (-got +want):\n%s", diff) } } + +func expectReconciled(t *testing.T, r *Reconciler, ns, name string) { + t.Helper() + req := reconcile.Request{ + NamespacedName: types.NamespacedName{Namespace: ns, Name: name}, + } + res, err := r.Reconcile(context.Background(), req) + if err != nil { + t.Fatalf("Reconcile: unexpected error: %v", err) + } + if res.Requeue { + t.Fatalf("unexpected immediate requeue") + } +} + +func mustCreate(t *testing.T, c client.Client, obj client.Object) { + t.Helper() + if err := c.Create(context.Background(), obj); err != nil { + t.Fatalf("creating %q: %v", obj.GetName(), err) + } +} + +func mustDeleteAll(t *testing.T, c client.Client, objs ...client.Object) { + t.Helper() + for _, obj := range objs { + if err := c.Delete(context.Background(), obj); err != nil { + t.Fatalf("deleting %q: %v", obj.GetName(), err) + } + } +} + +func mustUpdate[T any, O interface { + client.Object + *T +}](t *testing.T, c client.Client, ns, name string, update func(O)) { + t.Helper() + obj := O(new(T)) + if err := c.Get(context.Background(), types.NamespacedName{Name: name, Namespace: ns}, obj); err != nil { + t.Fatalf("getting %q: %v", name, err) + } + update(obj) + if err := c.Update(context.Background(), obj); err != nil { + t.Fatalf("updating %q: %v", name, err) + } +} + +func mustUpdateStatus[T any, O interface { + client.Object + *T +}](t *testing.T, c client.Client, ns, name string, update func(O)) { + t.Helper() + obj := O(new(T)) + if err := c.Get(context.Background(), types.NamespacedName{Name: name, Namespace: ns}, obj); err != nil { + t.Fatalf("getting %q: %v", name, err) + } + update(obj) + if err := c.Status().Update(context.Background(), obj); err != nil { + t.Fatalf("updating status %q: %v", name, err) + } +} diff --git a/k8s-operator/reconciler/nameserver/manifests/cm.yaml b/k8s-operator/reconciler/nameserver/manifests/cm.yaml new file mode 100644 index 000000000..43bc5d0d5 --- /dev/null +++ b/k8s-operator/reconciler/nameserver/manifests/cm.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: dnsrecords diff --git a/k8s-operator/reconciler/nameserver/manifests/deploy.yaml b/k8s-operator/reconciler/nameserver/manifests/deploy.yaml new file mode 100644 index 000000000..f8794b064 --- /dev/null +++ b/k8s-operator/reconciler/nameserver/manifests/deploy.yaml @@ -0,0 +1,37 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nameserver +spec: + replicas: 1 + revisionHistoryLimit: 5 + selector: + matchLabels: + app: nameserver + strategy: + type: Recreate + template: + metadata: + labels: + app: nameserver + spec: + containers: + - imagePullPolicy: IfNotPresent + name: nameserver + ports: + - name: tcp + protocol: TCP + containerPort: 1053 + - name: udp + protocol: UDP + containerPort: 1053 + volumeMounts: + - name: dnsrecords + mountPath: /config + restartPolicy: Always + serviceAccount: nameserver + serviceAccountName: nameserver + volumes: + - name: dnsrecords + configMap: + name: dnsrecords diff --git a/k8s-operator/reconciler/nameserver/manifests/sa.yaml b/k8s-operator/reconciler/nameserver/manifests/sa.yaml new file mode 100644 index 000000000..96edece2c --- /dev/null +++ b/k8s-operator/reconciler/nameserver/manifests/sa.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: nameserver diff --git a/k8s-operator/reconciler/nameserver/manifests/svc.yaml b/k8s-operator/reconciler/nameserver/manifests/svc.yaml new file mode 100644 index 000000000..67e79a138 --- /dev/null +++ b/k8s-operator/reconciler/nameserver/manifests/svc.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: nameserver +spec: + selector: + app: nameserver + ports: + - name: udp + targetPort: 1053 + port: 53 + protocol: UDP + - name: tcp + targetPort: 1053 + port: 53 + protocol: TCP diff --git a/cmd/k8s-operator/nameserver.go b/k8s-operator/reconciler/nameserver/nameserver.go index 869e5bb26..132c3488e 100644 --- a/cmd/k8s-operator/nameserver.go +++ b/k8s-operator/reconciler/nameserver/nameserver.go @@ -3,7 +3,9 @@ //go:build !plan9 -package main +// Package nameserver provides reconciliation logic for the DNSConfig custom resource definition. +// It is responsible for creating and managing nameserver resources in response to DNSConfig objects. +package nameserver import ( "context" @@ -15,20 +17,24 @@ import ( "sync" "go.uber.org/zap" - xslices "golang.org/x/exp/slices" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/yaml" tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" "tailscale.com/kube/kubetypes" "tailscale.com/tstime" "tailscale.com/util/clientmetric" @@ -36,21 +42,37 @@ import ( ) const ( + reconcilerName = "nameserver-reconciler" + reasonNameserverCreationFailed = "NameserverCreationFailed" reasonMultipleDNSConfigsPresent = "MultipleDNSConfigsPresent" - reasonNameserverCreated = "NameserverCreated" + // ReasonNameserverCreated is the condition reason set when nameserver resources have been created successfully. + ReasonNameserverCreated = "NameserverCreated" messageNameserverCreationFailed = "Failed creating nameserver resources: %v" messageMultipleDNSConfigsPresent = "Multiple DNSConfig resources found in cluster. Please ensure no more than one is present." defaultNameserverImageRepo = "tailscale/k8s-nameserver" defaultNameserverImageTag = "stable" + + optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again" ) -// NameserverReconciler knows how to create nameserver resources in cluster in +var gaugeNameserverResources = clientmetric.NewGauge(kubetypes.MetricNameserverCount) + +// ReconcilerOptions contains the options for creating a new Reconciler. +type ReconcilerOptions struct { + Client client.Client + Recorder record.EventRecorder + TailscaleNamespace string + Logger *zap.SugaredLogger + Clock tstime.Clock +} + +// Reconciler knows how to create nameserver resources in cluster in // response to users applying DNSConfig. -type NameserverReconciler struct { +type Reconciler struct { client.Client logger *zap.SugaredLogger recorder record.EventRecorder @@ -61,15 +83,37 @@ type NameserverReconciler struct { managedNameservers set.Slice[types.UID] // one or none } -var gaugeNameserverResources = clientmetric.NewGauge(kubetypes.MetricNameserverCount) +// NewReconciler creates a new Reconciler. +func NewReconciler(options ReconcilerOptions) *Reconciler { + return &Reconciler{ + Client: options.Client, + recorder: options.Recorder, + tsNamespace: options.TailscaleNamespace, + logger: options.Logger.Named(reconcilerName), + clock: options.Clock, + } +} -func (a *NameserverReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) { - logger := a.logger.With("dnsConfig", req.Name) +// Register registers the nameserver reconciler with the controller manager. +func (r *Reconciler) Register(mgr manager.Manager) error { + nameserverFilter := handler.EnqueueRequestsFromMapFunc(reconciler.ManagedResourceHandlerForType("nameserver")) + return builder.ControllerManagedBy(mgr). + For(&tsapi.DNSConfig{}). + Named(reconcilerName). + Watches(&appsv1.Deployment{}, nameserverFilter). + Watches(&corev1.ConfigMap{}, nameserverFilter). + Watches(&corev1.Service{}, nameserverFilter). + Watches(&corev1.ServiceAccount{}, nameserverFilter). + Complete(r) +} + +func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) { + logger := r.logger.With("dnsConfig", req.Name) logger.Debugf("starting reconcile") defer logger.Debugf("reconcile finished") var dnsCfg tsapi.DNSConfig - err = a.Get(ctx, req.NamespacedName, &dnsCfg) + err = r.Get(ctx, req.NamespacedName, &dnsCfg) if apierrors.IsNotFound(err) { // Request object not found, could have been deleted after reconcile request. logger.Debugf("dnsconfig not found, assuming it was deleted") @@ -78,18 +122,18 @@ func (a *NameserverReconciler) Reconcile(ctx context.Context, req reconcile.Requ return reconcile.Result{}, fmt.Errorf("failed to get dnsconfig: %w", err) } if !dnsCfg.DeletionTimestamp.IsZero() { - ix := xslices.Index(dnsCfg.Finalizers, FinalizerName) + ix := slices.Index(dnsCfg.Finalizers, reconciler.FinalizerName) if ix < 0 { logger.Debugf("no finalizer, nothing to do") return reconcile.Result{}, nil } logger.Info("Cleaning up DNSConfig resources") - if err := a.maybeCleanup(&dnsCfg); err != nil { + if err := r.maybeCleanup(&dnsCfg); err != nil { logger.Errorf("error cleaning up reconciler resource: %v", err) return res, err } dnsCfg.Finalizers = append(dnsCfg.Finalizers[:ix], dnsCfg.Finalizers[ix+1:]...) - if err := a.Update(ctx, &dnsCfg); err != nil { + if err := r.Update(ctx, &dnsCfg); err != nil { logger.Errorf("error removing finalizer: %v", err) return reconcile.Result{}, err } @@ -99,36 +143,36 @@ func (a *NameserverReconciler) Reconcile(ctx context.Context, req reconcile.Requ oldCnStatus := dnsCfg.Status.DeepCopy() setStatus := func(dnsCfg *tsapi.DNSConfig, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) { - tsoperator.SetDNSConfigCondition(dnsCfg, tsapi.NameserverReady, status, reason, message, dnsCfg.Generation, a.clock, logger) + tsoperator.SetDNSConfigCondition(dnsCfg, tsapi.NameserverReady, status, reason, message, dnsCfg.Generation, r.clock, logger) if !apiequality.Semantic.DeepEqual(oldCnStatus, &dnsCfg.Status) { // An error encountered here should get returned by the Reconcile function. - if updateErr := a.Client.Status().Update(ctx, dnsCfg); updateErr != nil { + if updateErr := r.Client.Status().Update(ctx, dnsCfg); updateErr != nil { err = errors.Join(err, updateErr) } } return res, err } var dnsCfgs tsapi.DNSConfigList - if err := a.List(ctx, &dnsCfgs); err != nil { + if err := r.List(ctx, &dnsCfgs); err != nil { return res, fmt.Errorf("error listing DNSConfigs: %w", err) } if len(dnsCfgs.Items) > 1 { // enforce DNSConfig to be a singleton msg := "invalid cluster configuration: more than one tailscale.com/dnsconfigs found. Please ensure that no more than one is created." logger.Error(msg) - a.recorder.Event(&dnsCfg, corev1.EventTypeWarning, reasonMultipleDNSConfigsPresent, messageMultipleDNSConfigsPresent) + r.recorder.Event(&dnsCfg, corev1.EventTypeWarning, reasonMultipleDNSConfigsPresent, messageMultipleDNSConfigsPresent) setStatus(&dnsCfg, metav1.ConditionFalse, reasonMultipleDNSConfigsPresent, messageMultipleDNSConfigsPresent) } - if !slices.Contains(dnsCfg.Finalizers, FinalizerName) { + if !slices.Contains(dnsCfg.Finalizers, reconciler.FinalizerName) { logger.Infof("ensuring nameserver resources") - dnsCfg.Finalizers = append(dnsCfg.Finalizers, FinalizerName) - if err := a.Update(ctx, &dnsCfg); err != nil { + dnsCfg.Finalizers = append(dnsCfg.Finalizers, reconciler.FinalizerName) + if err := r.Update(ctx, &dnsCfg); err != nil { msg := fmt.Sprintf(messageNameserverCreationFailed, err) logger.Error(msg) return setStatus(&dnsCfg, metav1.ConditionFalse, reasonNameserverCreationFailed, msg) } } - if err = a.maybeProvision(ctx, &dnsCfg); err != nil { + if err = r.maybeProvision(ctx, &dnsCfg); err != nil { if strings.Contains(err.Error(), optimisticLockErrorMsg) { logger.Infof("optimistic lock error, retrying: %s", err) return reconcile.Result{}, nil @@ -137,39 +181,39 @@ func (a *NameserverReconciler) Reconcile(ctx context.Context, req reconcile.Requ } } - a.mu.Lock() - a.managedNameservers.Add(dnsCfg.UID) - a.mu.Unlock() - gaugeNameserverResources.Set(int64(a.managedNameservers.Len())) + r.mu.Lock() + r.managedNameservers.Add(dnsCfg.UID) + r.mu.Unlock() + gaugeNameserverResources.Set(int64(r.managedNameservers.Len())) svc := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{Name: "nameserver", Namespace: a.tsNamespace}, + ObjectMeta: metav1.ObjectMeta{Name: "nameserver", Namespace: r.tsNamespace}, } - if err := a.Client.Get(ctx, client.ObjectKeyFromObject(svc), svc); err != nil { + if err := r.Client.Get(ctx, client.ObjectKeyFromObject(svc), svc); err != nil { return res, fmt.Errorf("error getting Service: %w", err) } if ip := svc.Spec.ClusterIP; ip != "" && ip != "None" { dnsCfg.Status.Nameserver = &tsapi.NameserverStatus{ IP: ip, } - return setStatus(&dnsCfg, metav1.ConditionTrue, reasonNameserverCreated, reasonNameserverCreated) + return setStatus(&dnsCfg, metav1.ConditionTrue, ReasonNameserverCreated, ReasonNameserverCreated) } logger.Info("nameserver Service does not have an IP address allocated, waiting...") return reconcile.Result{}, nil } func nameserverResourceLabels(name, namespace string) map[string]string { - labels := childResourceLabels(name, namespace, "nameserver") + labels := reconciler.ChildResourceLabels(name, namespace, "nameserver") labels["app.kubernetes.io/name"] = "tailscale" labels["app.kubernetes.io/component"] = "nameserver" return labels } -func (a *NameserverReconciler) maybeProvision(ctx context.Context, tsDNSCfg *tsapi.DNSConfig) error { - labels := nameserverResourceLabels(tsDNSCfg.Name, a.tsNamespace) +func (r *Reconciler) maybeProvision(ctx context.Context, tsDNSCfg *tsapi.DNSConfig) error { + labels := nameserverResourceLabels(tsDNSCfg.Name, r.tsNamespace) dCfg := &deployConfig{ ownerRefs: []metav1.OwnerReference{*metav1.NewControllerRef(tsDNSCfg, tsapi.SchemeGroupVersion.WithKind("DNSConfig"))}, - namespace: a.tsNamespace, + namespace: r.tsNamespace, labels: labels, imageRepo: defaultNameserverImageRepo, imageTag: defaultNameserverImageTag, @@ -192,22 +236,22 @@ func (a *NameserverReconciler) maybeProvision(ctx context.Context, tsDNSCfg *tsa dCfg.tolerations = tsDNSCfg.Spec.Nameserver.Pod.Tolerations } - for _, deployable := range []deployable{saDeployable, deployDeployable, svcDeployable, cmDeployable} { - if err := deployable.updateObj(ctx, dCfg, a.Client); err != nil { - return fmt.Errorf("error reconciling %s: %w", deployable.kind, err) + for _, d := range []deployable{saDeployable, deployDeployable, svcDeployable, cmDeployable} { + if err := d.updateObj(ctx, dCfg, r.Client); err != nil { + return fmt.Errorf("error reconciling %s: %w", d.kind, err) } } return nil } // maybeCleanup removes DNSConfig from being tracked. The cluster resources -// created, will be automatically garbage collected as they are owned by the +// created will be automatically garbage collected as they are owned by the // DNSConfig. -func (a *NameserverReconciler) maybeCleanup(dnsCfg *tsapi.DNSConfig) error { - a.mu.Lock() - a.managedNameservers.Remove(dnsCfg.UID) - a.mu.Unlock() - gaugeNameserverResources.Set(int64(a.managedNameservers.Len())) +func (r *Reconciler) maybeCleanup(dnsCfg *tsapi.DNSConfig) error { + r.mu.Lock() + r.managedNameservers.Remove(dnsCfg.UID) + r.mu.Unlock() + gaugeNameserverResources.Set(int64(r.managedNameservers.Len())) return nil } @@ -228,13 +272,13 @@ type deployConfig struct { } var ( - //go:embed deploy/manifests/nameserver/cm.yaml + //go:embed manifests/cm.yaml cmYaml []byte - //go:embed deploy/manifests/nameserver/deploy.yaml + //go:embed manifests/deploy.yaml deployYaml []byte - //go:embed deploy/manifests/nameserver/sa.yaml + //go:embed manifests/sa.yaml saYaml []byte - //go:embed deploy/manifests/nameserver/svc.yaml + //go:embed manifests/svc.yaml svcYaml []byte deployDeployable = deployable{ @@ -301,3 +345,98 @@ var ( }, } ) + +type ptrObject[T any] interface { + client.Object + *T +} + +// createOrMaybeUpdate adds obj to the k8s cluster, unless the object already exists, +// in which case update is called to make changes to it. If update is nil or returns +// an error, the object is returned unmodified. +// +// obj is looked up by its Name and Namespace if Name is set, otherwise it's +// looked up by labels. +func createOrMaybeUpdate[T any, O ptrObject[T]](ctx context.Context, c client.Client, ns string, obj O, update func(O) error) (O, error) { + var ( + existing O + err error + ) + if obj.GetName() != "" { + existing = new(T) + existing.SetName(obj.GetName()) + existing.SetNamespace(obj.GetNamespace()) + err = c.Get(ctx, client.ObjectKeyFromObject(obj), existing) + } else { + existing, err = getSingleObject[T, O](ctx, c, ns, obj.GetLabels()) + } + if err == nil && existing != nil { + if update != nil { + if err := update(existing); err != nil { + return nil, err + } + if err := c.Update(ctx, existing); err != nil { + return nil, err + } + } + return existing, nil + } + if err != nil && !apierrors.IsNotFound(err) { + return nil, fmt.Errorf("failed to get object: %w", err) + } + if err := c.Create(ctx, obj); err != nil { + return nil, err + } + return obj, nil +} + +// createOrUpdate adds obj to the k8s cluster, unless the object already exists, +// in which case update is called to make changes to it. If update is nil, the +// existing object is returned unmodified. +// +// obj is looked up by its Name and Namespace if Name is set, otherwise it's +// looked up by labels. +func createOrUpdate[T any, O ptrObject[T]](ctx context.Context, c client.Client, ns string, obj O, update func(O)) (O, error) { + return createOrMaybeUpdate(ctx, c, ns, obj, func(o O) error { + if update != nil { + update(o) + } + return nil + }) +} + +// getSingleObject searches for k8s objects of type T with the given labels, +// and returns it. Returns nil if no objects match the labels, and an error if +// more than one object matches. +func getSingleObject[T any, O ptrObject[T]](ctx context.Context, c client.Client, ns string, labels map[string]string) (O, error) { + ret := O(new(T)) + kinds, _, err := c.Scheme().ObjectKinds(ret) + if err != nil { + return nil, err + } + if len(kinds) != 1 { + return nil, fmt.Errorf("more than 1 GroupVersionKind for %T", ret) + } + + gvk := kinds[0] + gvk.Kind += "List" + lst := unstructured.UnstructuredList{} + lst.SetGroupVersionKind(gvk) + if err := c.List(ctx, &lst, client.InNamespace(ns), client.MatchingLabels(labels)); err != nil { + return nil, err + } + + if len(lst.Items) == 0 { + return nil, nil + } + if len(lst.Items) > 1 { + return nil, fmt.Errorf("found multiple matching %T objects", ret) + } + + item := lst.Items[0] + ret2 := O(new(T)) + if err := c.Scheme().Convert(&item, ret2, nil); err != nil { + return nil, err + } + return ret2, nil +} diff --git a/cmd/k8s-operator/nameserver_test.go b/k8s-operator/reconciler/nameserver/nameserver_test.go index b374c114f..fc56703fe 100644 --- a/cmd/k8s-operator/nameserver_test.go +++ b/k8s-operator/reconciler/nameserver/nameserver_test.go @@ -3,29 +3,35 @@ //go:build !plan9 -// tailscale-operator provides a way to expose services running in a Kubernetes -// cluster to your Tailnet and to make Tailscale nodes available to cluster -// workloads -package main +package nameserver import ( + "context" "encoding/json" + "reflect" "testing" "time" + "github.com/google/go-cmp/cmp" "go.uber.org/zap" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/yaml" operatorutils "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" "tailscale.com/tstest" "tailscale.com/util/mak" ) +const tsNamespace = "tailscale" + func TestNameserverReconciler(t *testing.T) { dnsConfig := &tsapi.DNSConfig{ TypeMeta: metav1.TypeMeta{Kind: "DNSConfig", APIVersion: "tailscale.com/v1alpha1"}, @@ -68,13 +74,13 @@ func TestNameserverReconciler(t *testing.T) { } clock := tstest.NewClock(tstest.ClockOpts{}) - reconciler := &NameserverReconciler{ + r := &Reconciler{ Client: fc, clock: clock, logger: logger.Sugar(), tsNamespace: tsNamespace, } - expectReconciled(t, reconciler, "", "test") + mustReconcile(t, r, reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}) ownerReference := metav1.NewControllerRef(dnsConfig, tsapi.SchemeGroupVersion.WithKind("DNSConfig")) nameserverLabels := nameserverResourceLabels(dnsConfig.Name, tsNamespace) @@ -114,22 +120,22 @@ func TestNameserverReconciler(t *testing.T) { }) t.Run("dns config status is set", func(t *testing.T) { - // Verify that DNSConfig advertizes the nameserver's Service IP address, + // Verify that DNSConfig advertises the nameserver's Service IP address, // has the ready status condition and tailscale finalizer. mustUpdate(t, fc, "tailscale", "nameserver", func(svc *corev1.Service) { svc.Spec.ClusterIP = "1.2.3.4" }) - expectReconciled(t, reconciler, "", "test") + mustReconcile(t, r, reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}) - dnsConfig.Finalizers = []string{FinalizerName} + dnsConfig.Finalizers = []string{reconciler.FinalizerName} dnsConfig.Status.Nameserver = &tsapi.NameserverStatus{ IP: "1.2.3.4", } dnsConfig.Status.Conditions = append(dnsConfig.Status.Conditions, metav1.Condition{ Type: string(tsapi.NameserverReady), Status: metav1.ConditionTrue, - Reason: reasonNameserverCreated, - Message: reasonNameserverCreated, + Reason: ReasonNameserverCreated, + Message: ReasonNameserverCreated, LastTransitionTime: metav1.Time{Time: clock.Now().Truncate(time.Second)}, }) @@ -141,7 +147,7 @@ func TestNameserverReconciler(t *testing.T) { mustUpdate(t, fc, "", "test", func(dnsCfg *tsapi.DNSConfig) { dnsCfg.Spec.Nameserver.Image.Tag = "v0.0.2" }) - expectReconciled(t, reconciler, "", "test") + mustReconcile(t, r, reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}) wantsDeploy.Spec.Template.Spec.Containers[0].Image = "test:v0.0.2" expectEqual(t, fc, wantsDeploy) }) @@ -159,7 +165,7 @@ func TestNameserverReconciler(t *testing.T) { mak.Set(&cm.Data, "records.json", string(bs)) }) - expectReconciled(t, reconciler, "", "test") + mustReconcile(t, r, reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}) wantCm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ @@ -177,12 +183,56 @@ func TestNameserverReconciler(t *testing.T) { t.Run("uses default nameserver image", func(t *testing.T) { // Verify that if dnsconfig.spec.nameserver.image.{repo,tag} are unset, - // the nameserver image defaults to tailscale/k8s-nameserver:unstable. + // the nameserver image defaults to tailscale/k8s-nameserver:stable. mustUpdate(t, fc, "", "test", func(dnsCfg *tsapi.DNSConfig) { dnsCfg.Spec.Nameserver.Image = nil }) - expectReconciled(t, reconciler, "", "test") + mustReconcile(t, r, reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}) wantsDeploy.Spec.Template.Spec.Containers[0].Image = "tailscale/k8s-nameserver:stable" expectEqual(t, fc, wantsDeploy) }) } + +func mustReconcile(t *testing.T, r *Reconciler, req reconcile.Request) { + t.Helper() + if _, err := r.Reconcile(context.Background(), req); err != nil { + t.Fatalf("unexpected reconcile error: %v", err) + } +} + +func mustUpdate[T any, O interface { + client.Object + *T +}](t *testing.T, c client.Client, ns, name string, update func(O)) { + t.Helper() + obj := O(new(T)) + if err := c.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: name}, obj); err != nil { + t.Fatalf("getting object: %v", err) + } + update(obj) + if err := c.Update(context.Background(), obj); err != nil { + t.Fatalf("updating object: %v", err) + } +} + +func expectEqual[T any, O interface { + client.Object + *T +}](t *testing.T, c client.Client, want O) { + t.Helper() + got := O(new(T)) + if err := c.Get(context.Background(), types.NamespacedName{ + Name: want.GetName(), + Namespace: want.GetNamespace(), + }, got); err != nil { + t.Fatalf("getting %q: %v", want.GetName(), err) + } + // The resource version changes eagerly whenever the operator does even a + // no-op update. Asserting a specific value leads to overly brittle tests, + // so just remove it from both got and want. + got.SetResourceVersion("") + want.SetResourceVersion("") + if diff := cmp.Diff(got, want); diff != "" { + t.Fatalf("unexpected %s (-got +want):\n%s", reflect.TypeOf(want).Elem().Name(), diff) + } +} diff --git a/k8s-operator/reconciler/proxyclass/proxyclass.go b/k8s-operator/reconciler/proxyclass/proxyclass.go index 28ca39afe..7f20375fa 100644 --- a/k8s-operator/reconciler/proxyclass/proxyclass.go +++ b/k8s-operator/reconciler/proxyclass/proxyclass.go @@ -93,8 +93,8 @@ type ( Client client.Client // Recorder is used to emit Kubernetes events. Recorder record.EventRecorder - // TsNamespace is the namespace the operator is installed in. - TsNamespace string + // TailscaleNamespace is the namespace the operator is installed in. + TailscaleNamespace string // Clock controls time-based functions. Typically modified for tests. Clock tstime.Clock // Logger is the logger to use for this Reconciler. @@ -109,9 +109,9 @@ func NewReconciler(options ReconcilerOptions) *Reconciler { logger := options.Logger.Named(reconcilerName) return &Reconciler{ Client: options.Client, - nodePortRange: getServicesNodePortRange(context.Background(), options.Client, options.TsNamespace, logger), + nodePortRange: getServicesNodePortRange(context.Background(), options.Client, options.TailscaleNamespace, logger), recorder: options.Recorder, - tsNamespace: options.TsNamespace, + tsNamespace: options.TailscaleNamespace, clock: options.Clock, logger: logger, } diff --git a/k8s-operator/reconciler/reconciler.go b/k8s-operator/reconciler/reconciler.go index fcad7201e..43df2f3ed 100644 --- a/k8s-operator/reconciler/reconciler.go +++ b/k8s-operator/reconciler/reconciler.go @@ -8,14 +8,25 @@ package reconciler import ( + "context" "slices" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "tailscale.com/kube/kubetypes" ) const ( // FinalizerName is the common finalizer used across all Tailscale Kubernetes resources. FinalizerName = "tailscale.com/finalizer" + + // Label constants for tracking parent resource relationships on child resources. + LabelParentType = "tailscale.com/parent-resource-type" + LabelParentName = "tailscale.com/parent-resource" + LabelParentNamespace = "tailscale.com/parent-resource-ns" ) // SetFinalizer adds the finalizer to the resource if not already present. @@ -37,3 +48,45 @@ func RemoveFinalizer(obj client.Object) { finalizers := obj.GetFinalizers() obj.SetFinalizers(append(finalizers[:idx], finalizers[idx+1:]...)) } + +// ChildResourceLabels returns labels applied to child resources created for a given parent resource. +func ChildResourceLabels(name, ns, typ string) map[string]string { + return map[string]string{ + kubetypes.LabelManaged: "true", + LabelParentName: name, + LabelParentNamespace: ns, + LabelParentType: typ, + } +} + +// IsManagedResource reports whether the object is managed by the Tailscale operator. +func IsManagedResource(o client.Object) bool { + return o.GetLabels()[kubetypes.LabelManaged] == "true" +} + +// IsManagedByType reports whether the object is a managed child resource of the given parent type. +func IsManagedByType(o client.Object, typ string) bool { + return IsManagedResource(o) && o.GetLabels()[LabelParentType] == typ +} + +// ParentFromObjectLabels returns the namespaced name of the parent resource encoded in the object's labels. +func ParentFromObjectLabels(o client.Object) types.NamespacedName { + ls := o.GetLabels() + return types.NamespacedName{ + Namespace: ls[LabelParentNamespace], + Name: ls[LabelParentName], + } +} + +// ManagedResourceHandlerForType returns a handler.MapFunc that enqueues the +// parent resource for any managed child object of the given type. +func ManagedResourceHandlerForType(typ string) handler.MapFunc { + return func(_ context.Context, o client.Object) []reconcile.Request { + if !IsManagedByType(o, typ) { + return nil + } + return []reconcile.Request{ + {NamespacedName: ParentFromObjectLabels(o)}, + } + } +} |
