cmd/k8s-operator,k8s-operator/apis/v1alpha1: allow Connector to route traffic to a single IPirbekrm/dnat

Add a new connector.spec.dnat field that can be used to route
traffic to a single IP address reachable from cluster.
This can be used to expose to tailnet a cloud service that can be
reached from cluster and does not have a DNS name (cloud services that
have DNS names can be exposed to tailnet using ExternalName Services, which is
a probably preferable way.)

Updates tailscale/tailscale#12919

Signed-off-by: Irbe Krumina <irbe@tailscale.com>

8 files changed, 144 insertions, 25 deletions

diff --git a/cmd/k8s-operator/connector.go b/cmd/k8s-operator/connector.go
index 4586dfdbf..e4a7ce2ed 100644
--- a/cmd/k8s-operator/connector.go
+++ b/cmd/k8s-operator/connector.go
@@ -57,6 +57,7 @@ type ConnectorReconciler struct {
 
 	subnetRouters set.Slice[types.UID] // for subnet routers gauge
 	exitNodes     set.Slice[types.UID] // for exit nodes gauge
+	dnats         set.Slice[types.UID] // for dnat gauge
 }
 
 var (
@@ -66,6 +67,7 @@ var (
 	gaugeConnectorSubnetRouterResources = clientmetric.NewGauge("k8s_connector_subnetrouter_resources")
 	// gaugeConnectorExitNodeResources tracks the number of Connectors currently managed by this operator instance that are exit nodes.
 	gaugeConnectorExitNodeResources = clientmetric.NewGauge("k8s_connector_exitnode_resources")
+	gaugeConnectorDNATResources     = clientmetric.NewGauge("k8s_connector_dnat_resources")
 )
 
 func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
@@ -149,6 +151,9 @@ func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Reque
 		cn.Status.SubnetRoutes = cn.Spec.SubnetRouter.AdvertiseRoutes.Stringify()
 		return setStatus(cn, tsapi.ConnectorReady, metav1.ConditionTrue, reasonConnectorCreated, reasonConnectorCreated)
 	}
+	if len(cn.Spec.DNAT) != 0 {
+		cn.Status.DNAT = cn.Spec.DNAT[0]
+	}
 	cn.Status.SubnetRoutes = ""
 	return setStatus(cn, tsapi.ConnectorReady, metav1.ConditionTrue, reasonConnectorCreated, reasonConnectorCreated)
 }
@@ -178,33 +183,42 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge
 		Hostname:            hostname,
 		ChildResourceLabels: crl,
 		Tags:                cn.Spec.Tags.Stringify(),
-		Connector: &connector{
-			isExitNode: cn.Spec.ExitNode,
-		},
-		ProxyClassName: proxyClass,
+		ProxyClassName:      proxyClass,
+		isExitNode:          cn.Spec.ExitNode,
 	}
 
 	if cn.Spec.SubnetRouter != nil && len(cn.Spec.SubnetRouter.AdvertiseRoutes) > 0 {
-		sts.Connector.routes = cn.Spec.SubnetRouter.AdvertiseRoutes.Stringify()
+		sts.routes = cn.Spec.SubnetRouter.AdvertiseRoutes.Stringify()
+	}
+
+	if len(cn.Spec.DNAT) != 0 {
+		sts.ClusterTargetIP = cn.Spec.DNAT[0]
 	}
 
 	a.mu.Lock()
-	if sts.Connector.isExitNode {
+	if sts.isExitNode {
 		a.exitNodes.Add(cn.UID)
 	} else {
 		a.exitNodes.Remove(cn.UID)
 	}
-	if sts.Connector.routes != "" {
+	if sts.routes != "" {
 		a.subnetRouters.Add(cn.GetUID())
 	} else {
 		a.subnetRouters.Remove(cn.GetUID())
 	}
+	if sts.ClusterTargetIP != "" {
+		a.dnats.Add(cn.GetUID())
+	} else {
+		a.dnats.Remove(cn.GetUID())
+	}
 	a.mu.Unlock()
 	gaugeConnectorSubnetRouterResources.Set(int64(a.subnetRouters.Len()))
 	gaugeConnectorExitNodeResources.Set(int64(a.exitNodes.Len()))
+	gaugeConnectorDNATResources.Set(int64(a.exitNodes.Len()))
 	var connectors set.Slice[types.UID]
 	connectors.AddSlice(a.exitNodes.Slice())
 	connectors.AddSlice(a.subnetRouters.Slice())
+	connectors.AddSlice(a.dnats.Slice())
 	gaugeConnectorResources.Set(int64(connectors.Len()))
 
 	_, err := a.ssr.Provision(ctx, logger, sts)
@@ -247,12 +261,15 @@ func (a *ConnectorReconciler) maybeCleanupConnector(ctx context.Context, logger
 	a.mu.Lock()
 	a.subnetRouters.Remove(cn.UID)
 	a.exitNodes.Remove(cn.UID)
+	a.dnats.Remove(cn.UID)
 	a.mu.Unlock()
 	gaugeConnectorExitNodeResources.Set(int64(a.exitNodes.Len()))
 	gaugeConnectorSubnetRouterResources.Set(int64(a.subnetRouters.Len()))
+	gaugeConnectorDNATResources.Set(int64(a.dnats.Len()))
 	var connectors set.Slice[types.UID]
 	connectors.AddSlice(a.exitNodes.Slice())
 	connectors.AddSlice(a.subnetRouters.Slice())
+	connectors.AddSlice(a.dnats.Slice())
 	gaugeConnectorResources.Set(int64(connectors.Len()))
 	return true, nil
 }
@@ -261,8 +278,11 @@ func (a *ConnectorReconciler) validate(cn *tsapi.Connector) error {
 	// Connector fields are already validated at apply time with CEL validation
 	// on custom resource fields. The checks here are a backup in case the
 	// CEL validation breaks without us noticing.
-	if !(cn.Spec.SubnetRouter != nil || cn.Spec.ExitNode) {
-		return errors.New("invalid spec: a Connector must expose subnet routes or act as an exit node (or both)")
+	if !(cn.Spec.SubnetRouter != nil || cn.Spec.ExitNode || len(cn.Spec.DNAT) != 0) {
+		return errors.New("invalid spec: a Connector must expose subnet routes or act as an exit node (or both) or have DNAT set")
+	}
+	if (cn.Spec.SubnetRouter != nil || cn.Spec.ExitNode) && len(cn.Spec.DNAT) != 0 {
+		return errors.New("invalid spec: a Connector must not be both a subnet router and an exit node as well as have a DNAT set")
 	}
 	if cn.Spec.SubnetRouter == nil {
 		return nil
diff --git a/cmd/k8s-operator/connector_test.go b/cmd/k8s-operator/connector_test.go
index 8a7a5dd53..c6481a316 100644
--- a/cmd/k8s-operator/connector_test.go
+++ b/cmd/k8s-operator/connector_test.go
@@ -191,6 +191,42 @@ func TestConnector(t *testing.T) {
 
 	expectMissing[appsv1.StatefulSet](t, fc, "operator-ns", shortName)
 	expectMissing[corev1.Secret](t, fc, "operator-ns", fullName)
+
+	// Create a Connector that configures DNAT
+	cn = &tsapi.Connector{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "test",
+			UID:  types.UID("1234-UID"),
+		},
+		TypeMeta: metav1.TypeMeta{
+			Kind:       tsapi.ConnectorKind,
+			APIVersion: "tailscale.io/v1alpha1",
+		},
+		Spec: tsapi.ConnectorSpec{
+			DNAT: []string{"10.44.0.1"},
+		},
+	}
+	mustCreate(t, fc, cn)
+	expectReconciled(t, cr, "", "test")
+	fullName, shortName = findGenName(t, fc, "", "test", "connector")
+
+	opts = configOpts{
+		stsName:         shortName,
+		secretName:      fullName,
+		parentType:      "connector",
+		clusterTargetIP: "10.44.0.1",
+		hostname:        "test-connector",
+	}
+	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
+	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
+
+	// Update DNAT value
+	mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
+		conn.Spec.DNAT = []string{"10.44.0.2"}
+	})
+	opts.clusterTargetIP = "10.44.0.2"
+	expectReconciled(t, cr, "", "test")
+	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
 }
 
 func TestConnectorWithProxyClass(t *testing.T) {
diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml
index 66ff060d4..ea62b84a8 100644
--- a/cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml
+++ b/cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml
@@ -24,6 +24,10 @@ spec:
           jsonPath: .status.isExitNode
           name: IsExitNode
           type: string
+        - description: DNAT of the Connector if any.
+          jsonPath: .status.dnat
+          name: DNAT
+          type: string
         - description: Status of the deployed Connector resources.
           jsonPath: .status.conditions[?(@.type == "ConnectorReady")].reason
           name: Status
@@ -66,6 +70,17 @@ spec:
                 https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status
               type: object
               properties:
+                dnat:
+                  description: |-
+                    DNAT is an address routable from within cluster that tailnet
+                    traffic should be routed to. DNAT cannot be set together with
+                    .spec.subnetRouter or .spec.exitNode.
+                    DNAT is currently restricted to a list of a single IP address.
+                  type: array
+                  maxItems: 1
+                  minItems: 1
+                  items:
+                    type: string
                 exitNode:
                   description: |-
                     ExitNode defines whether the Connector node should act as a
@@ -125,8 +140,10 @@ spec:
                     type: string
                     pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$
               x-kubernetes-validations:
-                - rule: has(self.subnetRouter) || self.exitNode == true
-                  message: A Connector needs to be either an exit node or a subnet router, or both.
+                - rule: (has(self.subnetRouter) || self.exitNode == true) ||  has(self.dnat)
+                  message: A Connector needs to be either an exit node or a subnet router, or both or have .spec.dnat set.
+                - rule: (has(self.subnetRouter) || (has(self.exitNode) && self.exitNode == true)) !=  has(self.dnat)
+                  message: A Connector with .spec.dnat set must not be an exit node or subnet router.
             status:
               description: |-
                 ConnectorStatus describes the status of the Connector. This is set
@@ -194,6 +211,11 @@ spec:
                   x-kubernetes-list-map-keys:
                     - type
                   x-kubernetes-list-type: map
+                dnat:
+                  description: |-
+                    DNAT is a cluster routable IP address that the tailnet traffic to
+                    this node is routed to.
+                  type: string
                 hostname:
                   description: |-
                     Hostname is the fully qualified domain name of the Connector node.
diff --git a/cmd/k8s-operator/deploy/manifests/operator.yaml b/cmd/k8s-operator/deploy/manifests/operator.yaml
index 4633ba3a4..9222cf245 100644
--- a/cmd/k8s-operator/deploy/manifests/operator.yaml
+++ b/cmd/k8s-operator/deploy/manifests/operator.yaml
@@ -53,6 +53,10 @@ spec:
               jsonPath: .status.isExitNode
               name: IsExitNode
               type: string
+            - description: DNAT of the Connector if any.
+              jsonPath: .status.dnat
+              name: DNAT
+              type: string
             - description: Status of the deployed Connector resources.
               jsonPath: .status.conditions[?(@.type == "ConnectorReady")].reason
               name: Status
@@ -91,6 +95,17 @@ spec:
                             More info:
                             https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status
                         properties:
+                            dnat:
+                                description: |-
+                                    DNAT is an address routable from within cluster that tailnet
+                                    traffic should be routed to. DNAT cannot be set together with
+                                    .spec.subnetRouter or .spec.exitNode.
+                                    DNAT is currently restricted to a list of a single IP address.
+                                items:
+                                    type: string
+                                maxItems: 1
+                                minItems: 1
+                                type: array
                             exitNode:
                                 description: |-
                                     ExitNode defines whether the Connector node should act as a
@@ -151,8 +166,10 @@ spec:
                                 type: array
                         type: object
                         x-kubernetes-validations:
-                            - message: A Connector needs to be either an exit node or a subnet router, or both.
-                              rule: has(self.subnetRouter) || self.exitNode == true
+                            - message: A Connector needs to be either an exit node or a subnet router, or both or have .spec.dnat set.
+                              rule: (has(self.subnetRouter) || self.exitNode == true) ||  has(self.dnat)
+                            - message: A Connector with .spec.dnat set must not be an exit node or subnet router.
+                              rule: (has(self.subnetRouter) || (has(self.exitNode) && self.exitNode == true)) !=  has(self.dnat)
                     status:
                         description: |-
                             ConnectorStatus describes the status of the Connector. This is set
@@ -219,6 +236,11 @@ spec:
                                 x-kubernetes-list-map-keys:
                                     - type
                                 x-kubernetes-list-type: map
+                            dnat:
+                                description: |-
+                                    DNAT is a cluster routable IP address that the tailnet traffic to
+                                    this node is routed to.
+                                type: string
                             hostname:
                                 description: |-
                                     Hostname is the fully qualified domain name of the Connector node.
diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go
index 17cc047d0..e9204ff6a 100644
--- a/cmd/k8s-operator/sts.go
+++ b/cmd/k8s-operator/sts.go
@@ -119,21 +119,17 @@ type tailscaleSTSConfig struct {
 	Hostname string
 	Tags     []string // if empty, use defaultTags
 
-	// Connector specifies a configuration of a Connector instance if that's
-	// what this StatefulSet should be created for.
-	Connector *connector
+	// routes is a list of subnet routes that this proxy should expose.
+	routes string
+
+	// isExitNode defines whether this proxy should act as an exit node.
+	isExitNode bool
 
 	ProxyClassName string // name of ProxyClass if one needs to be applied to the proxy
 
 	ProxyClass *tsapi.ProxyClass // ProxyClass that needs to be applied to the proxy (if there is one)
 }
 
-type connector struct {
-	// routes is a list of subnet routes that this Connector should expose.
-	routes string
-	// isExitNode defines whether this Connector should act as an exit node.
-	isExitNode bool
-}
 type tsnetServer interface {
 	CertDomains() []string
 }
@@ -774,8 +770,8 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *co
 	if stsC.TailnetTargetFQDN != "" || stsC.TailnetTargetIP != "" {
 		conf.NoStatefulFiltering = "true"
 	}
-	if stsC.Connector != nil {
-		routes, err := netutil.CalcAdvertiseRoutes(stsC.Connector.routes, stsC.Connector.isExitNode)
+	if len(stsC.routes) != 0 || stsC.isExitNode {
+		routes, err := netutil.CalcAdvertiseRoutes(stsC.routes, stsC.isExitNode)
 		if err != nil {
 			return nil, fmt.Errorf("error calculating routes: %w", err)
 		}
diff --git a/k8s-operator/api.md b/k8s-operator/api.md
index 1b72df0f2..77177e4e0 100644
--- a/k8s-operator/api.md
+++ b/k8s-operator/api.md
@@ -84,6 +84,7 @@ _Appears in:_
 | `proxyClass` _string_ | ProxyClass is the name of the ProxyClass custom resource that<br />contains configuration options that should be applied to the<br />resources created for this Connector. If unset, the operator will<br />create resources with the default configuration. |  |  |
 | `subnetRouter` _[SubnetRouter](#subnetrouter)_ | SubnetRouter defines subnet routes that the Connector node should<br />expose to tailnet. If unset, none are exposed.<br />https://tailscale.com/kb/1019/subnets/ |  |  |
 | `exitNode` _boolean_ | ExitNode defines whether the Connector node should act as a<br />Tailscale exit node. Defaults to false.<br />https://tailscale.com/kb/1103/exit-nodes |  |  |
+| `dnat` _[dnat](#dnat)_ | DNAT is an address routable from within cluster that tailnet<br />traffic should be routed to. DNAT cannot be set together with<br />.spec.subnetRouter or .spec.exitNode.<br />DNAT is currently restricted to a list of a single IP address. |  | MaxItems: 1 <br />MinItems: 1 <br /> |
 
 
 #### ConnectorStatus
@@ -101,6 +102,7 @@ _Appears in:_
 | --- | --- | --- | --- |
 | `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#condition-v1-meta) array_ | List of status conditions to indicate the status of the Connector.<br />Known condition types are `ConnectorReady`. |  |  |
 | `subnetRoutes` _string_ | SubnetRoutes are the routes currently exposed to tailnet via this<br />Connector instance. |  |  |
+| `dnat` _string_ | DNAT is a cluster routable IP address that the tailnet traffic to<br />this node is routed to. |  |  |
 | `isExitNode` _boolean_ | IsExitNode is set to true if the Connector acts as an exit node. |  |  |
 | `tailnetIPs` _string array_ | TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6)<br />assigned to the Connector node. |  |  |
 | `hostname` _string_ | Hostname is the fully qualified domain name of the Connector node.<br />If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the<br />node. |  |  |
diff --git a/k8s-operator/apis/v1alpha1/types_connector.go b/k8s-operator/apis/v1alpha1/types_connector.go
index c33ad3c39..e36c915c6 100644
--- a/k8s-operator/apis/v1alpha1/types_connector.go
+++ b/k8s-operator/apis/v1alpha1/types_connector.go
@@ -22,6 +22,7 @@ var ConnectorKind = "Connector"
 // +kubebuilder:resource:scope=Cluster,shortName=cn
 // +kubebuilder:printcolumn:name="SubnetRoutes",type="string",JSONPath=`.status.subnetRoutes`,description="CIDR ranges exposed to tailnet by a subnet router defined via this Connector instance."
 // +kubebuilder:printcolumn:name="IsExitNode",type="string",JSONPath=`.status.isExitNode`,description="Whether this Connector instance defines an exit node."
+// +kubebuilder:printcolumn:name="DNAT",type="string",JSONPath=`.status.dnat`,description="DNAT of the Connector if any."
 // +kubebuilder:printcolumn:name="Status",type="string",JSONPath=`.status.conditions[?(@.type == "ConnectorReady")].reason`,description="Status of the deployed Connector resources."
 
 // Connector defines a Tailscale node that will be deployed in the cluster. The
@@ -55,7 +56,8 @@ type ConnectorList struct {
 }
 
 // ConnectorSpec describes a Tailscale node to be deployed in the cluster.
-// +kubebuilder:validation:XValidation:rule="has(self.subnetRouter) || self.exitNode == true",message="A Connector needs to be either an exit node or a subnet router, or both."
+// +kubebuilder:validation:XValidation:rule="(has(self.subnetRouter) || self.exitNode == true) ||  has(self.dnat)",message="A Connector needs to be either an exit node or a subnet router, or both or have .spec.dnat set."
+// +kubebuilder:validation:XValidation:rule="(has(self.subnetRouter) || (has(self.exitNode) && self.exitNode == true)) !=  has(self.dnat)",message="A Connector with .spec.dnat set must not be an exit node or subnet router."
 type ConnectorSpec struct {
 	// Tags that the Tailscale node will be tagged with.
 	// Defaults to [tag:k8s].
@@ -92,8 +94,18 @@ type ConnectorSpec struct {
 	// https://tailscale.com/kb/1103/exit-nodes
 	// +optional
 	ExitNode bool `json:"exitNode"`
+	// DNAT is an address routable from within cluster that tailnet
+	// traffic should be routed to. DNAT cannot be set together with
+	// .spec.subnetRouter or .spec.exitNode.
+	// DNAT is currently restricted to a list of a single IP address.
+	// +optional
+	DNAT dnat `json:"dnat,omitempty"`
 }
 
+// +kubebuilder:validation:MaxItems=1
+// +kubebuilder:validation:MinItems=1
+type dnat []string
+
 // SubnetRouter defines subnet routes that should be exposed to tailnet via a
 // Connector node.
 type SubnetRouter struct {
@@ -153,6 +165,10 @@ type ConnectorStatus struct {
 	// Connector instance.
 	// +optional
 	SubnetRoutes string `json:"subnetRoutes"`
+	// DNAT is a cluster routable IP address that the tailnet traffic to
+	// this node is routed to.
+	// +optional
+	DNAT string `json:"dnat,omitempty"`
 	// IsExitNode is set to true if the Connector acts as an exit node.
 	// +optional
 	IsExitNode bool `json:"isExitNode"`
diff --git a/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go b/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go
index 648a6875b..f7b50c378 100644
--- a/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go
+++ b/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go
@@ -85,6 +85,11 @@ func (in *ConnectorSpec) DeepCopyInto(out *ConnectorSpec) {
 		*out = new(SubnetRouter)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.DNAT != nil {
+		in, out := &in.DNAT, &out.DNAT
+		*out = make(dnat, len(*in))
+		copy(*out, *in)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectorSpec.