cmd/k8s-operator: annotate proxy StatefulSets with operator versionirbekrm/operatorversion

Add a tailscale.com/operator-last-version annotation to StatefulSets for ingress/egress proxies and Connectors.
Set it to the operator version of the current operator version each time the StatefulSet is re-synced.
This will help us to determine the potential proxy state when making changes in proxy configuration in the future.

Updates tailscale/tailscale#10407

Signed-off-by: Irbe Krumina <irbe@tailscale.com>

4 files changed, 71 insertions, 2 deletions

diff --git a/cmd/k8s-operator/operator.go b/cmd/k8s-operator/operator.go
index 236c83ad8..00f213de5 100644
--- a/cmd/k8s-operator/operator.go
+++ b/cmd/k8s-operator/operator.go
@@ -247,6 +247,7 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string
 		proxyImage:             image,
 		proxyPriorityClassName: priorityClassName,
 		tsFirewallMode:         tsFirewallMode,
+		operatorVersion:        version.Short(),
 	}
 	err = builder.
 		ControllerManagedBy(mgr).
diff --git a/cmd/k8s-operator/operator_test.go b/cmd/k8s-operator/operator_test.go
index 90ed243bb..1eb02f32a 100644
--- a/cmd/k8s-operator/operator_test.go
+++ b/cmd/k8s-operator/operator_test.go
@@ -868,6 +868,67 @@ func TestCustomHostname(t *testing.T) {
 	expectEqual(t, fc, want)
 }
 
+func TestOperatorVersion(t *testing.T) {
+	fc := fake.NewFakeClient()
+	ft := &fakeTSClient{}
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
+	sr := &ServiceReconciler{
+		Client: fc,
+		ssr: &tailscaleSTSReconciler{
+			Client:            fc,
+			tsClient:          ft,
+			defaultTags:       []string{"tag:k8s"},
+			operatorNamespace: "operator-ns",
+			proxyImage:        "tailscale/tailscale",
+			operatorVersion:   "v1.2.3",
+		},
+		logger: zl.Sugar(),
+	}
+	// Create a service that we should manage, and check that the initial round
+	// of objects looks right.
+	mustCreate(t, fc, &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: "default",
+			// The apiserver is supposed to set the UID, but the fake client
+			// doesn't. So, set it explicitly because other code later depends
+			// on it being set.
+			UID: types.UID("1234-UID"),
+			Annotations: map[string]string{
+				"tailscale.com/expose": "true",
+			},
+		},
+		Spec: corev1.ServiceSpec{
+			ClusterIP: "10.20.30.40",
+			Type:      corev1.ServiceTypeClusterIP,
+		},
+	})
+
+	expectReconciled(t, sr, "default", "test")
+
+	fullName, shortName := findGenName(t, fc, "default", "test", "svc")
+	o := configOpts{
+		stsName:         shortName,
+		secretName:      fullName,
+		namespace:       "default",
+		parentType:      "svc",
+		hostname:        "default-test",
+		clusterTargetIP: "10.20.30.40",
+		operatorVersion: "v1.2.3",
+	}
+	expectEqual(t, fc, expectedSTS(t, fc, o))
+
+	// value of annotationOperatorVersion changes when a new version of the
+	// operator re-syncs the proxy.
+	sr.ssr.operatorVersion = "v1.2.4"
+	expectReconciled(t, sr, "default", "test")
+	o.operatorVersion = "v1.2.4"
+	expectEqual(t, fc, expectedSTS(t, fc, o))
+}
+
 func TestCustomPriorityClassName(t *testing.T) {
 	fc := fake.NewFakeClient()
 	ft := &fakeTSClient{}
diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go
index 87f114b63..6e103b4c1 100644
--- a/cmd/k8s-operator/sts.go
+++ b/cmd/k8s-operator/sts.go
@@ -55,7 +55,7 @@ const (
 
 	FinalizerName = "tailscale.com/finalizer"
 
-	// Annotations settable by users on services.
+	// Annotations settable by users on Services.
 	AnnotationExpose             = "tailscale.com/expose"
 	AnnotationTags               = "tailscale.com/tags"
 	AnnotationHostname           = "tailscale.com/hostname"
@@ -92,6 +92,8 @@ const (
 	// podAnnotationLastSetConfigFileHash is sha256 hash of the current tailscaled configuration contents.
 	podAnnotationLastSetConfigFileHash = "tailscale.com/operator-last-set-config-file-hash"
 
+	annotationOperatorVersion = "tailscale.com/operator-last-version" // version of tailscale operator that last updated this component
+
 	// tailscaledConfigKey is the name of the key in proxy Secret Data that
 	// holds the tailscaled config contents.
 	tailscaledConfigKey = "tailscaled"
@@ -101,7 +103,7 @@ var (
 	// tailscaleManagedLabels are label keys that tailscale operator sets on StatefulSets and Pods.
 	tailscaleManagedLabels = []string{LabelManaged, LabelParentType, LabelParentName, LabelParentNamespace, "app"}
 	// tailscaleManagedAnnotations are annotation keys that tailscale operator sets on StatefulSets and Pods.
-	tailscaleManagedAnnotations = []string{podAnnotationLastSetClusterIP, podAnnotationLastSetHostname, podAnnotationLastSetTailnetTargetIP, podAnnotationLastSetTailnetTargetFQDN, podAnnotationLastSetConfigFileHash}
+	tailscaleManagedAnnotations = []string{podAnnotationLastSetClusterIP, podAnnotationLastSetHostname, podAnnotationLastSetTailnetTargetIP, podAnnotationLastSetTailnetTargetFQDN, podAnnotationLastSetConfigFileHash, annotationOperatorVersion}
 )
 
 type tailscaleSTSConfig struct {
@@ -148,6 +150,7 @@ type tailscaleSTSReconciler struct {
 	proxyImage             string
 	proxyPriorityClassName string
 	tsFirewallMode         string
+	operatorVersion        string // current version of the operator as returned by tailscale.com/version
 }
 
 func (sts tailscaleSTSReconciler) validate() error {
@@ -571,6 +574,7 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
 			},
 		})
 	}
+	mak.Set(&ss.ObjectMeta.Annotations, annotationOperatorVersion, a.operatorVersion)
 	logger.Debugf("reconciling statefulset %s/%s", ss.GetNamespace(), ss.GetName())
 	if sts.ProxyClass != "" {
 		logger.Debugf("configuring proxy resources with ProxyClass %s", sts.ProxyClass)
diff --git a/cmd/k8s-operator/testutils_test.go b/cmd/k8s-operator/testutils_test.go
index 8e4834873..aaefeacd3 100644
--- a/cmd/k8s-operator/testutils_test.go
+++ b/cmd/k8s-operator/testutils_test.go
@@ -49,6 +49,7 @@ type configOpts struct {
 	serveConfig                                    *ipn.ServeConfig
 	shouldEnableForwardingClusterTrafficViaIngress bool
 	proxyClass                                     string // configuration from the named ProxyClass should be applied to proxy resources
+	operatorVersion                                string
 }
 
 func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.StatefulSet {
@@ -197,6 +198,7 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
 			},
 		},
 	}
+	mak.Set(&ss.ObjectMeta.Annotations, annotationOperatorVersion, opts.operatorVersion)
 	// If opts.proxyClass is set, retrieve the ProxyClass and apply
 	// configuration from that to the StatefulSet.
 	if opts.proxyClass != "" {
@@ -269,6 +271,7 @@ func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *apps
 			},
 		},
 	}
+	mak.Set(&ss.ObjectMeta.Annotations, annotationOperatorVersion, opts.operatorVersion)
 	// If opts.proxyClass is set, retrieve the ProxyClass and apply
 	// configuration from that to the StatefulSet.
 	if opts.proxyClass != "" {