k8s-operator: adding final testsirbekrm/ingress_services

Signed-off-by: chaosinthecrd <tom@tmlabs.co.uk>

3 files changed, 178 insertions, 121 deletions

diff --git a/cmd/k8s-operator/operator_test.go b/cmd/k8s-operator/operator_test.go
index 175003ac7..f4b0db01c 100644
--- a/cmd/k8s-operator/operator_test.go
+++ b/cmd/k8s-operator/operator_test.go
@@ -1802,6 +1802,54 @@ func Test_metricsResourceCreation(t *testing.T) {
 	// object). We cannot test this using the fake client.
 }
 
+func TestIgnorePGService(t *testing.T) {
+	// NOTE: creating proxygroup stuff just to be sure that it's all ignored
+	_, _, fc, _ := setupServiceTest(t)
+
+	ft := &fakeTSClient{}
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
+	clock := tstest.NewClock(tstest.ClockOpts{})
+	sr := &ServiceReconciler{
+		Client: fc,
+		ssr: &tailscaleSTSReconciler{
+			Client:            fc,
+			tsClient:          ft,
+			defaultTags:       []string{"tag:k8s"},
+			operatorNamespace: "operator-ns",
+			proxyImage:        "tailscale/tailscale",
+		},
+		logger: zl.Sugar(),
+		clock:  clock,
+	}
+
+	// Create a service that we should manage, and check that the initial round
+	// of objects looks right.
+	mustCreate(t, fc, &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: "default",
+			// The apiserver is supposed to set the UID, but the fake client
+			// doesn't. So, set it explicitly because other code later depends
+			// on it being set.
+			UID: types.UID("1234-UID"),
+			Annotations: map[string]string{
+				"tailscale.com/proxygroup": "test-pg",
+			},
+		},
+		Spec: corev1.ServiceSpec{
+			ClusterIP: "10.20.30.40",
+			Type:      corev1.ServiceTypeClusterIP,
+		},
+	})
+
+	expectReconciled(t, sr, "default", "test")
+
+	findNoGenName(t, fc, "default", "test", "svc")
+}
+
 func toFQDN(t *testing.T, s string) dnsname.FQDN {
 	t.Helper()
 	fqdn, err := dnsname.ToFQDN(s)
diff --git a/cmd/k8s-operator/svc-for-pg_test.go b/cmd/k8s-operator/svc-for-pg_test.go
index 239b56ff7..1ba85e861 100644
--- a/cmd/k8s-operator/svc-for-pg_test.go
+++ b/cmd/k8s-operator/svc-for-pg_test.go
@@ -145,7 +145,7 @@ func setupServiceTest(t *testing.T) (*HAServiceReconciler, *corev1.Secret, clien
 			Labels:    pgSecretLabels("test-pg", "config"),
 		},
 		Data: map[string][]byte{
-			tsoperator.TailscaledConfigFileName(106): []byte("{}"),
+			tsoperator.TailscaledConfigFileName(106): []byte(`{"Version":""}`),
 		},
 	}
 
@@ -224,139 +224,74 @@ func setupServiceTest(t *testing.T) (*HAServiceReconciler, *corev1.Secret, clien
 	return svcPGR, pgStateSecret, fc, ft
 }
 
-func setupTestService(t *testing.T, svcName string, hostname string, clusterIP string, fc client.Client, stateSecret *corev1.Secret) (svc *corev1.Service, eps *discoveryv1.EndpointSlice) {
-	uid := rand.IntN(100)
-	svc = &corev1.Service{
-		TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"},
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      svcName,
-			Namespace: "default",
-			UID:       types.UID(fmt.Sprintf("%d-UID", uid)),
-			Annotations: map[string]string{
-				"tailscale.com/proxy-group": "test-pg",
-			},
-		},
-		Spec: corev1.ServiceSpec{
-			Type:              corev1.ServiceTypeLoadBalancer,
-			LoadBalancerClass: ptr.To("tailscale"),
-			ClusterIP:         clusterIP,
-			ClusterIPs:        []string{clusterIP},
-		},
+func TestServicePGReconciler_MultiCluster(t *testing.T) {
+	var ft *fakeTSClient
+	var lc localClient
+	for i := 0; i <= 10; i++ {
+		pgr, stateSecret, fc, fti := setupServiceTest(t)
+		if i == 0 {
+			ft = fti
+			lc = pgr.lc
+		} else {
+			pgr.tsClient = ft
+			pgr.lc = lc
+		}
+
+		svc, _ := setupTestService(t, "test-multi-cluster", "", "4.3.2.1", fc, stateSecret)
+		expectReconciled(t, pgr, "default", svc.Name)
+
+		vipSvcs, err := ft.ListVIPServices(context.Background())
+		if err != nil {
+			t.Fatalf("getting VIPService: %v", err)
+		}
+
+		if len(vipSvcs) != 1 {
+			t.Fatalf("unexpected number of VIPServices (%d)", len(vipSvcs))
+		}
+
+		for name := range vipSvcs {
+			t.Logf("found vip service with name %q", name.String())
+		}
 	}
+}
 
-	eps = &discoveryv1.EndpointSlice{
-		TypeMeta: metav1.TypeMeta{Kind: "EndpointSlice", APIVersion: "v1"},
+func TestIgnoreRegularService(t *testing.T) {
+	pgr, _, fc, ft := setupServiceTest(t)
+
+	svc := &corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      svcName,
+			Name:      "test",
 			Namespace: "default",
-			Labels: map[string]string{
-				discoveryv1.LabelServiceName: svcName,
+			// The apiserver is supposed to set the UID, but the fake client
+			// doesn't. So, set it explicitly because other code later depends
+			// on it being set.
+			UID: types.UID("1234-UID"),
+			Annotations: map[string]string{
+				"tailscale.com/expose": "true",
 			},
 		},
-		AddressType: discoveryv1.AddressTypeIPv4,
-		Endpoints: []discoveryv1.Endpoint{
-			{
-				Addresses: []string{"4.3.2.1"},
-				Conditions: discoveryv1.EndpointConditions{
-					Ready: ptr.To(true),
-				},
-			},
+		Spec: corev1.ServiceSpec{
+			ClusterIP: "10.20.30.40",
+			Type:      corev1.ServiceTypeClusterIP,
 		},
 	}
 
-	updateIngressConfigSecret(t, fc, stateSecret, fmt.Sprintf("default-%s", svcName), clusterIP)
+	// hostname := svc.Namespace + svc.Name
 
 	mustCreate(t, fc, svc)
-	mustCreate(t, fc, eps)
+	expectReconciled(t, pgr, "default", "test")
 
-	return svc, eps
-}
+	verifyTailscaledConfig(t, fc, nil)
+
+	vipSvcs, err := ft.ListVIPServices(context.Background())
+	if err == nil {
+		t.Fatalf("failed to list VIPServices")
+	}
 
-//	func TestServicePGReconciler_MultiCluster(t *testing.T) {
-//		ingPGR, fc, ft := setupIngressTest(t)
-//		ingPGR.operatorID = "operator-1"
-//
-//		// Create initial Ingress
-//		ing := &networkingv1.Ingress{
-//			TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
-//			ObjectMeta: metav1.ObjectMeta{
-//				Name:      "test-ingress",
-//				Namespace: "default",
-//				UID:       types.UID("1234-UID"),
-//				Annotations: map[string]string{
-//					"tailscale.com/proxy-group": "test-pg",
-//				},
-//			},
-//			Spec: networkingv1.IngressSpec{
-//				IngressClassName: ptr.To("tailscale"),
-//				TLS: []networkingv1.IngressTLS{
-//					{Hosts: []string{"my-svc"}},
-//				},
-//			},
-//		}
-//		mustCreate(t, fc, ing)
-//
-//		// Simulate existing VIPService from another cluster
-//		existingVIPSvc := &tailscale.VIPService{
-//			Name: "svc:my-svc",
-//			Annotations: map[string]string{
-//				ownerAnnotation: `{"ownerrefs":[{"operatorID":"operator-2"}]}`,
-//			},
-//		}
-//		ft.vipServices = map[tailcfg.ServiceName]*tailscale.VIPService{
-//			"svc:my-svc": existingVIPSvc,
-//		}
-//
-//		// Verify reconciliation adds our operator reference
-//		expectReconciled(t, ingPGR, "default", "test-ingress")
-//
-//		vipSvc, err := ft.GetVIPService(context.Background(), "svc:my-svc")
-//		if err != nil {
-//			t.Fatalf("getting VIPService: %v", err)
-//		}
-//		if vipSvc == nil {
-//			t.Fatal("VIPService not found")
-//		}
-//
-//		o, err := parseOwnerAnnotation(vipSvc)
-//		if err != nil {
-//			t.Fatalf("parsing owner annotation: %v", err)
-//		}
-//
-//		wantOwnerRefs := []OwnerRef{
-//			{OperatorID: "operator-2"},
-//			{OperatorID: "operator-1"},
-//		}
-//		if !reflect.DeepEqual(o.OwnerRefs, wantOwnerRefs) {
-//			t.Errorf("incorrect owner refs\ngot:  %+v\nwant: %+v", o.OwnerRefs, wantOwnerRefs)
-//		}
-//
-//		// Delete the Ingress and verify VIPService still exists with one owner ref
-//		if err := fc.Delete(context.Background(), ing); err != nil {
-//			t.Fatalf("deleting Ingress: %v", err)
-//		}
-//		expectRequeue(t, ingPGR, "default", "test-ingress")
-//
-//		vipSvc, err = ft.GetVIPService(context.Background(), "svc:my-svc")
-//		if err != nil {
-//			t.Fatalf("getting VIPService after deletion: %v", err)
-//		}
-//		if vipSvc == nil {
-//			t.Fatal("VIPService was incorrectly deleted")
-//		}
-//
-//		o, err = parseOwnerAnnotation(vipSvc)
-//		if err != nil {
-//			t.Fatalf("parsing owner annotation: %v", err)
-//		}
-//
-//		wantOwnerRefs = []OwnerRef{
-//			{OperatorID: "operator-2"},
-//		}
-//		if !reflect.DeepEqual(o.OwnerRefs, wantOwnerRefs) {
-//			t.Errorf("incorrect owner refs after deletion\ngot:  %+v\nwant: %+v", o.OwnerRefs, wantOwnerRefs)
-//		}
-//	}
+	if len(vipSvcs) > 0 {
+		t.Fatal("unexpected vip services found")
+	}
+}
 
 func removeEl(s []string, value string) []string {
 	result := s[:0]
@@ -392,3 +327,51 @@ func updateIngressConfigSecret(t *testing.T, fc client.Client, stateSecret *core
 		mak.Set(&sec.Data, ingressservices.IngressConfigKey, icJson)
 	})
 }
+
+func setupTestService(t *testing.T, svcName string, hostname string, clusterIP string, fc client.Client, stateSecret *corev1.Secret) (svc *corev1.Service, eps *discoveryv1.EndpointSlice) {
+	uid := rand.IntN(100)
+	svc = &corev1.Service{
+		TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      svcName,
+			Namespace: "default",
+			UID:       types.UID(fmt.Sprintf("%d-UID", uid)),
+			Annotations: map[string]string{
+				"tailscale.com/proxy-group": "test-pg",
+			},
+		},
+		Spec: corev1.ServiceSpec{
+			Type:              corev1.ServiceTypeLoadBalancer,
+			LoadBalancerClass: ptr.To("tailscale"),
+			ClusterIP:         clusterIP,
+			ClusterIPs:        []string{clusterIP},
+		},
+	}
+
+	eps = &discoveryv1.EndpointSlice{
+		TypeMeta: metav1.TypeMeta{Kind: "EndpointSlice", APIVersion: "v1"},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      svcName,
+			Namespace: "default",
+			Labels: map[string]string{
+				discoveryv1.LabelServiceName: svcName,
+			},
+		},
+		AddressType: discoveryv1.AddressTypeIPv4,
+		Endpoints: []discoveryv1.Endpoint{
+			{
+				Addresses: []string{"4.3.2.1"},
+				Conditions: discoveryv1.EndpointConditions{
+					Ready: ptr.To(true),
+				},
+			},
+		},
+	}
+
+	updateIngressConfigSecret(t, fc, stateSecret, fmt.Sprintf("default-%s", svcName), clusterIP)
+
+	mustCreate(t, fc, svc)
+	mustCreate(t, fc, eps)
+
+	return svc, eps
+}
diff --git a/cmd/k8s-operator/testutils_test.go b/cmd/k8s-operator/testutils_test.go
index 209864011..3d9bdbf9a 100644
--- a/cmd/k8s-operator/testutils_test.go
+++ b/cmd/k8s-operator/testutils_test.go
@@ -565,6 +565,23 @@ func expectedSecret(t *testing.T, cl client.Client, opts configOpts) *corev1.Sec
 	return s
 }
 
+func findNoGenName(t *testing.T, client client.Client, ns, name, typ string) {
+	t.Helper()
+	labels := map[string]string{
+		kubetypes.LabelManaged: "true",
+		LabelParentName:        name,
+		LabelParentNamespace:   ns,
+		LabelParentType:        typ,
+	}
+	s, err := getSingleObject[corev1.Secret](context.Background(), client, "operator-ns", labels)
+	if err != nil {
+		t.Fatalf("finding secrets for %q: %v", name, err)
+	}
+	if s != nil {
+		t.Fatalf("found unexpected secret with name %q", s.GetName())
+	}
+}
+
 func findGenName(t *testing.T, client client.Client, ns, name, typ string) (full, noSuffix string) {
 	t.Helper()
 	labels := map[string]string{
@@ -893,6 +910,15 @@ func (c *fakeTSClient) GetVIPService(ctx context.Context, name tailcfg.ServiceNa
 	return svc, nil
 }
 
+func (c *fakeTSClient) ListVIPServices(ctx context.Context) (map[tailcfg.ServiceName]*tailscale.VIPService, error) {
+	c.Lock()
+	defer c.Unlock()
+	if c.vipServices == nil {
+		return nil, &tailscale.ErrResponse{Status: http.StatusNotFound}
+	}
+	return c.vipServices, nil
+}
+
 func (c *fakeTSClient) CreateOrUpdateVIPService(ctx context.Context, svc *tailscale.VIPService) error {
 	c.Lock()
 	defer c.Unlock()