Try to pass config file to tailscaledirbekrm/conf

Signed-off-by: Irbe Krumina <irbe@tailscale.com>

5 files changed, 162 insertions, 76 deletions

diff --git a/cmd/containerboot/main.go b/cmd/containerboot/main.go
index 6604d5e99..93a11c43d 100644
--- a/cmd/containerboot/main.go
+++ b/cmd/containerboot/main.go
@@ -106,7 +106,7 @@ func main() {
 		Hostname:          defaultEnv("TS_HOSTNAME", ""),
 		Routes:            defaultEnvPointer("TS_ROUTES"),
 		ServeConfigPath:   defaultEnv("TS_SERVE_CONFIG", ""),
-		ConfigFilePath:    defaultEnv("TS_CONFIG_FILE_PATH", ""),
+		ConfigFilePath:    defaultEnv("TS_CONFIGFILE_PATH", ""),
 		ProxyTo:           defaultEnv("TS_DEST_IP", ""),
 		TailnetTargetIP:   defaultEnv("TS_TAILNET_TARGET_IP", ""),
 		TailnetTargetFQDN: defaultEnv("TS_TAILNET_TARGET_FQDN", ""),
@@ -294,13 +294,13 @@ authLoop:
 	ctx, cancel := contextWithExitSignalWatch()
 	defer cancel()
 
-	if cfg.AuthOnce {
-		// Now that we are authenticated, we can set/reset any of the
-		// settings that we need to.
-		if err := tailscaleSet(ctx, cfg); err != nil {
-			log.Fatalf("failed to auth tailscale: %v", err)
-		}
-	}
+	// if cfg.AuthOnce {
+	// 	// Now that we are authenticated, we can set/reset any of the
+	// 	// settings that we need to.
+	// 	// if err := tailscaleSet(ctx, cfg); err != nil {
+	// 	// 	log.Fatalf("failed to auth tailscale: %v", err)
+	// 	// }
+	// }
 
 	if cfg.ServeConfigPath != "" {
 		// Remove any serve config that may have been set by a previous run of
@@ -314,10 +314,10 @@ authLoop:
 		// We were told to only auth once, so any secret-bound
 		// authkey is no longer needed. We don't strictly need to
 		// wipe it, but it's good hygiene.
-		log.Printf("Deleting authkey from kube secret")
-		if err := deleteAuthKey(ctx, cfg.KubeSecret); err != nil {
-			log.Fatalf("deleting authkey from kube secret: %v", err)
-		}
+		// log.Printf("Deleting authkey from kube secret")
+		// if err := deleteAuthKey(ctx, cfg.KubeSecret); err != nil {
+		// 	log.Fatalf("deleting authkey from kube secret: %v", err)
+		// }
 	}
 
 	w, err = client.WatchIPNBus(ctx, ipn.NotifyInitialNetMap|ipn.NotifyInitialState)
@@ -647,34 +647,40 @@ func tailscaledArgs(cfg *settings) []string {
 // tailscaleUp uses cfg to run 'tailscale up' everytime containerboot starts, or
 // if TS_AUTH_ONCE is set, only the first time containerboot starts.
 func tailscaleUp(ctx context.Context, cfg *settings) error {
-	args := []string{"--socket=" + cfg.Socket, "up"}
-	if cfg.AcceptDNS {
-		args = append(args, "--accept-dns=true")
-	} else {
-		args = append(args, "--accept-dns=false")
+	command := "up"
+	if cfg.ConfigFilePath != "" {
+		command = "login"
 	}
+	args := []string{"--socket=" + cfg.Socket, command}
 	if cfg.AuthKey != "" {
 		args = append(args, "--authkey="+cfg.AuthKey)
 	}
-	// --advertise-routes can be passed an empty string to configure a
-	// device (that might have previously advertised subnet routes) to not
-	// advertise any routes. Respect an empty string passed by a user and
-	// use it to explicitly unset the routes.
-	if cfg.Routes != nil {
-		args = append(args, "--advertise-routes="+*cfg.Routes)
-	}
-	if cfg.Hostname != "" {
-		args = append(args, "--hostname="+cfg.Hostname)
-	}
-	if cfg.ExtraArgs != "" {
-		args = append(args, strings.Fields(cfg.ExtraArgs)...)
+	if cfg.ConfigFilePath == "" {
+		if cfg.AcceptDNS {
+			args = append(args, "--accept-dns=true")
+		} else {
+			args = append(args, "--accept-dns=false")
+		}
+		// --advertise-routes can be passed an empty string to configure a
+		// device (that might have previously advertised subnet routes) to not
+		// advertise any routes. Respect an empty string passed by a user and
+		// use it to explicitly unset the routes.
+		if cfg.Routes != nil {
+			args = append(args, "--advertise-routes="+*cfg.Routes)
+		}
+		if cfg.Hostname != "" {
+			args = append(args, "--hostname="+cfg.Hostname)
+		}
+		if cfg.ExtraArgs != "" {
+			args = append(args, strings.Fields(cfg.ExtraArgs)...)
+		}
 	}
-	log.Printf("Running 'tailscale up'")
+	log.Printf("Running 'tailscale %s' with args %#+v", command, args)
 	cmd := exec.CommandContext(ctx, "tailscale", args...)
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
 	if err := cmd.Run(); err != nil {
-		return fmt.Errorf("tailscale up failed: %v", err)
+		log.Printf("tailscale up with args %#+v failed: %v", args, err)
 	}
 	return nil
 }
@@ -682,32 +688,35 @@ func tailscaleUp(ctx context.Context, cfg *settings) error {
 // tailscaleSet uses cfg to run 'tailscale set' to set any known configuration
 // options that are passed in via environment variables. This is run after the
 // node is in Running state and only if TS_AUTH_ONCE is set.
-func tailscaleSet(ctx context.Context, cfg *settings) error {
-	args := []string{"--socket=" + cfg.Socket, "set"}
-	if cfg.AcceptDNS {
-		args = append(args, "--accept-dns=true")
-	} else {
-		args = append(args, "--accept-dns=false")
-	}
-	// --advertise-routes can be passed an empty string to configure a
-	// device (that might have previously advertised subnet routes) to not
-	// advertise any routes. Respect an empty string passed by a user and
-	// use it to explicitly unset the routes.
-	if cfg.Routes != nil {
-		args = append(args, "--advertise-routes="+*cfg.Routes)
-	}
-	if cfg.Hostname != "" {
-		args = append(args, "--hostname="+cfg.Hostname)
-	}
-	log.Printf("Running 'tailscale set'")
-	cmd := exec.CommandContext(ctx, "tailscale", args...)
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	if err := cmd.Run(); err != nil {
-		return fmt.Errorf("tailscale set failed: %v", err)
-	}
-	return nil
-}
+// func tailscaleSet(ctx context.Context, cfg *settings) error {
+// 	args := []string{"--socket=" + cfg.Socket, "set"}
+// 	// TODO: fix
+// 	if cfg.ConfigFilePath == "" {
+// 		if cfg.AcceptDNS {
+// 			args = append(args, "--accept-dns=true")
+// 		} else {
+// 			args = append(args, "--accept-dns=false")
+// 		}
+// 		// --advertise-routes can be passed an empty string to configure a
+// 		// device (that might have previously advertised subnet routes) to not
+// 		// advertise any routes. Respect an empty string passed by a user and
+// 		// use it to explicitly unset the routes.
+// 		if cfg.Routes != nil {
+// 			args = append(args, "--advertise-routes="+*cfg.Routes)
+// 		}
+// 		if cfg.Hostname != "" {
+// 			args = append(args, "--hostname="+cfg.Hostname)
+// 		}
+// 	}
+// 	log.Printf("Running 'tailscale set'")
+// 	cmd := exec.CommandContext(ctx, "tailscale", args...)
+// 	cmd.Stdout = os.Stdout
+// 	cmd.Stderr = os.Stderr
+// 	if err := cmd.Run(); err != nil {
+// 		return fmt.Errorf("tailscale set failed: %v", err)
+// 	}
+// 	return nil
+// }
 
 // ensureTunFile checks that /dev/net/tun exists, creating it if
 // missing.
diff --git a/cmd/k8s-operator/connector.go b/cmd/k8s-operator/connector.go
index a6235da3b..30298f07f 100644
--- a/cmd/k8s-operator/connector.go
+++ b/cmd/k8s-operator/connector.go
@@ -7,6 +7,7 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"net/netip"
 	"slices"
@@ -24,8 +25,10 @@ import (
 	"k8s.io/client-go/tools/record"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"tailscale.com/ipn"
 	tsoperator "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/net/netutil"
 	"tailscale.com/tstime"
 	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/set"
@@ -198,6 +201,59 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge
 	return err
 }
 
+func (a *tailscaleSTSReconciler) tsConfigCM(ctx context.Context, name, namespace string, logger *zap.SugaredLogger, sts *tailscaleSTSConfig) error {
+	confFile, err := confFile(sts)
+	if err != nil {
+		return fmt.Errorf("error provisioning config: %v", err)
+	}
+
+	jsonBytes, err := json.Marshal(confFile)
+	if err != nil {
+		return fmt.Errorf("error marshaling config file: %v", err)
+	}
+	cm := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: namespace,
+			Labels:    sts.ChildResourceLabels,
+		},
+		Data: map[string]string{
+			"tailscaled": string(jsonBytes),
+		},
+	}
+	_, err = createOrUpdate(ctx, a.Client, namespace, cm, func(config *corev1.ConfigMap) { config.Labels = cm.Labels; config.Data = cm.Data })
+	if err != nil {
+		return fmt.Errorf("error creating a ConfigMap: %v", err)
+	}
+	return nil
+}
+
+func confFile(sts *tailscaleSTSConfig) (*ipn.ConfigVAlpha, error) {
+	var (
+		routes []netip.Prefix
+		err    error
+	)
+	if sts.Connector != nil {
+		routes, err = netutil.CalcAdvertiseRoutes(sts.Connector.routes, sts.Connector.isExitNode)
+		if err != nil {
+			return nil, fmt.Errorf("error calculating routes: %v", err)
+		}
+	}
+	conf := &ipn.ConfigVAlpha{
+		Version:         "alpha0",
+		AdvertiseRoutes: routes,
+		AcceptDNS:       "false",
+		Hostname:        &sts.Hostname,
+		// Not sure how to log in if it's locked?
+		Locked: "false",
+	}
+	// fix - don't put the key there
+	if sts.key != "" {
+		conf.AuthKey = &sts.key
+	}
+	return conf, nil
+}
+
 func (a *ConnectorReconciler) maybeCleanupConnector(ctx context.Context, logger *zap.SugaredLogger, cn *tsapi.Connector) (bool, error) {
 	if done, err := a.ssr.Cleanup(ctx, logger, childResourceLabels(cn.Name, a.tsnamespace, "connector")); err != nil {
 		return false, fmt.Errorf("failed to cleanup Connector resources: %w", err)
diff --git a/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml b/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml
index fbd83e7e1..74693e7b2 100644
--- a/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml
+++ b/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml
@@ -21,6 +21,9 @@ rules:
 - apiGroups: ["tailscale.com"]
   resources: ["connectors", "connectors/status"]
   verbs: ["get", "list", "watch", "update"]
+- apiGroups: [""]
+  resources: ["configmaps"]
+  verbs: ["*"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go
index a785ed6fb..d24a6eb02 100644
--- a/cmd/k8s-operator/sts.go
+++ b/cmd/k8s-operator/sts.go
@@ -11,6 +11,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"log"
 	"net/http"
 	"os"
 	"strings"
@@ -26,7 +27,6 @@ import (
 	"sigs.k8s.io/yaml"
 	"tailscale.com/client/tailscale"
 	"tailscale.com/ipn"
-	"tailscale.com/net/netutil"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tsnet"
 	"tailscale.com/types/opt"
@@ -83,6 +83,9 @@ type tailscaleSTSConfig struct {
 	// Connector specifies a configuration of a Connector instance if that's
 	// what this StatefulSet should be created for.
 	Connector *connector
+
+	// temp fix whilst prototyping, remove
+	key string
 }
 
 type connector struct {
@@ -297,6 +300,7 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *
 		mak.Set(&secret.StringData, "serve-config", string(j))
 	}
 	if orig != nil {
+		log.Printf("Patching existing secret %s", secret.Name)
 		if err := a.Patch(ctx, secret, client.MergeFrom(orig)); err != nil {
 			return "", err
 		}
@@ -304,7 +308,9 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *
 		if err := a.Create(ctx, secret); err != nil {
 			return "", err
 		}
+		log.Printf("Created secret %s", secret.Name)
 	}
+	log.Printf("Created secret %s", secret.Name)
 	return secret.Name, nil
 }
 
@@ -428,17 +434,37 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
 			},
 		})
 	} else if sts.Connector != nil {
-		// We need to provide these env vars even if the values are empty to
-		// ensure that a transition from a Connector with a defined subnet
-		// router or exit node to one without succeeds.
+		// TODO: definitely not the right place for this
+		a.tsConfigCM(ctx, headlessSvc.Name, a.operatorNamespace, logger, sts)
 		container.Env = append(container.Env, corev1.EnvVar{
-			Name:  "TS_EXTRA_ARGS",
-			Value: fmt.Sprintf("--advertise-exit-node=%v", sts.Connector.isExitNode),
+			Name:  "TS_CONFIGFILE_PATH",
+			Value: "/tsconfig/tailscaled",
 		})
-		container.Env = append(container.Env, corev1.EnvVar{
-			Name:  "TS_ROUTES",
-			Value: sts.Connector.routes,
+		ss.Spec.Template.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, corev1.Volume{
+			Name: "configfile",
+			VolumeSource: corev1.VolumeSource{
+				ConfigMap: &corev1.ConfigMapVolumeSource{
+					LocalObjectReference: corev1.LocalObjectReference{
+						Name: headlessSvc.Name,
+					},
+				},
+			},
 		})
+		container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{
+			Name:      "configfile",
+			MountPath: "/tsconfig",
+		})
+		// We need to provide these env vars even if the values are empty to
+		// ensure that a transition from a Connector with a defined subnet
+		// router or exit node to one without succeeds.
+		// container.Env = append(container.Env, corev1.EnvVar{
+		// 	Name:  "TS_EXTRA_ARGS",
+		// 	Value: fmt.Sprintf("--advertise-exit-node=%v", sts.Connector.isExitNode),
+		// })
+		// container.Env = append(container.Env, corev1.EnvVar{
+		// 	Name:  "TS_ROUTES",
+		// 	Value: sts.Connector.routes,
+		// })
 	}
 	if a.tsFirewallMode != "" {
 		container.Env = append(container.Env, corev1.EnvVar{
@@ -482,16 +508,6 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
 	return createOrUpdate(ctx, a.Client, a.operatorNamespace, &ss, func(s *appsv1.StatefulSet) { s.Spec = ss.Spec })
 }
 
-func confFile(sts *tailscaleSTSConfig) (*ipn.ConfigVAlpha, error) {
-	if sts.connector != nil {}
-	routes, err := netutil.CalcAdvertiseRoutes(sts., advertiseDefaultRoute)
-
-	return &ipn.ConfigVAlpha{
-		Hostname: &sts.Hostname,
-	}, nil
-
-}
-
 // ptrObject is a type constraint for pointer types that implement
 // client.Object.
 type ptrObject[T any] interface {
diff --git a/cmd/tailscaled/tailscaled.go b/cmd/tailscaled/tailscaled.go
index 390c6f39e..4ecbd8e56 100644
--- a/cmd/tailscaled/tailscaled.go
+++ b/cmd/tailscaled/tailscaled.go
@@ -332,11 +332,13 @@ func run() (err error) {
 	// Parse config, if specified, to fail early if it's invalid.
 	var conf *conffile.Config
 	if args.confFile != "" {
+		logf("loading config file")
 		conf, err = conffile.Load(args.confFile)
 		if err != nil {
 			return fmt.Errorf("error reading config file: %w", err)
 		}
 		sys.InitialConfig = conf
+		logf("loaded initial config: %#+v", string(sys.InitialConfig.Raw))
 	}
 
 	var netMon *netmon.Monitor