WIP: Support --config=kube:<secret-name>tomhjp/tailscaled-kube-conf

11 files changed, 138 insertions, 94 deletions

diff --git a/cmd/containerboot/kube.go b/cmd/containerboot/kube.go
index 908cc01ef..8396f660d 100644
--- a/cmd/containerboot/kube.go
+++ b/cmd/containerboot/kube.go
@@ -8,11 +8,9 @@ package main
 import (
 	"context"
 	"encoding/json"
-	"fmt"
 	"log"
 	"net/http"
 	"net/netip"
-	"os"
 
 	"tailscale.com/kube/kubeapi"
 	"tailscale.com/kube/kubeclient"
@@ -85,9 +83,4 @@ func initKubeClient(root string) {
 	if err != nil {
 		log.Fatalf("Error creating kube client: %v", err)
 	}
-	if (root != "/") || os.Getenv("TS_KUBERNETES_READ_API_SERVER_ADDRESS_FROM_ENV") == "true" {
-		// Derive the API server address from the environment variables
-		// Used to set http server in tests, or optionally enabled by flag
-		kc.SetURL(fmt.Sprintf("https://%s:%s", os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT_HTTPS")))
-	}
 }
diff --git a/cmd/containerboot/main.go b/cmd/containerboot/main.go
index 4c8ba5807..dcc0f026b 100644
--- a/cmd/containerboot/main.go
+++ b/cmd/containerboot/main.go
@@ -96,13 +96,13 @@ import (
 	"errors"
 	"fmt"
 	"io/fs"
+	"iter"
 	"log"
 	"math"
 	"net"
 	"net/netip"
 	"os"
 	"os/signal"
-	"path"
 	"path/filepath"
 	"slices"
 	"strings"
@@ -721,26 +721,26 @@ func tailscaledConfigFilePath() string {
 	if err != nil {
 		log.Fatalf("error reading tailscaled config directory %q: %v", dir, err)
 	}
-	maxCompatVer := tailcfg.CapabilityVersion(-1)
-	for _, e := range fe {
-		// We don't check if type if file as in most cases this will
-		// come from a mounted kube Secret, where the directory contents
-		// will be various symlinks.
-		if e.Type().IsDir() {
-			continue
-		}
-		cv, err := kubeutils.CapVerFromFileName(e.Name())
-		if err != nil {
-			log.Printf("skipping file %q in tailscaled config directory %q: %v", e.Name(), dir, err)
-			continue
-		}
-		if cv > maxCompatVer && cv <= tailcfg.CurrentCapabilityVersion {
-			maxCompatVer = cv
-		}
-	}
-	if maxCompatVer == -1 {
+	selectedFile := kubeutils.SelectConfigFileName(fileNames(fe))
+	if selectedFile == "" {
 		log.Fatalf("no tailscaled config file found in %q for current capability version %q", dir, tailcfg.CurrentCapabilityVersion)
 	}
-	log.Printf("Using tailscaled config file %q for capability version %q", maxCompatVer, tailcfg.CurrentCapabilityVersion)
-	return path.Join(dir, kubeutils.TailscaledConfigFileName(maxCompatVer))
+	log.Printf("Using tailscaled config file %q", selectedFile)
+	return filepath.Join(dir, selectedFile)
+}
+
+func fileNames(fe []fs.DirEntry) iter.Seq[string] {
+	return func(yield func(string) bool) {
+		for _, e := range fe {
+			// We don't check if type if file as in most cases this will
+			// come from a mounted kube Secret, where the directory contents
+			// will be various symlinks.
+			if e.Type().IsDir() {
+				continue
+			}
+			if !yield(e.Name()) {
+				return
+			}
+		}
+	}
 }
diff --git a/cmd/k8s-operator/proxygroup.go b/cmd/k8s-operator/proxygroup.go
index 1f9983aa9..875bb089a 100644
--- a/cmd/k8s-operator/proxygroup.go
+++ b/cmd/k8s-operator/proxygroup.go
@@ -459,6 +459,7 @@ func pgTailscaledConfig(pg *tsapi.ProxyGroup, class *tsapi.ProxyClass, idx int32
 		conf.AuthKey = key
 	}
 	capVerConfigs := make(map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha)
+	capVerConfigs[0] = *conf // Becomes "tailscaled" key.
 	capVerConfigs[106] = *conf
 	return capVerConfigs, nil
 }
diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go
index 6378a8263..7582ee34c 100644
--- a/cmd/k8s-operator/sts.go
+++ b/cmd/k8s-operator/sts.go
@@ -12,6 +12,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"maps"
 	"net/http"
 	"os"
 	"slices"
@@ -359,19 +360,12 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *
 		return "", "", nil, fmt.Errorf("error calculating hash of tailscaled configs: %w", err)
 	}
 
-	latest := tailcfg.CapabilityVersion(-1)
-	var latestConfig ipn.ConfigVAlpha
-	for key, val := range configs {
-		fn := tsoperator.TailscaledConfigFileName(key)
-		b, err := json.Marshal(val)
+	for capVer, cfg := range configs {
+		b, err := json.Marshal(cfg)
 		if err != nil {
 			return "", "", nil, fmt.Errorf("error marshalling tailscaled config: %w", err)
 		}
-		mak.Set(&secret.StringData, fn, string(b))
-		if key > latest {
-			latest = key
-			latestConfig = val
-		}
+		mak.Set(&secret.StringData, tsoperator.TailscaledConfigFileName(capVer), string(b))
 	}
 
 	if stsC.ServeConfig != nil {
@@ -383,12 +377,12 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *
 	}
 
 	if orig != nil {
-		logger.Debugf("patching the existing proxy Secret with tailscaled config %s", sanitizeConfigBytes(latestConfig))
+		logger.Debugf("patching the existing proxy Secret with tailscaled config %s", sanitizeConfigBytes(configs))
 		if err := a.Patch(ctx, secret, client.MergeFrom(orig)); err != nil {
 			return "", "", nil, err
 		}
 	} else {
-		logger.Debugf("creating a new Secret for the proxy with tailscaled config %s", sanitizeConfigBytes(latestConfig))
+		logger.Debugf("creating a new Secret for the proxy with tailscaled config %s", sanitizeConfigBytes(configs))
 		if err := a.Create(ctx, secret); err != nil {
 			return "", "", nil, err
 		}
@@ -396,13 +390,21 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *
 	return secret.Name, hash, configs, nil
 }
 
-// sanitizeConfigBytes returns ipn.ConfigVAlpha in string form with redacted
-// auth key.
-func sanitizeConfigBytes(c ipn.ConfigVAlpha) string {
-	if c.AuthKey != nil {
-		c.AuthKey = ptr.To("**redacted**")
+// sanitizeConfigBytes returns latest ipn.ConfigVAlpha in string form with
+// redacted auth key.
+func sanitizeConfigBytes(c tailscaledConfigs) string {
+	maxCapVer := tailcfg.CapabilityVersion(-1)
+	var latestConfig ipn.ConfigVAlpha
+	for capVer, cfg := range c {
+		if (capVer > maxCapVer && maxCapVer != 0) || capVer == 0 {
+			maxCapVer = capVer
+			latestConfig = cfg
+		}
 	}
-	sanitizedBytes, err := json.Marshal(c)
+	if latestConfig.AuthKey != nil {
+		latestConfig.AuthKey = ptr.To("**redacted**")
+	}
+	sanitizedBytes, err := json.Marshal(latestConfig)
 	if err != nil {
 		return "invalid config"
 	}
@@ -831,6 +833,7 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *co
 		conf.AuthKey = key
 	}
 	capVerConfigs := make(map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha)
+	capVerConfigs[0] = *conf // Becomes "tailscaled" key.
 	capVerConfigs[95] = *conf
 	// legacy config should not contain NoStatefulFiltering field.
 	conf.NoStatefulFiltering.Clear()
@@ -838,30 +841,16 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *co
 	return capVerConfigs, nil
 }
 
-func authKeyFromSecret(s *corev1.Secret) (key *string, err error) {
-	latest := tailcfg.CapabilityVersion(-1)
-	latestStr := ""
-	for k, data := range s.Data {
-		// write to StringData, read from Data as StringData is write-only
-		if len(data) == 0 {
-			continue
-		}
-		v, err := tsoperator.CapVerFromFileName(k)
-		if err != nil {
-			continue
-		}
-		if v > latest {
-			latestStr = k
-			latest = v
-		}
-	}
+func authKeyFromSecret(s *corev1.Secret) (*string, error) {
+	selectedKey := tsoperator.SelectConfigFileName(maps.Keys(s.Data))
 	// Allow for configs that don't contain an auth key. Perhaps
 	// users have some mechanisms to delete them. Auth key is
 	// normally not needed after the initial login.
-	if latestStr != "" {
-		return readAuthKey(s, latestStr)
+	if selectedKey == "" {
+		return nil, nil
 	}
-	return key, nil
+
+	return readAuthKey(s, selectedKey)
 }
 
 // shouldRetainAuthKey returns true if the state stored in a proxy's state Secret suggests that auth key should be
diff --git a/cmd/tailscaled/tailscaled.go b/cmd/tailscaled/tailscaled.go
index 2831b4061..23b615274 100644
--- a/cmd/tailscaled/tailscaled.go
+++ b/cmd/tailscaled/tailscaled.go
@@ -119,7 +119,7 @@ var args struct {
 	tunname string
 
 	cleanUp        bool
-	confFile       string // empty, file path, or "vm:user-data"
+	confFile       string // empty, file path, or "vm:user-data", or "kube:<secret-name>"
 	debug          string
 	port           uint16
 	statepath      string
@@ -166,13 +166,13 @@ func main() {
 	flag.StringVar(&args.httpProxyAddr, "outbound-http-proxy-listen", "", `optional [ip]:port to run an outbound HTTP proxy (e.g. "localhost:8080")`)
 	flag.StringVar(&args.tunname, "tun", defaultTunName(), `tunnel interface name; use "userspace-networking" (beta) to not use TUN`)
 	flag.Var(flagtype.PortValue(&args.port, defaultPort()), "port", "UDP port to listen on for WireGuard and peer-to-peer traffic; 0 means automatically select")
-	flag.StringVar(&args.statepath, "state", "", "absolute path of state file; use 'kube:<secret-name>' to use Kubernetes secrets or 'arn:aws:ssm:...' to store in AWS SSM; use 'mem:' to not store state and register as an ephemeral node. If empty and --statedir is provided, the default is <statedir>/tailscaled.state. Default: "+paths.DefaultTailscaledStateFile())
+	flag.StringVar(&args.statepath, "state", "", "absolute path of state file; use 'kube:<secret-name>' to use a Kubernetes Secret or 'arn:aws:ssm:...' to store in AWS SSM; use 'mem:' to not store state and register as an ephemeral node. If empty and --statedir is provided, the default is <statedir>/tailscaled.state. Default: "+paths.DefaultTailscaledStateFile())
 	flag.StringVar(&args.statedir, "statedir", "", "path to directory for storage of config state, TLS certs, temporary incoming Taildrop files, etc. If empty, it's derived from --state when possible.")
 	flag.StringVar(&args.socketpath, "socket", paths.DefaultTailscaledSocket(), "path of the service unix socket")
 	flag.StringVar(&args.birdSocketPath, "bird-socket", "", "path of the bird unix socket")
 	flag.BoolVar(&printVersion, "version", false, "print version information and exit")
 	flag.BoolVar(&args.disableLogs, "no-logs-no-support", false, "disable log uploads; this also disables any technical support")
-	flag.StringVar(&args.confFile, "config", "", "path to config file, or 'vm:user-data' to use the VM's user-data (EC2)")
+	flag.StringVar(&args.confFile, "config", "", "path to config file, or 'vm:user-data' to use the VM's user-data (EC2), or 'kube:<secret-name>' to read the '.data.tailscaled' key from a Kubernetes Secret")
 
 	if len(os.Args) > 0 && filepath.Base(os.Args[0]) == "tailscale" && beCLI != nil {
 		beCLI()
diff --git a/ipn/conffile/conffile.go b/ipn/conffile/conffile.go
index 0b4670c42..d81f2f154 100644
--- a/ipn/conffile/conffile.go
+++ b/ipn/conffile/conffile.go
@@ -10,6 +10,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"strings"
 
 	"github.com/tailscale/hujson"
 	"tailscale.com/ipn"
@@ -17,7 +18,7 @@ import (
 
 // Config describes a config file.
 type Config struct {
-	Path    string // disk path of HuJSON, or VMUserDataPath
+	Path    string // disk path of HuJSON, or [VMUserDataPath], or kube:<secret-name>
 	Raw     []byte // raw bytes from disk, in HuJSON form
 	Std     []byte // standardized JSON form
 	Version string // "alpha0" for now
@@ -35,9 +36,16 @@ func (c *Config) WantRunning() bool {
 	return c != nil && !c.Parsed.Enabled.EqualBool(false)
 }
 
-// VMUserDataPath is a sentinel value for Load to use to get the data
-// from the VM's metadata service's user-data field.
-const VMUserDataPath = "vm:user-data"
+const (
+	// VMUserDataPath is a sentinel value for Load to use to get the data
+	// from the VM's metadata service's user-data field.
+	VMUserDataPath = "vm:user-data"
+
+	// kubePrefix indicates the config should be read from a Kubernetes Secret.
+	// The remaining string should be the name of the Secret within the same
+	// namespace as tailscaled's own pod.
+	kubePrefix = "kube:"
+)
 
 // Load reads and parses the config file at the provided path on disk.
 func Load(path string) (*Config, error) {
@@ -45,9 +53,11 @@ func Load(path string) (*Config, error) {
 	c.Path = path
 	var err error
 
-	switch path {
-	case VMUserDataPath:
+	switch {
+	case path == VMUserDataPath:
 		c.Raw, err = readVMUserData()
+	case strings.HasPrefix(path, "kube:"):
+		c.Raw, err = readKubeSecret(strings.TrimPrefix(path, "kube:"))
 	default:
 		c.Raw, err = os.ReadFile(path)
 	}
@@ -74,7 +84,9 @@ func Load(path string) (*Config, error) {
 	c.Version = ver.Version
 
 	jd := json.NewDecoder(bytes.NewReader(c.Std))
-	jd.DisallowUnknownFields()
+	// Not not disallow unknown fields. Older clients need to be able to read
+	// newer configs to support version tearing between the creator and
+	// consumer of config files.
 	err = jd.Decode(&c.Parsed)
 	if err != nil {
 		return nil, fmt.Errorf("error parsing config file %s: %w", path, err)
diff --git a/ipn/conffile/kube.go b/ipn/conffile/kube.go
new file mode 100644
index 000000000..aeb4f8748
--- /dev/null
+++ b/ipn/conffile/kube.go
@@ -0,0 +1,32 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package conffile
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"tailscale.com/kube/kubeclient"
+)
+
+func readKubeSecret(name string) ([]byte, error) {
+	c, err := kubeclient.New()
+	if err != nil {
+		return nil, err
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	secret, err := c.GetSecret(ctx, name)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read config from Secret %q: %w", name, err)
+	}
+
+	if contents, ok := secret.Data["tailscaled.hujson"]; ok {
+		return contents, nil
+	}
+
+	return secret.Data["tailscaled"], nil
+}
diff --git a/ipn/store/kubestore/store_kube.go b/ipn/store/kubestore/store_kube.go
index 00950bd3b..7ba7e2756 100644
--- a/ipn/store/kubestore/store_kube.go
+++ b/ipn/store/kubestore/store_kube.go
@@ -8,7 +8,6 @@ import (
 	"context"
 	"fmt"
 	"net"
-	"os"
 	"strings"
 	"time"
 
@@ -31,10 +30,6 @@ func New(_ logger.Logf, secretName string) (*Store, error) {
 	if err != nil {
 		return nil, err
 	}
-	if os.Getenv("TS_KUBERNETES_READ_API_SERVER_ADDRESS_FROM_ENV") == "true" {
-		// Derive the API server address from the environment variables
-		c.SetURL(fmt.Sprintf("https://%s:%s", os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT_HTTPS")))
-	}
 	canPatch, _, err := c.CheckSecretPermissions(context.Background(), secretName)
 	if err != nil {
 		return nil, err
diff --git a/k8s-operator/utils.go b/k8s-operator/utils.go
index a1f225fe6..2f818c38a 100644
--- a/k8s-operator/utils.go
+++ b/k8s-operator/utils.go
@@ -8,6 +8,7 @@ package kube
 
 import (
 	"fmt"
+	"iter"
 
 	"tailscale.com/tailcfg"
 )
@@ -41,10 +42,33 @@ func TailscaledConfigFileName(cap tailcfg.CapabilityVersion) string {
 // CapVerFromFileName parses the capability version from a tailscaled
 // config file name previously generated by TailscaledConfigFileNameForCap.
 func CapVerFromFileName(name string) (tailcfg.CapabilityVersion, error) {
-	if name == "tailscaled" {
+	switch name {
+	case "tailscaled", "tailscaled.hujson":
+		// Unversioned names.
 		return 0, nil
+	default:
+		var cap tailcfg.CapabilityVersion
+		_, err := fmt.Sscanf(name, "cap-%d.hujson", &cap)
+		return cap, err
 	}
-	var cap tailcfg.CapabilityVersion
-	_, err := fmt.Sscanf(name, "cap-%d.hujson", &cap)
-	return cap, err
+}
+
+func SelectConfigFileName(files iter.Seq[string]) string {
+	maxCapVer := tailcfg.CapabilityVersion(-1)
+	var selectedName string
+	for fileName := range files {
+		capVer, err := CapVerFromFileName(fileName)
+		if err != nil {
+			continue
+		}
+		// 0 is "unversioned" (by capability - there is still a version inside
+		// the config itself). Always prefer it to files that have a capability
+		// version.
+		if (capVer > maxCapVer && maxCapVer != 0) || capVer == 0 {
+			maxCapVer = capVer
+			selectedName = fileName
+		}
+	}
+
+	return selectedName
 }
diff --git a/kube/kubeclient/client.go b/kube/kubeclient/client.go
index e8ddec75d..09c413deb 100644
--- a/kube/kubeclient/client.go
+++ b/kube/kubeclient/client.go
@@ -33,6 +33,8 @@ import (
 const (
 	saPath     = "/var/run/secrets/kubernetes.io/serviceaccount"
 	defaultURL = "https://kubernetes.default.svc"
+	envAPIHost = "KUBERNETES_SERVICE_HOST"
+	envAPIPort = "KUBERNETES_SERVICE_PORT_HTTPS"
 )
 
 // rootPathForTests is set by tests to override the root path to the
@@ -61,7 +63,6 @@ type Client interface {
 	JSONPatchSecret(context.Context, string, []JSONPatch) error
 	CheckSecretPermissions(context.Context, string) (bool, bool, error)
 	SetDialer(dialer func(context.Context, string, string) (net.Conn, error))
-	SetURL(string)
 }
 
 type client struct {
@@ -87,8 +88,12 @@ func New() (Client, error) {
 	if ok := cp.AppendCertsFromPEM(caCert); !ok {
 		return nil, fmt.Errorf("kube: error in creating root cert pool")
 	}
+	url := defaultURL
+	if host, port := os.Getenv(envAPIHost), os.Getenv(envAPIPort); host != "" && port != "" {
+		url = fmt.Sprintf("https://%s:%s", host, port)
+	}
 	return &client{
-		url: defaultURL,
+		url: url,
 		ns:  string(ns),
 		client: &http.Client{
 			Transport: &http.Transport{
@@ -100,12 +105,6 @@ func New() (Client, error) {
 	}, nil
 }
 
-// SetURL sets the URL to use for the Kubernetes API.
-// This is used only for testing.
-func (c *client) SetURL(url string) {
-	c.url = url
-}
-
 // SetDialer sets the dialer to use when establishing a connection
 // to the Kubernetes API server.
 func (c *client) SetDialer(dialer func(ctx context.Context, network, addr string) (net.Conn, error)) {
diff --git a/kube/kubeclient/fake_client.go b/kube/kubeclient/fake_client.go
index 3cef3d27e..04aa6cf2d 100644
--- a/kube/kubeclient/fake_client.go
+++ b/kube/kubeclient/fake_client.go
@@ -23,7 +23,6 @@ func (fc *FakeClient) CheckSecretPermissions(ctx context.Context, name string) (
 func (fc *FakeClient) GetSecret(ctx context.Context, name string) (*kubeapi.Secret, error) {
 	return fc.GetSecretImpl(ctx, name)
 }
-func (fc *FakeClient) SetURL(_ string) {}
 func (fc *FakeClient) SetDialer(dialer func(ctx context.Context, network, addr string) (net.Conn, error)) {
 }
 func (fc *FakeClient) StrategicMergePatchSecret(context.Context, string, *kubeapi.Secret, string) error {