control/controlclient: try reconnecting to last successful addrandrew/controlclient-use-last-addr

If we lose our connection to the control server (e.g. due to a restart,
or a network blip, etc), try reconnecting to the same address first
before going through the whole control dialplan and/or DNS flow.

This ensures that we're a bit "sticky", which makes load balancing
easier by improving the chances that this client will hit a server with
a warm cache. It also reduces the thundering herd of requests that hit
other servers after we restart a single one.

Updates #TODO

Signed-off-by: Andrew Dunham <andrew@du.nham.ca>
Change-Id: I6c3ef0b088468a8888c05cf8e3813056118ec835

2 files changed, 17 insertions, 0 deletions

diff --git a/control/controlhttp/client.go b/control/controlhttp/client.go
index fb220fd0b..b7675f254 100644
--- a/control/controlhttp/client.go
+++ b/control/controlhttp/client.go
@@ -95,6 +95,17 @@ func (a *Dialer) httpsFallbackDelay() time.Duration {
 var _ = envknob.RegisterBool("TS_USE_CONTROL_DIAL_PLAN") // to record at init time whether it's in use
 
 func (a *Dialer) dial(ctx context.Context) (*ClientConn, error) {
+	// If we have a last used address, try that first, but time out fairly
+	// aggressively in case it's actually down.
+	if a.LastServerAddr.IsValid() {
+		lastDialCtx, lastDialCancel := context.WithTimeout(ctx, 5*time.Second)
+		defer lastDialCancel()
+		conn, err := a.dialHost(lastDialCtx, a.LastServerAddr)
+		if err == nil {
+			return conn, nil
+		}
+	}
+
 	// If we don't have a dial plan, just fall back to dialing the single
 	// host we know about.
 	useDialPlan := envknob.BoolDefaultTrue("TS_USE_CONTROL_DIAL_PLAN")
diff --git a/control/controlhttp/constants.go b/control/controlhttp/constants.go
index 72161336e..130ad6be2 100644
--- a/control/controlhttp/constants.go
+++ b/control/controlhttp/constants.go
@@ -5,6 +5,7 @@ package controlhttp
 
 import (
 	"net/http"
+	"net/netip"
 	"net/url"
 	"time"
 
@@ -84,6 +85,11 @@ type Dialer struct {
 	// plan before falling back to DNS.
 	DialPlan *tailcfg.ControlDialPlan
 
+	// LastServerAddr, if valid, is the address that was last used to
+	// (successfully) connect to the control server. It will be prioritized
+	// when making a connection to the server.
+	LastServerAddr netip.Addr
+
 	proxyFunc func(*http.Request) (*url.URL, error) // or nil
 
 	// For tests only