ipn/ipnlocal: run "tailscale update" via systemd-run on Linux (#10229)

When we run tailscled under systemd, restarting the unit kills all child
processes, including "tailscale update". And during update, the package
manager will restart the tailscaled unit. Specifically on Debian-based
distros, interrupting `apt-get install` can get the system into a wedged
state which requires the user to manually run `dpkg --configure` to
recover.

To avoid all this, use `systemd-run` where available to run the
`tailscale update` process. This launches it in a separate temporary
unit and doesn't kill it when parent unit is restarted.

Also, detect when `apt-get install` complains about aborted update and
try to restore the system by running `dpkg --configure tailscale`. This
could help if the system unexpectedly shuts down during our auto-update.

Fixes https://github.com/tailscale/corp/issues/15771

Signed-off-by: Andrew Lytvynov <awly@tailscale.com>

2 files changed, 32 insertions, 10 deletions

diff --git a/clientupdate/clientupdate.go b/clientupdate/clientupdate.go
index 78bc322da..7e14aa16e 100644
--- a/clientupdate/clientupdate.go
+++ b/clientupdate/clientupdate.go
@@ -415,17 +415,25 @@ func (up *Updater) updateDebLike() error {
 		// we're not updating them:
 		"-o", "APT::Get::List-Cleanup=0",
 	)
-	cmd.Stdout = up.Stdout
-	cmd.Stderr = up.Stderr
-	if err := cmd.Run(); err != nil {
-		return err
+	if out, err := cmd.CombinedOutput(); err != nil {
+		return fmt.Errorf("apt-get update failed: %w; output:\n%s", err, out)
 	}
 
-	cmd = exec.Command("apt-get", "install", "--yes", "--allow-downgrades", "tailscale="+ver)
-	cmd.Stdout = up.Stdout
-	cmd.Stderr = up.Stderr
-	if err := cmd.Run(); err != nil {
-		return err
+	for i := 0; i < 2; i++ {
+		out, err := exec.Command("apt-get", "install", "--yes", "--allow-downgrades", "tailscale="+ver).CombinedOutput()
+		if err != nil {
+			if !bytes.Contains(out, []byte(`dpkg was interrupted`)) {
+				return fmt.Errorf("apt-get install failed: %w; output:\n%s", err, out)
+			}
+			up.Logf("apt-get install failed: %s; output:\n%s", err, out)
+			up.Logf("running dpkg --configure tailscale")
+			out, err = exec.Command("dpkg", "--force-confdef,downgrade", "--configure", "tailscale").CombinedOutput()
+			if err != nil {
+				return fmt.Errorf("dpkg --configure tailscale failed: %w; output:\n%s", err, out)
+			}
+			continue
+		}
+		break
 	}
 
 	return nil
diff --git a/ipn/ipnlocal/c2n.go b/ipn/ipnlocal/c2n.go
index c48c1edce..50e5da550 100644
--- a/ipn/ipnlocal/c2n.go
+++ b/ipn/ipnlocal/c2n.go
@@ -280,7 +280,7 @@ func handleC2NUpdatePost(b *LocalBackend, w http.ResponseWriter, r *http.Request
 		return
 	}
 
-	cmd := exec.Command(cmdTS, "update", "--yes")
+	cmd := tailscaleUpdateCmd(cmdTS)
 	buf := new(bytes.Buffer)
 	cmd.Stdout = buf
 	cmd.Stderr = buf
@@ -412,6 +412,20 @@ func findCmdTailscale() (string, error) {
 	return "", errors.New("tailscale executable not found in expected place")
 }
 
+func tailscaleUpdateCmd(cmdTS string) *exec.Cmd {
+	if runtime.GOOS != "linux" {
+		return exec.Command(cmdTS, "update", "--yes")
+	}
+	if _, err := exec.LookPath("systemd-run"); err != nil {
+		return exec.Command(cmdTS, "update", "--yes")
+	}
+	// When systemd-run is available, use it to run the update command. This
+	// creates a new temporary unit separate from the tailscaled unit. When
+	// tailscaled is restarted during the update, systemd won't kill this
+	// temporary update unit, which could cause unexpected breakage.
+	return exec.Command("systemd-run", "--wait", "--pipe", "--collect", cmdTS, "update", "--yes")
+}
+
 func regularFileExists(path string) bool {
 	fi, err := os.Stat(path)
 	return err == nil && fi.Mode().IsRegular()