ipn/ipnlocal: add c2n /debug/{goroutines,prefs,metrics}

* and move goroutine scrubbing code to its own package for reuse
* bump capver to 45

Change-Id: I9b4dfa5af44d2ecada6cc044cd1b5674ee427575
Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>

6 files changed, 116 insertions, 89 deletions

diff --git a/cmd/tailscaled/depaware.txt b/cmd/tailscaled/depaware.txt
index 614f2f0db..c85a08962 100644
--- a/cmd/tailscaled/depaware.txt
+++ b/cmd/tailscaled/depaware.txt
@@ -275,6 +275,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
      💣 tailscale.com/util/deephash                                  from tailscale.com/ipn/ipnlocal+
         tailscale.com/util/dnsname                                   from tailscale.com/hostinfo+
   LW    tailscale.com/util/endian                                    from tailscale.com/net/dns+
+        tailscale.com/util/goroutines                                from tailscale.com/control/controlclient+
         tailscale.com/util/groupmember                               from tailscale.com/ipn/ipnserver
      💣 tailscale.com/util/hashx                                     from tailscale.com/util/deephash
         tailscale.com/util/lineread                                  from tailscale.com/hostinfo+
diff --git a/control/controlclient/debug.go b/control/controlclient/debug.go
index b14643f7a..bc58b4fd5 100644
--- a/control/controlclient/debug.go
+++ b/control/controlclient/debug.go
@@ -8,12 +8,11 @@ import (
 	"bytes"
 	"compress/gzip"
 	"context"
-	"fmt"
 	"log"
 	"net/http"
-	"runtime"
-	"strconv"
 	"time"
+
+	"tailscale.com/util/goroutines"
 )
 
 func dumpGoroutinesToURL(c *http.Client, targetURL string) {
@@ -22,7 +21,7 @@ func dumpGoroutinesToURL(c *http.Client, targetURL string) {
 
 	zbuf := new(bytes.Buffer)
 	zw := gzip.NewWriter(zbuf)
-	zw.Write(scrubbedGoroutineDump())
+	zw.Write(goroutines.ScrubbedGoroutineDump())
 	zw.Close()
 
 	req, err := http.NewRequestWithContext(ctx, "PUT", targetURL, zbuf)
@@ -40,83 +39,3 @@ func dumpGoroutinesToURL(c *http.Client, targetURL string) {
 		log.Printf("dumpGoroutinesToURL complete to %v (after %v)", targetURL, d)
 	}
 }
-
-// scrubbedGoroutineDump returns the list of all current goroutines, but with the actual
-// values of arguments scrubbed out, lest it contain some private key material.
-func scrubbedGoroutineDump() []byte {
-	var buf []byte
-	// Grab stacks multiple times into increasingly larger buffer sizes
-	// to minimize the risk that we blow past our iOS memory limit.
-	for size := 1 << 10; size <= 1<<20; size += 1 << 10 {
-		buf = make([]byte, size)
-		buf = buf[:runtime.Stack(buf, true)]
-		if len(buf) < size {
-			// It fit.
-			break
-		}
-	}
-	return scrubHex(buf)
-}
-
-func scrubHex(buf []byte) []byte {
-	saw := map[string][]byte{} // "0x123" => "v1%3" (unique value 1 and its value mod 8)
-
-	foreachHexAddress(buf, func(in []byte) {
-		if string(in) == "0x0" {
-			return
-		}
-		if v, ok := saw[string(in)]; ok {
-			for i := range in {
-				in[i] = '_'
-			}
-			copy(in, v)
-			return
-		}
-		inStr := string(in)
-		u64, err := strconv.ParseUint(string(in[2:]), 16, 64)
-		for i := range in {
-			in[i] = '_'
-		}
-		if err != nil {
-			in[0] = '?'
-			return
-		}
-		v := []byte(fmt.Sprintf("v%d%%%d", len(saw)+1, u64%8))
-		saw[inStr] = v
-		copy(in, v)
-	})
-	return buf
-}
-
-var ohx = []byte("0x")
-
-// foreachHexAddress calls f with each subslice of b that matches
-// regexp `0x[0-9a-f]*`.
-func foreachHexAddress(b []byte, f func([]byte)) {
-	for len(b) > 0 {
-		i := bytes.Index(b, ohx)
-		if i == -1 {
-			return
-		}
-		b = b[i:]
-		hx := hexPrefix(b)
-		f(hx)
-		b = b[len(hx):]
-	}
-}
-
-func hexPrefix(b []byte) []byte {
-	for i, c := range b {
-		if i < 2 {
-			continue
-		}
-		if !isHexByte(c) {
-			return b[:i]
-		}
-	}
-	return b
-}
-
-func isHexByte(b byte) bool {
-	return '0' <= b && b <= '9' || 'a' <= b && b <= 'f' || 'A' <= b && b <= 'F'
-}
diff --git a/ipn/ipnlocal/c2n.go b/ipn/ipnlocal/c2n.go
index 83e91883a..596f1ad04 100644
--- a/ipn/ipnlocal/c2n.go
+++ b/ipn/ipnlocal/c2n.go
@@ -10,14 +10,28 @@ import (
 	"net/http"
 
 	"tailscale.com/tailcfg"
+	"tailscale.com/util/clientmetric"
+	"tailscale.com/util/goroutines"
 )
 
 func (b *LocalBackend) handleC2N(w http.ResponseWriter, r *http.Request) {
+	writeJSON := func(v any) {
+		w.Header().Set("Content-Type", "application/json")
+		json.NewEncoder(w).Encode(v)
+	}
 	switch r.URL.Path {
 	case "/echo":
 		// Test handler.
 		body, _ := io.ReadAll(r.Body)
 		w.Write(body)
+	case "/debug/goroutines":
+		w.Header().Set("Content-Type", "text/plain")
+		w.Write(goroutines.ScrubbedGoroutineDump())
+	case "/debug/prefs":
+		writeJSON(b.Prefs())
+	case "/debug/metrics":
+		w.Header().Set("Content-Type", "text/plain")
+		clientmetric.WritePrometheusExpositionFormat(w)
 	case "/ssh/usernames":
 		var req tailcfg.C2NSSHUsernamesRequest
 		if r.Method == "POST" {
@@ -31,8 +45,7 @@ func (b *LocalBackend) handleC2N(w http.ResponseWriter, r *http.Request) {
 			http.Error(w, err.Error(), 500)
 			return
 		}
-		w.Header().Set("Content-Type", "application/json")
-		json.NewEncoder(w).Encode(res)
+		writeJSON(res)
 	default:
 		http.Error(w, "unknown c2n path", http.StatusBadRequest)
 	}
diff --git a/tailcfg/tailcfg.go b/tailcfg/tailcfg.go
index dc64b076a..ff1a9cfa2 100644
--- a/tailcfg/tailcfg.go
+++ b/tailcfg/tailcfg.go
@@ -81,7 +81,8 @@ type CapabilityVersion int
 //   - 42: 2022-09-06: NextDNS DoH support; see https://github.com/tailscale/tailscale/pull/5556
 //   - 43: 2022-09-21: clients can return usernames for SSH
 //   - 44: 2022-09-22: MapResponse.ControlDialPlan
-const CurrentCapabilityVersion CapabilityVersion = 44
+//   - 45: 2022-09-26: c2n /debug/{goroutines,prefs,metrics}
+const CurrentCapabilityVersion CapabilityVersion = 45
 
 type StableID string
 
diff --git a/util/goroutines/goroutines.go b/util/goroutines/goroutines.go
new file mode 100644
index 000000000..b3fc1389e
--- /dev/null
+++ b/util/goroutines/goroutines.go
@@ -0,0 +1,93 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// The goroutines package contains utilities for getting active goroutines.
+package goroutines
+
+import (
+	"bytes"
+	"fmt"
+	"runtime"
+	"strconv"
+)
+
+// ScrubbedGoroutineDump returns the list of all current goroutines, but with the actual
+// values of arguments scrubbed out, lest it contain some private key material.
+func ScrubbedGoroutineDump() []byte {
+	var buf []byte
+	// Grab stacks multiple times into increasingly larger buffer sizes
+	// to minimize the risk that we blow past our iOS memory limit.
+	for size := 1 << 10; size <= 1<<20; size += 1 << 10 {
+		buf = make([]byte, size)
+		buf = buf[:runtime.Stack(buf, true)]
+		if len(buf) < size {
+			// It fit.
+			break
+		}
+	}
+	return scrubHex(buf)
+}
+
+func scrubHex(buf []byte) []byte {
+	saw := map[string][]byte{} // "0x123" => "v1%3" (unique value 1 and its value mod 8)
+
+	foreachHexAddress(buf, func(in []byte) {
+		if string(in) == "0x0" {
+			return
+		}
+		if v, ok := saw[string(in)]; ok {
+			for i := range in {
+				in[i] = '_'
+			}
+			copy(in, v)
+			return
+		}
+		inStr := string(in)
+		u64, err := strconv.ParseUint(string(in[2:]), 16, 64)
+		for i := range in {
+			in[i] = '_'
+		}
+		if err != nil {
+			in[0] = '?'
+			return
+		}
+		v := []byte(fmt.Sprintf("v%d%%%d", len(saw)+1, u64%8))
+		saw[inStr] = v
+		copy(in, v)
+	})
+	return buf
+}
+
+var ohx = []byte("0x")
+
+// foreachHexAddress calls f with each subslice of b that matches
+// regexp `0x[0-9a-f]*`.
+func foreachHexAddress(b []byte, f func([]byte)) {
+	for len(b) > 0 {
+		i := bytes.Index(b, ohx)
+		if i == -1 {
+			return
+		}
+		b = b[i:]
+		hx := hexPrefix(b)
+		f(hx)
+		b = b[len(hx):]
+	}
+}
+
+func hexPrefix(b []byte) []byte {
+	for i, c := range b {
+		if i < 2 {
+			continue
+		}
+		if !isHexByte(c) {
+			return b[:i]
+		}
+	}
+	return b
+}
+
+func isHexByte(b byte) bool {
+	return '0' <= b && b <= '9' || 'a' <= b && b <= 'f' || 'A' <= b && b <= 'F'
+}
diff --git a/control/controlclient/debug_test.go b/util/goroutines/goroutines_test.go
index bbcca51ed..4a5559e9c 100644
--- a/control/controlclient/debug_test.go
+++ b/util/goroutines/goroutines_test.go
@@ -2,12 +2,12 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package controlclient
+package goroutines
 
 import "testing"
 
 func TestScrubbedGoroutineDump(t *testing.T) {
-	t.Logf("Got:\n%s\n", scrubbedGoroutineDump())
+	t.Logf("Got:\n%s\n", ScrubbedGoroutineDump())
 }
 
 func TestScrubHex(t *testing.T) {