wipcatzkorn/otel-init

8 files changed, 256 insertions, 11 deletions

diff --git a/cmd/derper/derper.go b/cmd/derper/derper.go
index f2573f504..d8bbccba5 100644
--- a/cmd/derper/derper.go
+++ b/cmd/derper/derper.go
@@ -29,6 +29,7 @@ import (
 	"tailscale.com/atomicfile"
 	"tailscale.com/derp"
 	"tailscale.com/derp/derphttp"
+	"tailscale.com/derp/derptrace"
 	"tailscale.com/metrics"
 	"tailscale.com/net/stun"
 	"tailscale.com/tsweb"
@@ -53,6 +54,10 @@ var (
 
 	acceptConnLimit = flag.Float64("accept-connection-limit", math.Inf(+1), "rate limit for accepting new connection")
 	acceptConnBurst = flag.Int("accept-connection-burst", math.MaxInt, "burst limit for accepting new connection")
+
+	// Tracing
+	collectorAddr = flag.String("collector-addr", "", "optional tracing collector address, if empty tracing is no op.")
+	sampleRate    = flag.Float64("sample-rate", 0.01, "adjust sample rate of tracing. default at 1%")
 )
 
 var (
@@ -72,6 +77,7 @@ var (
 )
 
 func init() {
+
 	stats.Set("counter_requests", stunDisposition)
 	stats.Set("counter_addrfamily", stunAddrFamily)
 	expvar.Publish("stun", stats)
@@ -145,6 +151,19 @@ func main() {
 
 	cfg := loadConfig()
 
+	if *collectorAddr != "" || *dev {
+		ctx := context.Background()
+		shutdown, err := derptrace.InitTracing(ctx, *collectorAddr, *hostname, *sampleRate)
+		if err != nil {
+			log.Fatalf("unable to initialize tracing: %v", err)
+		}
+		defer func() {
+			if err := shutdown(ctx); err != nil {
+				log.Fatalf("failed to shut down TracerProvider: %v", err)
+			}
+		}()
+	}
+
 	serveTLS := tsweb.IsProd443(*addr) || *certMode == "manual"
 
 	s := derp.NewServer(cfg.PrivateKey, log.Printf)
diff --git a/cmd/derper/derper_test.go b/cmd/derper/derper_test.go
index 4adab4e8b..9fb8f1058 100644
--- a/cmd/derper/derper_test.go
+++ b/cmd/derper/derper_test.go
@@ -65,5 +65,4 @@ func BenchmarkServerSTUN(b *testing.B) {
 			b.Fatal(err)
 		}
 	}
-
 }
diff --git a/cmd/testderper/test.go b/cmd/testderper/test.go
new file mode 100644
index 000000000..eb4756ea2
--- /dev/null
+++ b/cmd/testderper/test.go
@@ -0,0 +1,79 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"time"
+
+	"tailscale.com/derp/derphttp"
+	"tailscale.com/types/key"
+)
+
+// THIS FILE WILL NOT BE INCLUDED, ITS JUST FOR LOCAL TESTING.
+// PLEASE IGNORE :)
+
+func main() {
+
+	dothestuff()
+
+}
+
+func dothestuff() {
+
+	clientPriv := key.NewNode()
+
+	c, err := derphttp.NewClient(clientPriv, "http://localhost:3340/derp", log.Printf)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer c.Close()
+
+	err = c.Connect(context.Background())
+	if err != nil {
+		fmt.Println(err)
+		panic("connect")
+	}
+
+	n := key.NewNode()
+
+	nc, err := derphttp.NewClient(clientPriv, "http://localhost:3340/derp", log.Printf)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer nc.Close()
+
+	err = nc.Connect(context.Background())
+	if err != nil {
+		fmt.Println(err)
+		panic("connect")
+	}
+
+	fmt.Println(c.ServerPublicKey())
+
+	for i := 0; i < 2; i++ {
+
+		fmt.Println("hi?")
+
+		if err := c.SendPing([8]byte{1, 2, 3, 4, 5, 6, 7, 8}); err != nil {
+			fmt.Println(err)
+		}
+		time.Sleep(time.Millisecond * 100)
+
+		if err := c.Send(n.Public(), []byte{1, 2, 3, 4, 5, 6, 7, 8}); err != nil {
+			fmt.Println(err)
+		}
+
+		time.Sleep(time.Millisecond * 100)
+
+		if err := nc.Send(clientPriv.Public(), []byte{1, 2, 3, 4, 5, 6, 7, 8}); err != nil {
+			fmt.Println(err)
+		}
+
+		c.NotePreferred(true)
+
+		time.Sleep(time.Millisecond * 100)
+
+	}
+
+}
diff --git a/derp/derp_server.go b/derp/derp_server.go
index 52680cace..c7dc38ff8 100644
--- a/derp/derp_server.go
+++ b/derp/derp_server.go
@@ -34,6 +34,9 @@ import (
 	"sync/atomic"
 	"time"
 
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
 	"go4.org/mem"
 	"golang.org/x/sync/errgroup"
 	"golang.org/x/time/rate"
@@ -710,6 +713,11 @@ var (
 // run serves the client until there's an error.
 // If the client hangs up or the server is closed, run returns nil, otherwise run returns an error.
 func (c *sclient) run(ctx context.Context) error {
+
+	tracer := otel.Tracer("")
+	ctx, span := tracer.Start(ctx, "run", trace.WithAttributes(attribute.KeyValue{Key: "NodeKey", Value: attribute.StringValue(c.key.String())}))
+	defer span.End()
+
 	// Launch sender, but don't return from run until sender goroutine is done.
 	var grp errgroup.Group
 	sendCtx, cancelSender := context.WithCancel(ctx)
@@ -732,26 +740,42 @@ func (c *sclient) run(ctx context.Context) error {
 				c.logf("closing; server closed")
 				return nil
 			}
+			span.RecordError(err)
 			return fmt.Errorf("client %x: readFrameHeader: %w", c.key, err)
 		}
 		c.s.noteClientActivity(c)
 		switch ft {
 		case frameNotePreferred:
+			_, span := tracer.Start(ctx, "note preferred")
 			err = c.handleFrameNotePreferred(ft, fl)
+			span.End()
 		case frameSendPacket:
+			_, span := tracer.Start(ctx, "send packet")
 			err = c.handleFrameSendPacket(ft, fl)
+			span.End()
 		case frameForwardPacket:
+			_, span := tracer.Start(ctx, "forward packet")
 			err = c.handleFrameForwardPacket(ft, fl)
+			span.End()
 		case frameWatchConns:
+			_, span := tracer.Start(ctx, "watch conns")
 			err = c.handleFrameWatchConns(ft, fl)
+			span.End()
 		case frameClosePeer:
+			_, span := tracer.Start(ctx, "close peer")
 			err = c.handleFrameClosePeer(ft, fl)
+			span.End()
 		case framePing:
+			_, span := tracer.Start(ctx, "ping")
 			err = c.handleFramePing(ft, fl)
+			span.End()
 		default:
+			_, span := tracer.Start(ctx, "unknown")
 			err = c.handleUnknownFrame(ft, fl)
+			span.End()
 		}
 		if err != nil {
+			span.RecordError(err)
 			return err
 		}
 	}
diff --git a/derp/derphttp/derphttp_server.go b/derp/derphttp/derphttp_server.go
index 17edb8c21..e58d466aa 100644
--- a/derp/derphttp/derphttp_server.go
+++ b/derp/derphttp/derphttp_server.go
@@ -10,6 +10,9 @@ import (
 	"net/http"
 	"strings"
 
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
 	"tailscale.com/derp"
 )
 
@@ -56,6 +59,10 @@ func Handler(s *derp.Server) http.Handler {
 				pubKey.UntypedHexString())
 		}
 
-		s.Accept(r.Context(), netConn, conn, netConn.RemoteAddr().String())
+		tracer := otel.Tracer("")
+		ctx, span := tracer.Start(r.Context(), "/derp", trace.WithAttributes(attribute.KeyValue{Key: "ID", Value: attribute.StringValue("a")}))
+		defer span.End()
+
+		s.Accept(ctx, netConn, conn, netConn.RemoteAddr().String())
 	})
 }
diff --git a/derp/derptrace/derptrace.go b/derp/derptrace/derptrace.go
new file mode 100644
index 000000000..638da8b1c
--- /dev/null
+++ b/derp/derptrace/derptrace.go
@@ -0,0 +1,66 @@
+package derptrace
+
+import (
+	"context"
+	"fmt"
+
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
+	"go.opentelemetry.io/otel/sdk/resource"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	semconv "go.opentelemetry.io/otel/semconv/v1.8.0"
+)
+
+func InitTracing(ctx context.Context, collectorAddr string, hostname string, sampleRate float64) (func(context.Context) error, error) {
+	exporter, err := exporter(ctx, collectorAddr)
+	if err != nil {
+		return nil, fmt.Errorf("unable to create exporter: %w", err)
+	}
+
+	res, err := resource.New(ctx, resource.WithAttributes(
+		semconv.ServiceNamespaceKey.String("derpers"),
+		semconv.ServiceNameKey.String(hostname),
+	))
+	if err != nil {
+		return nil, fmt.Errorf("unable to create resource: %w", err)
+	}
+
+	tracerProvider := sdktrace.NewTracerProvider(
+		// TODO: configure in some manner
+		sdktrace.WithSampler(sdktrace.TraceIDRatioBased(sampleRate)),
+		// Traces are sent in batches. The defaults are:
+		// The maximum queue size is 2048. If this is reached, spans are
+		// dropped.
+		// Spans are sent, regardless of the queue size being reached, after 5000ms.
+		// Exports time out after 3000ms
+		// The maximum batch size is 512. Multiple batches of spans are sent sequentially.
+		// BlockOnQueueFull - we don't use this.
+		sdktrace.WithBatcher(exporter),
+		sdktrace.WithResource(res),
+	)
+	otel.SetTracerProvider(tracerProvider)
+
+	return tracerProvider.Shutdown, nil
+}
+
+// exporter is a http opentelementry exporter
+func exporter(ctx context.Context, collectorAddr string) (*otlptrace.Exporter, error) {
+	var opts []otlptracehttp.Option
+	if collectorAddr == "" {
+		opts = append(opts, otlptracehttp.WithInsecure())
+	} else {
+		opts = append(opts, otlptracehttp.WithTLSClientConfig(nil), otlptracehttp.WithEndpoint(collectorAddr))
+	}
+
+	exporter, err := otlptrace.New(ctx,
+		otlptracehttp.NewClient(
+			opts...,
+		),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("unable to create http trace exporter: %w", err)
+	}
+
+	return exporter, nil
+}
diff --git a/go.mod b/go.mod
index 5257e85aa..aad5cffce 100644
--- a/go.mod
+++ b/go.mod
@@ -2,6 +2,22 @@ module tailscale.com
 
 go 1.19
 
+replace go.opentelemetry.io/otel/exporters/otlp/otlptrace => github.com/tailscale/opentelemetry-go/exporters/otlp/otlptrace v1.9.0-ts
+
+replace go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp => github.com/tailscale/opentelemetry-go/exporters/otlp/otlptrace/otlptracehttp v1.9.0-ts
+
+replace go.opentelemetry.io/otel/exporters/otlp/otlpconfig => github.com/tailscale/opentelemetry-go/exporters/otlp/otlpconfig v1.9.0-ts
+
+replace go.opentelemetry.io/otel => github.com/tailscale/opentelemetry-go v1.9.0-ts
+
+replace go.opentelemetry.io/otel/sdk => github.com/tailscale/opentelemetry-go/sdk v1.9.0-ts
+
+replace go.opentelemetry.io/otel/trace => github.com/tailscale/opentelemetry-go/trace v1.9.0-ts
+
+replace go.opentelemetry.io/otel/exporters/otlp/internal/retry => github.com/tailscale/opentelemetry-go/exporters/otlp/internal/retry v1.9.0-ts
+
+replace go.opentelemetry.io/proto/otlp => github.com/tailscale/opentelemetry-proto-go/otlp v0.18.0-ts
+
 require (
 	filippo.io/mkcert v1.4.3
 	github.com/akutz/memconn v0.1.0
@@ -53,6 +69,11 @@ require (
 	github.com/toqueteos/webbrowser v1.2.0
 	github.com/u-root/u-root v0.8.0
 	github.com/vishvananda/netlink v1.1.1-0.20211118161826-650dca95af54
+	go.opentelemetry.io/otel v1.9.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.9.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.8.0
+	go.opentelemetry.io/otel/sdk v1.9.0
+	go.opentelemetry.io/otel/trace v1.9.0
 	go4.org/mem v0.0.0-20210711025021-927187094b94
 	go4.org/netipx v0.0.0-20220725152314-7e7bdc8411bf
 	golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f
@@ -110,6 +131,7 @@ require (
 	github.com/breml/bidichk v0.2.1 // indirect
 	github.com/butuzov/ireturn v0.1.1 // indirect
 	github.com/cavaliercoder/go-cpio v0.0.0-20180626203310-925f9528c45e // indirect
+	github.com/cenkalti/backoff/v4 v4.1.3 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/charithe/durationcheck v0.0.9 // indirect
 	github.com/chavacava/garif v0.0.0-20210405164556-e8a0a408d6af // indirect
@@ -133,6 +155,8 @@ require (
 	github.com/go-git/gcfg v1.5.0 // indirect
 	github.com/go-git/go-billy/v5 v5.3.1 // indirect
 	github.com/go-git/go-git/v5 v5.4.2 // indirect
+	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-toolsmith/astcast v1.0.0 // indirect
 	github.com/go-toolsmith/astcopy v1.0.0 // indirect
 	github.com/go-toolsmith/astequal v1.0.1 // indirect
@@ -226,7 +250,7 @@ require (
 	github.com/quasilyte/go-ruleguard v0.3.13 // indirect
 	github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
-	github.com/rogpeppe/go-internal v1.8.1-0.20211023094830-115ce09fd6b4 // indirect
+	github.com/rogpeppe/go-internal v1.8.1 // indirect
 	github.com/ryancurrah/gomodguard v1.2.3 // indirect
 	github.com/ryanrolds/sqlclosecheck v0.3.0 // indirect
 	github.com/sanposhiho/wastedassign/v2 v2.0.7 // indirect
@@ -245,8 +269,8 @@ require (
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/spf13/viper v1.9.0 // indirect
 	github.com/ssgreg/nlreturn/v2 v2.2.1 // indirect
-	github.com/stretchr/objx v0.3.0 // indirect
-	github.com/stretchr/testify v1.7.0 // indirect
+	github.com/stretchr/objx v0.4.0 // indirect
+	github.com/stretchr/testify v1.7.5 // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
 	github.com/sylvia7788/contextcheck v1.0.4 // indirect
 	github.com/tdakkota/asciicheck v0.1.1 // indirect
@@ -264,6 +288,8 @@ require (
 	github.com/x448/float16 v0.8.4 // indirect
 	github.com/xanzy/ssh-agent v0.3.1 // indirect
 	github.com/yeya24/promlinter v0.1.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.9.0 // indirect
+	go.opentelemetry.io/proto/otlp v0.18.0 // indirect
 	golang.org/x/exp/typeparams v0.0.0-20220328175248-053ad81199eb // indirect
 	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
 	golang.org/x/text v0.3.7 // indirect
@@ -272,7 +298,7 @@ require (
 	gopkg.in/ini.v1 v1.66.2 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 	howett.net/plist v1.0.0 // indirect
 	mvdan.cc/gofumpt v0.2.0 // indirect
 	mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed // indirect
diff --git a/go.sum b/go.sum
index 5938ecbc0..051f61345 100644
--- a/go.sum
+++ b/go.sum
@@ -191,6 +191,8 @@ github.com/butuzov/ireturn v0.1.1 h1:QvrO2QF2+/Cx1WA/vETCIYBKtRjc30vesdoPUNo1EbY
 github.com/butuzov/ireturn v0.1.1/go.mod h1:Wh6Zl3IMtTpaIKbmwzqi6olnM9ptYQxxVacMsOEFPoc=
 github.com/cavaliercoder/go-cpio v0.0.0-20180626203310-925f9528c45e h1:hHg27A0RSSp2Om9lubZpiMgVbvn39bsUmW9U5h0twqc=
 github.com/cavaliercoder/go-cpio v0.0.0-20180626203310-925f9528c45e/go.mod h1:oDpT4efm8tSYHXV5tHSdRvBet/b/QzxZ+XyyPehvm3A=
+github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4=
+github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
@@ -334,6 +336,11 @@ github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vb
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
+github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
 github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
@@ -975,8 +982,9 @@ github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE
 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/rogpeppe/go-internal v1.6.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/rogpeppe/go-internal v1.8.1-0.20210923151022-86f73c517451/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o=
-github.com/rogpeppe/go-internal v1.8.1-0.20211023094830-115ce09fd6b4 h1:Ha8xCaq6ln1a+R91Km45Oq6lPXj2Mla6CRJYcuV2h1w=
 github.com/rogpeppe/go-internal v1.8.1-0.20211023094830-115ce09fd6b4/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o=
+github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XFkP+Eg=
+github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o=
 github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNlmVHqNRXXJNAY=
@@ -1063,8 +1071,9 @@ github.com/ssgreg/nlreturn/v2 v2.2.1 h1:X4XDI7jstt3ySqGU86YGAURbxw3oTDPK9sPEi6YE
 github.com/ssgreg/nlreturn/v2 v2.2.1/go.mod h1:E/iiPB78hV7Szg2YfRgyIrk1AD6JVMTRkkxBiELzh2I=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As=
 github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
+github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/testify v0.0.0-20170130113145-4d4bfba8f1d1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.1.4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
@@ -1072,8 +1081,10 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.5 h1:s5PTfem8p8EbKQOctVV53k6jCJt3UX4IEJzwh+C324Q=
+github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/sylvia7788/contextcheck v1.0.4 h1:MsiVqROAdr0efZc/fOCt0c235qm9XJqHtWwM+2h2B04=
@@ -1094,6 +1105,20 @@ github.com/tailscale/mkctr v0.0.0-20220601142259-c0b937af2e89 h1:7xU7AFQE83h0wz/
 github.com/tailscale/mkctr v0.0.0-20220601142259-c0b937af2e89/go.mod h1:OGMqrTzDqmJkGumUTtOv44Rp3/4xS+QFbE8Rn0AGlaU=
 github.com/tailscale/netlink v1.1.1-0.20211101221916-cabfb018fe85 h1:zrsUcqrG2uQSPhaUPjUQwozcRdDdSxxqhNgNZ3drZFk=
 github.com/tailscale/netlink v1.1.1-0.20211101221916-cabfb018fe85/go.mod h1:NzVQi3Mleb+qzq8VmcWpSkcSYxXIg0DkI6XDzpVkhJ0=
+github.com/tailscale/opentelemetry-go v1.9.0-ts h1:YFZeYdu5GD+0Ni+dX2w3+6ea8foeWbqzJ9dNbybuzLM=
+github.com/tailscale/opentelemetry-go v1.9.0-ts/go.mod h1:2Q13NgwCFppOZ1IqWOwFAO/Lrp1KHG25C005DzMW610=
+github.com/tailscale/opentelemetry-go/exporters/otlp/internal/retry v1.9.0-ts h1:v5Ybs0+X2caeCe9TQ6hHM3VyWUMWsifs2SX3Yn0W8HQ=
+github.com/tailscale/opentelemetry-go/exporters/otlp/internal/retry v1.9.0-ts/go.mod h1:qquF6e5B9PAb6PBelpYxtJPrd9TVJp9PwWmsxIMNsng=
+github.com/tailscale/opentelemetry-go/exporters/otlp/otlptrace v1.9.0-ts h1:2qSC5cIthbLMOflFvz7oU9ZuRKjbMqGwkr9SxBSftFk=
+github.com/tailscale/opentelemetry-go/exporters/otlp/otlptrace v1.9.0-ts/go.mod h1:nBGuynyUZ9uW8tcA+6yRiehbc2fJk77JlPDMM8YCNLo=
+github.com/tailscale/opentelemetry-go/exporters/otlp/otlptrace/otlptracehttp v1.9.0-ts h1:wz+c10VwLkwx6Gpo3bo7THPNscX6bFxLsMrV4+YwCVo=
+github.com/tailscale/opentelemetry-go/exporters/otlp/otlptrace/otlptracehttp v1.9.0-ts/go.mod h1:/i7zhGQIPpa3l5FUWNerzFsn1HPUDjtbOZstf4CC4kY=
+github.com/tailscale/opentelemetry-go/sdk v1.9.0-ts h1:qgitOO9JGeg6p38gAS1J/GmVoQbZEpFptMS7kQXGJfo=
+github.com/tailscale/opentelemetry-go/sdk v1.9.0-ts/go.mod h1:UVxSKedj3m390c5RE7PXq8KdQ+jGFdAWJsJ/cTrnSNg=
+github.com/tailscale/opentelemetry-go/trace v1.9.0-ts h1:rON5FNhhtKiZqKVceTQvw8ZKfZjYw+b9//17t+SoUn0=
+github.com/tailscale/opentelemetry-go/trace v1.9.0-ts/go.mod h1:Agz4mRAzwCK52m82o2NWXQOb5qZ/LQWjvSPEnmXec6s=
+github.com/tailscale/opentelemetry-proto-go/otlp v0.18.0-ts h1:OQ+YuUnJLS4TqBQNGFb4oDQjoOG7D5/P0X6oPsDtS9U=
+github.com/tailscale/opentelemetry-proto-go/otlp v0.18.0-ts/go.mod h1:dB28XgQo6bZwVyaPtnA6c7jEx7uNDgjrIRP638BNs3Q=
 github.com/tcnksm/go-httpstat v0.2.0 h1:rP7T5e5U2HfmOBmZzGgGZjBQ5/GluWUylujl0tJ04I0=
 github.com/tcnksm/go-httpstat v0.2.0/go.mod h1:s3JVJFtQxtBEBC9dwcdTTXS9xFnM3SXAZwPG41aurT8=
 github.com/tdakkota/asciicheck v0.0.0-20200416190851-d7f85be797a2/go.mod h1:yHp0ai0Z9gUljN3o0xMhYJnH/IcvkdTBOX2fmJ93JEM=
@@ -1206,7 +1231,6 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
 go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
-go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
@@ -1818,8 +1842,9 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0=
 gvisor.dev/gvisor v0.0.0-20220801230058-850e42eb4444 h1:0d3ygmOM5RgQB8rmsZNeAY/7Q98fKt1HrGO2XIp4pDI=
 gvisor.dev/gvisor v0.0.0-20220801230058-850e42eb4444/go.mod h1:TIvkJD0sxe8pIob3p6T8IzxXunlp6yfgktvTNp+DGNM=