doctor/kernellog: add new doctor check to parse kernel log for problemsandrew/doctor-conntrack

This currently only checks for lines indicating that the nf_conntrack
table is full, which can cause errors when running exit nodes and/or
subnet routers, but we can add additional checks as we need them.

Signed-off-by: Andrew Dunham <andrew@du.nham.ca>
Change-Id: I33c89f82d22595ae31f962ca2085a5fa211e951e

5 files changed, 142 insertions, 0 deletions

diff --git a/cmd/tailscaled/depaware.txt b/cmd/tailscaled/depaware.txt
index a090a8919..08661859c 100644
--- a/cmd/tailscaled/depaware.txt
+++ b/cmd/tailscaled/depaware.txt
@@ -223,6 +223,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/derp/derphttp                                  from tailscale.com/net/netcheck+
         tailscale.com/disco                                          from tailscale.com/derp+
         tailscale.com/doctor                                         from tailscale.com/ipn/ipnlocal
+        tailscale.com/doctor/kernellog                               from tailscale.com/ipn/ipnlocal
      💣 tailscale.com/doctor/permissions                             from tailscale.com/ipn/ipnlocal
         tailscale.com/doctor/routetable                              from tailscale.com/ipn/ipnlocal
         tailscale.com/envknob                                        from tailscale.com/control/controlclient+
diff --git a/doctor/kernellog/kernellog.go b/doctor/kernellog/kernellog.go
new file mode 100644
index 000000000..338dab645
--- /dev/null
+++ b/doctor/kernellog/kernellog.go
@@ -0,0 +1,13 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// Package kernellog provides a doctor.Check that checks for errors in the
+// system's kernel log.
+package kernellog
+
+// Check implements the doctor.Check interface.
+type Check struct{}
+
+func (Check) Name() string {
+	return "kernellog"
+}
diff --git a/doctor/kernellog/kernellog_default.go b/doctor/kernellog/kernellog_default.go
new file mode 100644
index 000000000..d2d19397c
--- /dev/null
+++ b/doctor/kernellog/kernellog_default.go
@@ -0,0 +1,17 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !linux
+
+package kernellog
+
+import (
+	"context"
+
+	"tailscale.com/types/logger"
+)
+
+func (Check) Run(_ context.Context, logf logger.Logf) error {
+	// Not supported; do nothing
+	return nil
+}
diff --git a/doctor/kernellog/kernellog_linux.go b/doctor/kernellog/kernellog_linux.go
new file mode 100644
index 000000000..f55a9f1fa
--- /dev/null
+++ b/doctor/kernellog/kernellog_linux.go
@@ -0,0 +1,109 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package kernellog
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"golang.org/x/sys/unix"
+	"tailscale.com/types/logger"
+)
+
+var lineRegexp = regexp.MustCompile(`\A\<(\d+)\>\[( *\d+\.\d+)\](.*)\z`)
+
+func (Check) Run(_ context.Context, logf logger.Logf) error {
+	var (
+		conntrackFull int
+	)
+	invalid, err := iterateKernelLog(func(level int, ts float64, text string) bool {
+		if strings.Contains(text, "nf_conntrack: table full, dropping packet") {
+			conntrackFull++
+		}
+		return true
+	})
+
+	if invalid > 0 {
+		logf("invalid log lines: %d", invalid)
+	}
+	if conntrackFull > 0 {
+		logf("nf_conntrack table full lines: %d", conntrackFull)
+	}
+	return err
+}
+
+func iterateKernelLog(cb func(int, float64, string) bool) (invalid int, err error) {
+	buf, err := readLogBuffer()
+	if err != nil {
+		return invalid, err
+	}
+
+	// Parse the logs
+	scanner := bufio.NewScanner(bytes.NewReader(buf))
+	for scanner.Scan() {
+		// Line format:
+		//    <3>[29037.645184] Message text goes here
+		//    xxx yyyyyyyyyyyy  zzzzzzzzzzzzzzzzzzzzzz
+		//     |       |                   |
+		//     level   |                   |
+		//       time since boot           |
+		//                          message string
+		matches := lineRegexp.FindStringSubmatch(scanner.Text())
+		if matches == nil {
+			invalid++
+			continue
+		}
+
+		level, err := strconv.Atoi(matches[1])
+		if err != nil {
+			invalid++
+			continue
+		}
+
+		// Convert the timestamp to a number
+		timestamp, err := strconv.ParseFloat(strings.TrimSpace(matches[2]), 64)
+		if err != nil {
+			invalid++
+			continue
+		}
+
+		// Don't require a space prefix, but if there is one, remove
+		// it. Multiple spaces might be intentional and thus should be
+		// preserved.
+		text := matches[3]
+		if text[0] == ' ' {
+			text = text[1:]
+		}
+
+		if !cb(level, timestamp, text) {
+			break
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		return invalid, err
+	}
+	return invalid, nil
+}
+
+func readLogBuffer() ([]byte, error) {
+	// Get the size of the kernel log buffer
+	sz, err := unix.Klogctl(unix.SYSLOG_ACTION_SIZE_BUFFER, nil)
+	if err != nil {
+		return nil, fmt.Errorf("getting kernel log buffer size: %w", err)
+	}
+
+	// Allocate a buffer and read the whole thing
+	buf := make([]byte, sz)
+	n, err := unix.Klogctl(unix.SYSLOG_ACTION_READ_ALL, buf)
+	if err != nil {
+		return nil, fmt.Errorf("reading kernel log buffer: %w", err)
+	}
+	return buf[:n], nil
+}
diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go
index b37ca3194..3d3081051 100644
--- a/ipn/ipnlocal/local.go
+++ b/ipn/ipnlocal/local.go
@@ -34,6 +34,7 @@ import (
 	"tailscale.com/client/tailscale/apitype"
 	"tailscale.com/control/controlclient"
 	"tailscale.com/doctor"
+	"tailscale.com/doctor/kernellog"
 	"tailscale.com/doctor/permissions"
 	"tailscale.com/doctor/routetable"
 	"tailscale.com/envknob"
@@ -4761,6 +4762,7 @@ func (b *LocalBackend) Doctor(ctx context.Context, logf logger.Logf) {
 	checks = append(checks,
 		permissions.Check{},
 		routetable.Check{},
+		kernellog.Check{},
 	)
 
 	// Print a log message if any of the global DNS resolvers are Tailscale