misc/genreadme,tempfork/pkgdoc,tsnet: generate README.md files from godocfserb/main

Adds a CI check to keep opted-in directories' README.md files in sync
with their package godoc. For now tsnet (and its sub-packages under
tsnet/example) is the only opted-in tree. The list of directories
lives in misc/genreadme/genreadme.go as defaultRoots, so CI and humans
both just run `./tool/go run ./misc/genreadme` with no arguments.

The check piggybacks on the existing go_generate job in test.yml and
fails if any README.md is out of date, pointing the user at the same
command.

Along the way:

 - tempfork/pkgdoc now emits Markdown instead of plain text: headings
   become level-2 with no {#hdr-...} anchors, and [Symbol] doc links
   resolve to pkg.go.dev URLs, including for symbols in the current
   package (which the default Printer would otherwise emit as bare
   #Name fragments with no backing anchor in a README). Parsing no
   longer uses parser.ImportsOnly, so doc.Package knows the package's
   symbols and can resolve [Symbol] links at all.

 - genreadme also emits a pkg.go.dev Go Reference badge at the top of
   a library package's README; suppressed for package main.

 - tsnet/tsnet.go's package godoc is expanded in idiomatic godoc
   syntax — [Type], [Type.Method], reference-style [link]: URL
   definitions — rather than Markdown-flavored [text](url) or
   backtick-quoted identifiers, so that both pkg.go.dev and the
   generated README.md render cleanly from a single source.

Fixes #19431
Fixes #19483
Fixes #19470

Change-Id: I8ca37e9e7b3bd446b8bfa7a91ac548f142688cb1
Co-authored-by: Brad Fitzpatrick <bradfitz@tailscale.com>
Signed-off-by: Walter Poupore <walterp@tailscale.com>
Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>

1 files changed, 70 insertions, 26 deletions

diff --git a/misc/genreadme/genreadme.go b/misc/genreadme/genreadme.go
index 779f4c8c4..97a8d9e16 100644
--- a/misc/genreadme/genreadme.go
+++ b/misc/genreadme/genreadme.go
@@ -20,6 +20,7 @@ import (
 	"io/fs"
 	"log"
 	"os"
+	"path"
 	"path/filepath"
 	"runtime"
 	"strings"
@@ -28,6 +29,9 @@ import (
 	"tailscale.com/tempfork/pkgdoc"
 )
 
+// modulePath is the current module's import path, read from go.mod at startup.
+var modulePath string
+
 var skip = map[string]bool{
 	"out": true,
 }
@@ -36,15 +40,25 @@ var skip = map[string]bool{
 // Buildkite because a deploy workflow is not set up for them.
 var bkSkip = map[string]bool{}
 
+// defaultRoots are the directory trees walked when genreadme is run with
+// no arguments. Add a directory here to opt its package (and any
+// sub-packages) into README.md generation from godoc.
+var defaultRoots = []string{
+	"tsnet",
+}
+
 func main() {
 	flag.Parse()
-	root := "."
+	modulePath = readModulePath("go.mod")
+	var roots []string
 	switch flag.NArg() {
 	case 0:
+		roots = defaultRoots
 	case 1:
-		root = flag.Arg(0)
+		root := flag.Arg(0)
 		root = strings.TrimPrefix(root, "./")
 		root = strings.TrimSuffix(root, "/")
+		roots = []string{root}
 	default:
 		log.Fatalf("Usage: genreadme [dir]")
 	}
@@ -54,27 +68,29 @@ func main() {
 		updateErrs = append(updateErrs, err)
 	}).Limit(runtime.NumCPU() * 2) // usually I/O bound
 
-	g.Go(func() error {
-		return fs.WalkDir(os.DirFS("."), root, func(path string, d fs.DirEntry, err error) error {
-			if err != nil {
-				return err
-			}
-			if !d.IsDir() {
+	for _, root := range roots {
+		g.Go(func() error {
+			return fs.WalkDir(os.DirFS("."), root, func(path string, d fs.DirEntry, err error) error {
+				if err != nil {
+					return err
+				}
+				if !d.IsDir() {
+					return nil
+				}
+				if skip[path] {
+					return fs.SkipDir
+				}
+				base := filepath.Base(path)
+				if base == "testdata" || (path != "." && base[0] == '.') {
+					return fs.SkipDir
+				}
+				run(func() error {
+					return update(path)
+				})
 				return nil
-			}
-			if skip[path] {
-				return fs.SkipDir
-			}
-			base := filepath.Base(path)
-			if base == "testdata" || (path != "." && base[0] == '.') {
-				return fs.SkipDir
-			}
-			run(func() error {
-				return update(path)
 			})
-			return nil
 		})
-	})
+	}
 	g.Wait()
 	if err := errors.Join(updateErrs...); err != nil {
 		log.Fatal(err)
@@ -126,7 +142,7 @@ func getNewContent(dir string) (newContent []byte, err error) {
 		quickTest func(dir string, dents []fs.DirEntry) bool
 		generate  func(dir string) ([]byte, error)
 	}{
-		{"go", hasPkgMainGoFiles, genGoDoc},
+		{"go", hasGoFiles, genGoDoc},
 	}
 	for _, gen := range generators {
 		if !gen.quickTest(dir, dents) {
@@ -147,7 +163,11 @@ func genGoDoc(dir string) ([]byte, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to get absolute path for %q: %w", dir, err)
 	}
-	godoc, err := pkgdoc.PackageDoc(abs)
+	var importPath string
+	if modulePath != "" {
+		importPath = path.Join(modulePath, filepath.ToSlash(dir))
+	}
+	godoc, err := pkgdoc.PackageDoc(abs, importPath)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get package doc for %q: %w", dir, err)
 	}
@@ -155,13 +175,22 @@ func genGoDoc(dir string) ([]byte, error) {
 		// No godoc; skipping.
 		return nil, nil
 	}
-	if bytes.HasPrefix(godoc, []byte("package ")) {
-		// Not a package main; skipping.
+	isLibrary := bytes.HasPrefix(godoc, []byte("package "))
+	if isLibrary {
+		// Strip the "package X // import Y\n\n" clause emitted for library packages.
+		if i := bytes.Index(godoc, []byte("\n\n")); i != -1 {
+			godoc = godoc[i+2:]
+		}
+	}
+	if len(bytes.TrimSpace(godoc)) == 0 {
 		return nil, nil
 	}
 	var buf bytes.Buffer
 	io.WriteString(&buf, genHeader)
 	fmt.Fprintf(&buf, "\n# %s\n\n", filepath.Base(dir))
+	if isLibrary && importPath != "" {
+		fmt.Fprintf(&buf, "[![Go Reference](https://pkg.go.dev/badge/%s.svg)](https://pkg.go.dev/%s)\n\n", importPath, importPath)
+	}
 	buf.Write(godoc)
 
 	if !bytes.Contains(godoc, []byte("## Deploying")) {
@@ -184,6 +213,21 @@ const genHeader = "<!-- README.md auto-generated by misc/genreadme; DO NOT EDIT.
 
 func isGenerated(b []byte) bool { return bytes.HasPrefix(b, []byte(genHeader)) }
 
+// readModulePath returns the module path declared in the given go.mod file,
+// or "" if it can't be read or parsed.
+func readModulePath(file string) string {
+	b, err := os.ReadFile(file)
+	if err != nil {
+		return ""
+	}
+	for line := range strings.Lines(string(b)) {
+		if rest, ok := strings.CutPrefix(strings.TrimSpace(line), "module "); ok {
+			return strings.Trim(strings.TrimSpace(rest), `"`)
+		}
+	}
+	return ""
+}
+
 func hasBuildkite(dir string) bool {
 	if bkSkip[dir] {
 		return false
@@ -192,7 +236,7 @@ func hasBuildkite(dir string) bool {
 	return flyErr != nil
 }
 
-func hasPkgMainGoFiles(dir string, dents []fs.DirEntry) bool {
+func hasGoFiles(dir string, dents []fs.DirEntry) bool {
 	var fset *token.FileSet
 
 	for _, de := range dents {
@@ -217,7 +261,7 @@ func hasPkgMainGoFiles(dir string, dents []fs.DirEntry) bool {
 			continue
 		}
 
-		return pkgFile.Name.Name == "main"
+		return pkgFile.Name.Name != ""
 	}
 	return false
 }