diff options
Diffstat (limited to 'tstest')
| -rw-r--r-- | tstest/natlab/vmtest/assets/event.html | 39 | ||||
| -rw-r--r-- | tstest/natlab/vmtest/assets/htmx-websocket.min.js.gz | bin | 0 -> 4249 bytes | |||
| -rw-r--r-- | tstest/natlab/vmtest/assets/htmx.min.js.gz | bin | 0 -> 16409 bytes | |||
| -rw-r--r-- | tstest/natlab/vmtest/assets/index.html | 107 | ||||
| -rw-r--r-- | tstest/natlab/vmtest/assets/style.css | 168 | ||||
| -rw-r--r-- | tstest/natlab/vmtest/fetch-htmx.go | 85 | ||||
| -rw-r--r-- | tstest/natlab/vmtest/qemu.go | 73 | ||||
| -rw-r--r-- | tstest/natlab/vmtest/vmstatus.go | 320 | ||||
| -rw-r--r-- | tstest/natlab/vmtest/vmtest.go | 307 | ||||
| -rw-r--r-- | tstest/natlab/vmtest/vmtest_test.go | 10 | ||||
| -rw-r--r-- | tstest/natlab/vmtest/web.go | 198 | ||||
| -rw-r--r-- | tstest/natlab/vnet/vnet.go | 19 |
12 files changed, 1318 insertions, 8 deletions
diff --git a/tstest/natlab/vmtest/assets/event.html b/tstest/natlab/vmtest/assets/event.html new file mode 100644 index 000000000..70d8e69cf --- /dev/null +++ b/tstest/natlab/vmtest/assets/event.html @@ -0,0 +1,39 @@ +{{if eq .Type "test_status"}} +<span class="test-status test-{{.Message}}" id="test-status" hx-swap-oob="outerHTML">{{.Message}} ({{.Detail}})</span> +{{end}} + +{{if eq .Type "step_changed"}} +<div class="step step-{{.Step.Status}}" id="step-{{.Step.Index}}" hx-swap-oob="outerHTML"> + <span class="step-icon">{{.Step.Status.Icon}}</span> + <span class="step-name">{{.Step.Name}}</span> + <span class="step-time">{{formatDuration .Step.Elapsed}}</span> +</div> +{{end}} + +{{if eq .Type "console_output"}} +<div id="console-{{.NodeName}}" hx-swap-oob="beforeend">{{ansi .Message}} +</div> +{{end}} + +{{if eq .Type "dhcp_discover"}} +<span id="dhcp-{{.NodeName}}-{{.NIC}}" hx-swap-oob="innerHTML">Discover sent</span> +{{end}} + +{{if eq .Type "dhcp_offer"}} +<span id="dhcp-{{.NodeName}}-{{.NIC}}" hx-swap-oob="innerHTML">Offered {{.Detail}}</span> +{{end}} + +{{if eq .Type "dhcp_request"}} +<span id="dhcp-{{.NodeName}}-{{.NIC}}" hx-swap-oob="innerHTML">Requesting {{.Detail}}</span> +{{end}} + +{{if eq .Type "dhcp_ack"}} +<span id="dhcp-{{.NodeName}}-{{.NIC}}" hx-swap-oob="innerHTML">Got {{.Detail}}</span> +{{end}} + +{{if eq .Type "tailscale"}} +<span id="ts-{{.NodeName}}" hx-swap-oob="innerHTML">{{.Detail}}</span> +{{end}} + +<div id="events" hx-swap-oob="beforeend"><div class="event event-{{.Type}}"><span class="event-time">{{.Time.Format "15:04:05.000"}}</span> {{if .NodeName}}<span class="event-node">[{{.NodeName}}]</span> {{end}}<span class="event-msg">{{.Message}}</span>{{if .Detail}} <span class="event-detail">{{.Detail}}</span>{{end}}</div> +</div> diff --git a/tstest/natlab/vmtest/assets/htmx-websocket.min.js.gz b/tstest/natlab/vmtest/assets/htmx-websocket.min.js.gz Binary files differnew file mode 100644 index 000000000..4ed53be49 --- /dev/null +++ b/tstest/natlab/vmtest/assets/htmx-websocket.min.js.gz diff --git a/tstest/natlab/vmtest/assets/htmx.min.js.gz b/tstest/natlab/vmtest/assets/htmx.min.js.gz Binary files differnew file mode 100644 index 000000000..b75fea8d1 --- /dev/null +++ b/tstest/natlab/vmtest/assets/htmx.min.js.gz diff --git a/tstest/natlab/vmtest/assets/index.html b/tstest/natlab/vmtest/assets/index.html new file mode 100644 index 000000000..a9e30c532 --- /dev/null +++ b/tstest/natlab/vmtest/assets/index.html @@ -0,0 +1,107 @@ +<!DOCTYPE html> +<html> +<head> + <meta charset="utf-8"> + <title>VMTest: {{.TestName}}</title> + <script src="htmx.min.js"></script> + <script src="htmx-websocket.min.js"></script> + <link rel="stylesheet" href="style.css"> +</head> +<body hx-ext="ws" ws-connect="ws"> + +<h1>VMTest: {{.TestName}} <span class="test-status test-{{.TestStatus.State}}" id="test-status">{{.TestStatus.State}} ({{formatDuration .TestStatus.Elapsed}})</span></h1> + +<div class="steps"> + <h2>Progress</h2> + {{range .Steps}} + <div class="step step-{{.Status}}" id="step-{{.Index}}"> + <span class="step-icon">{{.Status.Icon}}</span> + <span class="step-name">{{.Name}}</span> + <span class="step-time">{{if ne .Status.String "pending"}}{{formatDuration .Elapsed}}{{end}}</span> + </div> + {{end}} +</div> + +<div class="vm-grid"> + {{range $node := .Nodes}} + <div class="vm-card" id="vm-{{$node.Name}}"> + <div class="vm-header"> + <span class="vm-name">{{$node.Name}}</span> + <span class="vm-os">{{$node.OS}}</span> + </div> + <div class="vm-status"> + {{range $i, $nic := $node.NICs}} + <div class="vm-status-line"> + <span class="vm-status-label">DHCP{{if gt (len $node.NICs) 1}} ({{$nic.NetName}}){{end}}:</span> + <span class="vm-status-value" id="dhcp-{{$node.Name}}-{{$i}}">{{$nic.DHCP}}</span> + </div> + {{end}} + {{if $node.JoinsTailnet}} + <div class="vm-status-line"> + <span class="vm-status-label">Tailscale:</span> + <span class="vm-status-value" id="ts-{{$node.Name}}">{{$node.Tailscale}}</span> + </div> + {{end}} + </div> + <div class="console" id="console-{{$node.Name}}">{{range $node.Console}}{{ansi .}} +{{end}}</div> + </div> + {{end}} +</div> + +<div class="event-log"> + <h2>Events</h2> + <div class="events" id="events"></div> +</div> + +<script> +// Tick the elapsed time on the test status badge while the test is running. +(function() { + var startTime = {{.TestStatus.StartUnixMilli}}; + var el = document.getElementById("test-status"); + var timer = setInterval(function() { + if (!el || !el.classList.contains("test-Running")) { + clearInterval(timer); + return; + } + var elapsed = Date.now() - startTime; + var secs = elapsed / 1000; + var text; + if (secs < 1) { + text = Math.round(elapsed) + "ms"; + } else { + text = secs.toFixed(1) + "s"; + } + el.textContent = "Running (" + text + ")"; + }, 100); +})(); + +// Auto-scroll console divs to bottom unless user has scrolled up. +// Re-enable auto-scroll when user scrolls back to the bottom. +(function() { + var consoles = document.querySelectorAll(".console"); + consoles.forEach(function(el) { + el._autoScroll = true; + el.addEventListener("scroll", function() { + // At bottom if scrollTop + clientHeight >= scrollHeight - small threshold + var atBottom = el.scrollTop + el.clientHeight >= el.scrollHeight - 5; + el._autoScroll = atBottom; + }); + }); + // Use MutationObserver to detect when content is added to console divs. + var observer = new MutationObserver(function(mutations) { + mutations.forEach(function(m) { + var el = m.target; + if (el.classList && el.classList.contains("console") && el._autoScroll) { + el.scrollTop = el.scrollHeight; + } + }); + }); + consoles.forEach(function(el) { + observer.observe(el, { childList: true, characterData: true, subtree: true }); + }); +})(); +</script> + +</body> +</html> diff --git a/tstest/natlab/vmtest/assets/style.css b/tstest/natlab/vmtest/assets/style.css new file mode 100644 index 000000000..fff676dd5 --- /dev/null +++ b/tstest/natlab/vmtest/assets/style.css @@ -0,0 +1,168 @@ +/* CSS reset */ +*, *::before, *::after { box-sizing: border-box; } +* { margin: 0; } +body { + font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; + line-height: 1.5; + background: #1a1a2e; + color: #e0e0e0; + padding: 16px; +} + +h1 { + font-size: 1.4em; + margin-bottom: 16px; + color: #fff; +} + +.test-status { + font-size: 0.7em; + padding: 2px 10px; + border-radius: 4px; + font-weight: bold; + vertical-align: middle; +} + +.test-Running { background: #2563eb; color: #fff; } +.test-Passed { background: #16a34a; color: #fff; } +.test-Failed { background: #dc2626; color: #fff; } + +h2 { + font-size: 1.1em; + margin-bottom: 8px; + color: #ccc; +} + +/* Step progress panel */ +.steps { + background: #16213e; + border: 1px solid #333; + border-radius: 6px; + padding: 12px; + margin-bottom: 16px; +} + +.step { + display: flex; + align-items: center; + gap: 8px; + padding: 4px 8px; + font-family: monospace; + font-size: 13px; + border-radius: 3px; +} + +.step-pending { color: #666; } +.step-running { color: #4af; font-weight: bold; background: rgba(68, 170, 255, 0.1); } +.step-done { color: #4a4; } +.step-failed { color: #f44; font-weight: bold; background: rgba(255, 68, 68, 0.1); } + +.step-icon { width: 1.2em; text-align: center; } +.step-name { flex: 1; } +.step-time { color: #666; font-size: 12px; min-width: 6em; text-align: right; } + +/* VM card grid */ +.vm-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(400px, 1fr)); + gap: 12px; + margin-bottom: 16px; +} + +.vm-card { + background: #16213e; + border: 1px solid #333; + border-radius: 6px; + padding: 12px; +} + +.vm-header { + display: flex; + align-items: center; + gap: 8px; + margin-bottom: 8px; +} + +.vm-name { + font-weight: bold; + font-size: 1.1em; + color: #fff; +} + +.vm-os { + font-size: 0.8em; + background: #333; + padding: 1px 6px; + border-radius: 3px; + color: #aaa; +} + +.vm-status { + display: flex; + flex-direction: column; + gap: 2px; + margin-bottom: 8px; + font-family: monospace; + font-size: 13px; +} + +.vm-status-line { + display: flex; + gap: 8px; +} + +.vm-status-label { + color: #888; + min-width: 7em; +} + +.vm-status-value { + color: #4af; +} + +/* Console output */ +.console { + background: #0a0a0a; + color: #ccc; + font-family: "Cascadia Code", "Fira Code", "Consolas", monospace; + font-size: 11px; + line-height: 1.3; + max-height: 300px; + overflow-y: auto; + white-space: pre-wrap; + word-break: break-all; + padding: 8px; + border-radius: 4px; + border: 1px solid #222; +} + +/* Event log */ +.event-log { + background: #16213e; + border: 1px solid #333; + border-radius: 6px; + padding: 12px; +} + +.events { + max-height: 300px; + overflow-y: auto; +} + +.event { + font-family: monospace; + font-size: 12px; + padding: 1px 0; + border-bottom: 1px solid #1a1a2e; +} + +.event-time { color: #666; } +.event-node { color: #4af; font-weight: bold; } +.event-msg { color: #ccc; } +.event-detail { color: #888; } + +.event-dhcp_discover .event-msg, +.event-dhcp_request .event-msg { color: #fa4; } +.event-dhcp_offer .event-msg, +.event-dhcp_ack .event-msg { color: #4f4; } +.event-step_changed .event-msg { color: #aaf; } diff --git a/tstest/natlab/vmtest/fetch-htmx.go b/tstest/natlab/vmtest/fetch-htmx.go new file mode 100644 index 000000000..07082243a --- /dev/null +++ b/tstest/natlab/vmtest/fetch-htmx.go @@ -0,0 +1,85 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build ignore + +// Program fetch-htmx fetches and installs local copies of the HTMX +// library and its websocket extension, used by the vmtest web UI. +// It is meant to be run via go generate. +package main + +import ( + "compress/gzip" + "crypto/sha512" + "encoding/base64" + "fmt" + "io" + "log" + "net/http" + "os" +) + +func main() { + htmx, err := fetchHashed("https://unpkg.com/htmx.org@2.0.4", "HGfztofotfshcF7+8n44JQL2oJmowVChPTg48S+jvZoztPfvwD79OC/LTtG6dMp+") + if err != nil { + log.Fatalf("fetching htmx: %v", err) + } + + ws, err := fetchHashed("https://unpkg.com/htmx-ext-ws@2.0.2", "932iIqjARv+Gy0+r6RTGrfCkCKS5MsF539Iqf6Vt8L4YmbnnWI2DSFoMD90bvXd0") + if err != nil { + log.Fatalf("fetching htmx-websockets: %v", err) + } + + if err := writeGz("assets/htmx.min.js.gz", htmx); err != nil { + log.Fatalf("writing htmx.min.js.gz: %v", err) + } + if err := writeGz("assets/htmx-websocket.min.js.gz", ws); err != nil { + log.Fatalf("writing htmx-websocket.min.js.gz: %v", err) + } +} + +func writeGz(path string, bs []byte) error { + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + + g, err := gzip.NewWriterLevel(f, gzip.BestCompression) + if err != nil { + return err + } + + if _, err := g.Write(bs); err != nil { + return err + } + + if err := g.Flush(); err != nil { + return err + } + if err := f.Close(); err != nil { + return err + } + return nil +} + +func fetchHashed(url, wantHash string) ([]byte, error) { + resp, err := http.Get(url) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("fetching %q returned error status: %s", url, resp.Status) + } + ret, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading file from %q: %v", url, err) + } + h := sha512.Sum384(ret) + got := base64.StdEncoding.EncodeToString(h[:]) + if got != wantHash { + return nil, fmt.Errorf("wrong hash for %q: got %q, want %q", url, got, wantHash) + } + return ret, nil +} diff --git a/tstest/natlab/vmtest/qemu.go b/tstest/natlab/vmtest/qemu.go index df56322fa..a2ccd780c 100644 --- a/tstest/natlab/vmtest/qemu.go +++ b/tstest/natlab/vmtest/qemu.go @@ -5,6 +5,7 @@ package vmtest import ( "bytes" + "context" "encoding/json" "fmt" "net" @@ -13,6 +14,7 @@ import ( "path/filepath" "regexp" "strconv" + "strings" "time" "tailscale.com/tstest/natlab/vnet" @@ -163,6 +165,11 @@ func (e *Env) launchQEMU(name, logPath string, args []string) error { } e.t.Logf("launched QEMU for %s (pid %d), log: %s", name, cmd.Process.Pid, logPath) e.qemuProcs = append(e.qemuProcs, cmd) + + // Start tailing the VM console log for the web UI. + if e.ctx != nil { + go e.tailLogFile(e.ctx, name, logPath) + } e.t.Cleanup(func() { cmd.Process.Kill() cmd.Wait() @@ -237,3 +244,69 @@ func qmpQueryHostFwd(sockPath string) (int, error) { } return strconv.Atoi(m[1]) } + +// tailLogFile tails a VM's serial console log file and publishes each line +// as an EventConsoleOutput to the event bus for the web UI. +func (e *Env) tailLogFile(ctx context.Context, name, logPath string) { + // Wait for the file to appear (QEMU may not have created it yet). + var f *os.File + for { + var err error + f, err = os.Open(logPath) + if err == nil { + break + } + select { + case <-ctx.Done(): + return + case <-time.After(100 * time.Millisecond): + } + } + defer f.Close() + + // Read the file in a loop, tracking our position manually. + // We can't use bufio.Scanner because it caches EOF and won't + // pick up new data appended by QEMU after the first EOF. + var buf []byte + var partial string // incomplete line (no trailing newline yet) + readBuf := make([]byte, 4096) + for { + n, err := f.Read(readBuf) + if n > 0 { + buf = append(buf, readBuf[:n]...) + // Split into complete lines. + for { + idx := bytes.IndexByte(buf, '\n') + if idx < 0 { + break + } + line := partial + string(buf[:idx]) + partial = "" + buf = buf[idx+1:] + // Strip trailing \r from serial consoles. + line = strings.TrimRight(line, "\r") + if line == "" { + continue + } + e.appendConsoleLine(name, line) + e.eventBus.Publish(VMEvent{ + NodeName: name, + Type: EventConsoleOutput, + Message: line, + }) + } + if len(buf) > 0 { + partial = string(buf) + buf = buf[:0] + } + } + if err != nil || n == 0 { + // EOF or error — wait for more data. + select { + case <-ctx.Done(): + return + case <-time.After(100 * time.Millisecond): + } + } + } +} diff --git a/tstest/natlab/vmtest/vmstatus.go b/tstest/natlab/vmtest/vmstatus.go new file mode 100644 index 000000000..cc6cb4767 --- /dev/null +++ b/tstest/natlab/vmtest/vmstatus.go @@ -0,0 +1,320 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package vmtest + +import ( + "fmt" + "sync" + "time" +) + +// StepStatus is the state of a declared test step. +type StepStatus int + +const ( + StepPending StepStatus = iota // not yet started + StepRunning // Begin called + StepDone // End(nil) called + StepFailed // End(non-nil) called +) + +func (s StepStatus) String() string { + switch s { + case StepPending: + return "pending" + case StepRunning: + return "running" + case StepDone: + return "done" + case StepFailed: + return "failed" + } + return fmt.Sprintf("StepStatus(%d)", int(s)) +} + +// Icon returns a Unicode icon for the step status. +func (s StepStatus) Icon() string { + switch s { + case StepPending: + return "○" + case StepRunning: + return "◉" + case StepDone: + return "✓" + case StepFailed: + return "✗" + } + return "?" +} + +// Step is a declared stage of a test, created by [Env.AddStep]. +// The web UI shows all steps from the start, tracking their progress. +type Step struct { + mu sync.Mutex + name string + index int // 0-based position in Env.steps + env *Env + status StepStatus + err error + started time.Time + ended time.Time +} + +// Name returns the step's display name. +func (s *Step) Name() string { return s.name } + +// Index returns the step's 0-based position. +func (s *Step) Index() int { return s.index } + +// Status returns the current status. +func (s *Step) Status() StepStatus { + s.mu.Lock() + defer s.mu.Unlock() + return s.status +} + +// Err returns the error if the step failed, or nil. +func (s *Step) Err() error { + s.mu.Lock() + defer s.mu.Unlock() + return s.err +} + +// Elapsed returns how long the step has been running (if running) +// or how long it took (if done/failed). Returns 0 if pending. +func (s *Step) Elapsed() time.Duration { + s.mu.Lock() + defer s.mu.Unlock() + if s.started.IsZero() { + return 0 + } + if !s.ended.IsZero() { + return s.ended.Sub(s.started) + } + return time.Since(s.started) +} + +// Begin marks the step as running. Publishes an event to the web UI. +func (s *Step) Begin() { + s.mu.Lock() + if s.status != StepPending { + s.mu.Unlock() + panic(fmt.Sprintf("Step %q: Begin called in state %s", s.name, s.status)) + } + s.started = time.Now() + s.status = StepRunning + s.mu.Unlock() + s.env.publishStepChange(s) +} + +// End marks the step as done (err == nil) or failed (err != nil). +// It publishes a status change event to the web UI. +// It does not call t.Fatalf; callers should handle the error as appropriate +// (return it from errgroup, call t.Fatalf on the test goroutine, etc). +func (s *Step) End(err error) { + s.mu.Lock() + if s.status != StepRunning { + s.mu.Unlock() + panic(fmt.Sprintf("Step %q: End called in state %s", s.name, s.status)) + } + s.ended = time.Now() + if err != nil { + s.status = StepFailed + s.err = err + } else { + s.status = StepDone + } + s.mu.Unlock() + s.env.publishStepChange(s) +} + +// EventType identifies the kind of event published to the EventBus. +type EventType string + +const ( + EventStepChanged EventType = "step_changed" // a Step changed status + EventConsoleOutput EventType = "console_output" // serial console line + EventDHCPDiscover EventType = "dhcp_discover" // VM sent DHCP Discover + EventDHCPOffer EventType = "dhcp_offer" // server sent DHCP Offer + EventDHCPRequest EventType = "dhcp_request" // VM sent DHCP Request + EventDHCPAck EventType = "dhcp_ack" // server sent DHCP Ack + EventTailscale EventType = "tailscale" // Tailscale status change + EventTestStatus EventType = "test_status" // test Running/Passed/Failed +) + +// TestStatus tracks whether the overall test is running, passed, or failed. +type TestStatus struct { + mu sync.Mutex + state string // "Running", "Passed", "Failed" + started time.Time + ended time.Time +} + +func newTestStatus() *TestStatus { + return &TestStatus{state: "Running", started: time.Now()} +} + +// State returns the current test state. +func (ts *TestStatus) State() string { + ts.mu.Lock() + defer ts.mu.Unlock() + return ts.state +} + +// Elapsed returns total test duration. +func (ts *TestStatus) Elapsed() time.Duration { + ts.mu.Lock() + defer ts.mu.Unlock() + if !ts.ended.IsZero() { + return ts.ended.Sub(ts.started) + } + return time.Since(ts.started) +} + +// StartUnixMilli returns the test start time as Unix milliseconds, +// for the client-side elapsed timer. +func (ts *TestStatus) StartUnixMilli() int64 { + ts.mu.Lock() + defer ts.mu.Unlock() + return ts.started.UnixMilli() +} + +// finish marks the test as passed or failed. +func (ts *TestStatus) finish(failed bool) { + ts.mu.Lock() + defer ts.mu.Unlock() + ts.ended = time.Now() + if failed { + ts.state = "Failed" + } else { + ts.state = "Passed" + } +} + +// VMEvent is a single event published to the [EventBus]. +type VMEvent struct { + Time time.Time + NodeName string // "" for global events + Type EventType + Message string // human-readable description + Detail string // e.g. IP address, node key + Step *Step // non-nil for EventStepChanged + NIC int // NIC index for DHCP events (0-based); -1 if not applicable +} + +// NICStatus is the DHCP state for one NIC on a node. +type NICStatus struct { + NetName string // human label like "192.168.1.0/24" or "10.0.0.0/24" + DHCP string // "waiting", "Discover sent", "Got 10.0.0.101", etc. +} + +// NodeStatus tracks the current DHCP and Tailscale state of a VM node +// for rendering on the web UI's initial page load. +type NodeStatus struct { + Name string + OS string + NICs []NICStatus // one per NIC; index matches NIC index + JoinsTailnet bool // whether this node runs Tailscale + Tailscale string // "--", "Up (100.64.0.1)", etc. + Console []string // recent console output lines (ring buffer) +} + +const maxConsoleLines = 200 + +const ( + eventBusHistorySize = 500 + subscriberChannelSize = 1000 +) + +// EventBus broadcasts VMEvents to subscribers and keeps a history for +// late joiners. It is safe for concurrent use. +type EventBus struct { + mu sync.Mutex + history []VMEvent + subscribers map[*subscriber]struct{} +} + +func newEventBus() *EventBus { + return &EventBus{ + subscribers: make(map[*subscriber]struct{}), + } +} + +// Publish sends an event to all subscribers and appends it to the history. +// Non-blocking: slow subscribers are skipped. +func (b *EventBus) Publish(ev VMEvent) { + if ev.Time.IsZero() { + ev.Time = time.Now() + } + b.mu.Lock() + defer b.mu.Unlock() + b.history = append(b.history, ev) + if len(b.history) > eventBusHistorySize { + // Trim old events. + copy(b.history, b.history[len(b.history)-eventBusHistorySize:]) + b.history = b.history[:eventBusHistorySize] + } + for sub := range b.subscribers { + select { + case sub.ch <- ev: + default: + // Slow consumer, skip. + } + } +} + +// Subscribe returns a new subscriber that receives the event history +// followed by live events. +func (b *EventBus) Subscribe() *subscriber { + b.mu.Lock() + defer b.mu.Unlock() + sub := &subscriber{ + bus: b, + ch: make(chan VMEvent, subscriberChannelSize), + done: make(chan struct{}), + } + // Send history. + for _, ev := range b.history { + select { + case sub.ch <- ev: + default: + } + } + b.subscribers[sub] = struct{}{} + return sub +} + +func (b *EventBus) unsubscribe(sub *subscriber) { + b.mu.Lock() + defer b.mu.Unlock() + delete(b.subscribers, sub) +} + +// subscriber receives events from an [EventBus]. +type subscriber struct { + bus *EventBus + ch chan VMEvent + done chan struct{} + once sync.Once +} + +// Events returns the channel of events. Closed when Close is called. +func (s *subscriber) Events() <-chan VMEvent { + return s.ch +} + +// Close unsubscribes and closes the event channel. +func (s *subscriber) Close() { + s.once.Do(func() { + if s.bus != nil { + s.bus.unsubscribe(s) + } + close(s.done) + }) +} + +// Done returns a channel that's closed when Close is called. +func (s *subscriber) Done() <-chan struct{} { + return s.done +} diff --git a/tstest/natlab/vmtest/vmtest.go b/tstest/natlab/vmtest/vmtest.go index e6c89467f..39c453b4b 100644 --- a/tstest/natlab/vmtest/vmtest.go +++ b/tstest/natlab/vmtest/vmtest.go @@ -27,9 +27,11 @@ import ( "os/exec" "path/filepath" "strings" + "sync" "testing" "time" + "github.com/google/gopacket/layers" "golang.org/x/sync/errgroup" "tailscale.com/client/local" "tailscale.com/ipn" @@ -60,6 +62,15 @@ type Env struct { gokrazyKernel string // path to gokrazy kernel qemuProcs []*exec.Cmd // launched QEMU processes + + // Web UI support. + ctx context.Context // cancelled when test ends + eventBus *EventBus + testStatus *TestStatus + steps []*Step + + nodeStatusMu sync.Mutex + nodeStatus map[string]*NodeStatus // keyed by node name } // logVerbosef logs a message only when --verbose-vm-debug is set. @@ -70,6 +81,145 @@ func (e *Env) logVerbosef(format string, args ...any) { } } +// AddStep declares an expected stage of the test. The web UI shows all steps +// from the start, tracking their progress. Call before or during the test. +// Returns a *Step whose Begin/End methods drive the progress display. +func (e *Env) AddStep(name string) *Step { + s := &Step{ + name: name, + index: len(e.steps), + env: e, + } + e.steps = append(e.steps, s) + return s +} + +// Steps returns all declared steps in order. +func (e *Env) Steps() []*Step { + return e.steps +} + +// publishStepChange publishes a step status change event. +func (e *Env) publishStepChange(s *Step) { + e.eventBus.Publish(VMEvent{ + Type: EventStepChanged, + Message: fmt.Sprintf("%s %s", s.Status().Icon(), s.name), + Step: s, + }) +} + +// initNodeStatus initializes the NodeStatus for all nodes. Called after +// AddNode but before Start so the web UI can render them. +func (e *Env) initNodeStatus() { + e.nodeStatusMu.Lock() + defer e.nodeStatusMu.Unlock() + for _, n := range e.nodes { + nics := make([]NICStatus, len(n.nets)) + for i := range n.nets { + nics[i] = NICStatus{ + NetName: e.nicLabel(n, i), + DHCP: "waiting", + } + } + e.nodeStatus[n.name] = &NodeStatus{ + Name: n.name, + OS: n.os.Name, + NICs: nics, + JoinsTailnet: n.joinTailnet, + Tailscale: "--", + } + } +} + +// nicLabel returns a short human-readable label for a node's i-th NIC. +// After Start(), we can use the assigned LAN IP. Before that, we use "NIC N". +func (e *Env) nicLabel(n *Node, i int) string { + if n.vnetNode != nil { + ip := n.vnetNode.LanIP(n.nets[i]) + if ip.IsValid() { + return ip.String() + } + } + return fmt.Sprintf("NIC %d", i) +} + +// getNodeStatus returns the current status for a node. +func (e *Env) getNodeStatus(name string) NodeStatus { + e.nodeStatusMu.Lock() + defer e.nodeStatusMu.Unlock() + ns := e.nodeStatus[name] + if ns == nil { + return NodeStatus{Name: name, Tailscale: "--"} + } + return *ns +} + +// setNodeDHCP updates the DHCP status for a specific NIC on a node. +func (e *Env) setNodeDHCP(name string, nicIdx int, status string) { + e.nodeStatusMu.Lock() + ns := e.nodeStatus[name] + if ns != nil && nicIdx < len(ns.NICs) { + ns.NICs[nicIdx].DHCP = status + } + e.nodeStatusMu.Unlock() +} + +// setNodeTailscale updates the Tailscale status for a node and publishes +// an event so the web UI updates via WebSocket. +func (e *Env) setNodeTailscale(name, status string) { + e.nodeStatusMu.Lock() + ns := e.nodeStatus[name] + if ns != nil { + ns.Tailscale = status + } + e.nodeStatusMu.Unlock() + e.eventBus.Publish(VMEvent{ + NodeName: name, + Type: EventTailscale, + Message: "Tailscale: " + status, + Detail: status, + }) +} + +// appendConsoleLine adds a line to a node's console buffer. +func (e *Env) appendConsoleLine(name, line string) { + e.nodeStatusMu.Lock() + ns := e.nodeStatus[name] + if ns != nil { + ns.Console = append(ns.Console, line) + if len(ns.Console) > maxConsoleLines { + ns.Console = ns.Console[len(ns.Console)-maxConsoleLines:] + } + } + e.nodeStatusMu.Unlock() +} + +// nicIndexForMAC returns the NIC index (0-based) for a given MAC on a node. +// Returns -1 if not found. +func (e *Env) nicIndexForMAC(name string, mac vnet.MAC) int { + for _, n := range e.nodes { + if n.name != name { + continue + } + for i := range n.nets { + if n.vnetNode.NICMac(i) == mac { + return i + } + } + } + return -1 +} + +// nodeNameByNum returns the node name for a given vnet node number. +func (e *Env) nodeNameByNum(num int) string { + for _, n := range e.nodes { + if n.num == num { + return n.name + } + } + return fmt.Sprintf("node%d", num) +} + // New creates a new test environment. It skips the test if --run-vm-tests is not set. func New(t testing.TB) *Env { if !*runVMTests { @@ -77,11 +227,23 @@ func New(t testing.TB) *Env { } tempDir := t.TempDir() - return &Env{ - t: t, - tempDir: tempDir, - binDir: filepath.Join(tempDir, "bin"), + e := &Env{ + t: t, + tempDir: tempDir, + binDir: filepath.Join(tempDir, "bin"), + eventBus: newEventBus(), + testStatus: newTestStatus(), + nodeStatus: make(map[string]*NodeStatus), } + t.Cleanup(func() { + e.testStatus.finish(t.Failed()) + e.eventBus.Publish(VMEvent{ + Type: EventTestStatus, + Message: e.testStatus.State(), + Detail: formatDuration(e.testStatus.Elapsed()), + }) + }) + return e } // AddNetwork creates a new virtual network. Arguments follow the same pattern as @@ -179,6 +341,11 @@ func (e *Env) Start() { t := e.t ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) t.Cleanup(cancel) + e.ctx = ctx + + // Initialize node status and start web UI as early as possible. + e.initNodeStatus() + e.maybeStartWebServer() if err := os.MkdirAll(e.binDir, 0755); err != nil { t.Fatal(err) @@ -194,27 +361,94 @@ func (e *Env) Start() { } } + // Declare framework steps for the web UI. + // User-declared steps (from AddStep before Start) get moved to the end + // so framework steps (compile, image, QEMU, etc.) come first. + userSteps := e.steps + e.steps = nil + + compileSteps := map[platform]*Step{} + for _, p := range needPlatform.Slice() { + compileSteps[p] = e.AddStep(fmt.Sprintf("Compile %s_%s binaries", p.goos, p.goarch)) + } + imageSteps := map[string]*Step{} // keyed by OS name + didOS := set.Set[string]{} // dedup by image name + for _, n := range e.nodes { + if didOS.Contains(n.os.Name) { + continue + } + didOS.Add(n.os.Name) + if n.os.IsGokrazy { + imageSteps["gokrazy"] = e.AddStep("Build gokrazy image") + } else { + imageSteps[n.os.Name] = e.AddStep(fmt.Sprintf("Prepare %s image", n.os.Name)) + } + } + vnetStep := e.AddStep("Create virtual network") + + qemuSteps := map[string]*Step{} + agentSteps := map[string]*Step{} + tsUpSteps := map[string]*Step{} + for _, n := range e.nodes { + qemuSteps[n.name] = e.AddStep(fmt.Sprintf("Launch QEMU: %s", n.name)) + agentSteps[n.name] = e.AddStep(fmt.Sprintf("Wait for agent: %s", n.name)) + if n.joinTailnet { + tsUpSteps[n.name] = e.AddStep(fmt.Sprintf("Tailscale up: %s", n.name)) + } + } + + // Re-append user-declared steps after all framework steps. + for _, s := range userSteps { + s.index = len(e.steps) + e.steps = append(e.steps, s) + } + // Compile binaries and download/build images in parallel. // Any failure cancels the others via the errgroup context. eg, egCtx := errgroup.WithContext(ctx) for _, p := range needPlatform.Slice() { + step := compileSteps[p] eg.Go(func() error { - return e.compileBinariesForOS(egCtx, p.goos, p.goarch) + step.Begin() + err := e.compileBinariesForOS(egCtx, p.goos, p.goarch) + if err != nil { + step.End(err) + return err + } + step.End(nil) + return nil }) } - didOS := set.Set[string]{} // dedup by image name + didOS = set.Set[string]{} // reset for second pass for _, n := range e.nodes { if didOS.Contains(n.os.Name) { continue } didOS.Add(n.os.Name) if n.os.IsGokrazy { + step := imageSteps["gokrazy"] eg.Go(func() error { - return e.ensureGokrazy(egCtx) + step.Begin() + err := e.ensureGokrazy(egCtx) + if err != nil { + step.End(err) + return err + } + step.End(nil) + return nil }) } else { + step := imageSteps[n.os.Name] + osImg := n.os eg.Go(func() error { - return ensureImage(egCtx, n.os) + step.Begin() + err := ensureImage(egCtx, osImg) + if err != nil { + step.End(err) + return err + } + step.End(nil) + return nil }) } } @@ -223,6 +457,7 @@ func (e *Env) Start() { } // Create the vnet server. + vnetStep.Begin() var err error e.server, err = vnet.New(&e.cfg) if err != nil { @@ -230,6 +465,50 @@ func (e *Env) Start() { } t.Cleanup(func() { e.server.Close() }) + // Register DHCP event callback for the web UI. + e.server.SetDHCPCallback(func(mac vnet.MAC, nodeNum int, msgType layers.DHCPMsgType, ip netip.Addr) { + name := e.nodeNameByNum(nodeNum) + nicIdx := e.nicIndexForMAC(name, mac) + ipStr := ip.String() + switch msgType { + case layers.DHCPMsgTypeDiscover: + e.setNodeDHCP(name, nicIdx, "Discover sent") + e.eventBus.Publish(VMEvent{ + NodeName: name, + Type: EventDHCPDiscover, + Message: "DHCP Discover sent", + NIC: nicIdx, + }) + case layers.DHCPMsgTypeOffer: + e.setNodeDHCP(name, nicIdx, "Offered "+ipStr) + e.eventBus.Publish(VMEvent{ + NodeName: name, + Type: EventDHCPOffer, + Message: "DHCP Offer received", + Detail: ipStr, + NIC: nicIdx, + }) + case layers.DHCPMsgTypeRequest: + e.setNodeDHCP(name, nicIdx, "Requesting "+ipStr) + e.eventBus.Publish(VMEvent{ + NodeName: name, + Type: EventDHCPRequest, + Message: "DHCP Request sent", + Detail: ipStr, + NIC: nicIdx, + }) + case layers.DHCPMsgTypeAck: + e.setNodeDHCP(name, nicIdx, "Got "+ipStr) + e.eventBus.Publish(VMEvent{ + NodeName: name, + Type: EventDHCPAck, + Message: "DHCP Ack: got " + ipStr, + Detail: ipStr, + NIC: nicIdx, + }) + } + }) + // Register compiled binaries with the file server VIP. // Binaries are registered at <goos>_<goarch>/<name> (e.g. "linux_amd64/tta"). for _, p := range needPlatform.Slice() { @@ -242,6 +521,7 @@ func (e *Env) Start() { e.server.RegisterFile(dir+"/"+name, data) } } + vnetStep.End(nil) // Cloud-init config is delivered via local seed ISOs (created in startCloudQEMU), // not via the cloud-init HTTP VIP, because network-config must be available @@ -267,9 +547,12 @@ func (e *Env) Start() { // Launch QEMU processes. for _, n := range e.nodes { + step := qemuSteps[n.name] + step.Begin() if err := e.startQEMU(n); err != nil { t.Fatalf("startQEMU(%s): %v", n.name, err) } + step.End(nil) } // Set up agent clients and wait for all agents to connect. @@ -282,12 +565,15 @@ func (e *Env) Start() { var agentEg errgroup.Group for _, n := range e.nodes { agentEg.Go(func() error { + aStep := agentSteps[n.name] + aStep.Begin() t.Logf("[%s] waiting for agent...", n.name) st, err := n.agent.Status(ctx) if err != nil { return fmt.Errorf("[%s] agent status: %w", n.name, err) } t.Logf("[%s] agent connected, backend state: %s", n.name, st.BackendState) + aStep.End(nil) if n.vnetNode.HostFirewall() { if err := n.agent.EnableHostFirewall(ctx); err != nil { @@ -296,6 +582,8 @@ func (e *Env) Start() { } if n.joinTailnet { + tsStep := tsUpSteps[n.name] + tsStep.Begin() if err := e.tailscaleUp(ctx, n); err != nil { return fmt.Errorf("[%s] tailscale up: %w", n.name, err) } @@ -306,7 +594,10 @@ func (e *Env) Start() { if st.BackendState != "Running" { return fmt.Errorf("[%s] state = %q, want Running", n.name, st.BackendState) } + ips := fmt.Sprintf("%v", st.Self.TailscaleIPs) + e.setNodeTailscale(n.name, "Running "+ips) t.Logf("[%s] up with %v", n.name, st.Self.TailscaleIPs) + tsStep.End(nil) } return nil diff --git a/tstest/natlab/vmtest/vmtest_test.go b/tstest/natlab/vmtest/vmtest_test.go index 91c8359f1..ddd69c910 100644 --- a/tstest/natlab/vmtest/vmtest_test.go +++ b/tstest/natlab/vmtest/vmtest_test.go @@ -37,11 +37,21 @@ func testSubnetRouterForOS(t testing.TB, srOS vmtest.OSImage) { vmtest.DontJoinTailnet(), vmtest.WebServer(8080)) + // Declare test-specific steps for the web UI. + approveStep := env.AddStep("Approve subnet routes") + httpStep := env.AddStep("HTTP GET through subnet router") + env.Start() + + approveStep.Begin() env.ApproveRoutes(sr, "10.0.0.0/24") + approveStep.End(nil) + httpStep.Begin() body := env.HTTPGet(client, fmt.Sprintf("http://%s:8080/", backend.LanIP(internalNet))) if !strings.Contains(body, "Hello world I am backend") { + httpStep.End(fmt.Errorf("got %q", body)) t.Fatalf("got %q", body) } + httpStep.End(nil) } diff --git a/tstest/natlab/vmtest/web.go b/tstest/natlab/vmtest/web.go new file mode 100644 index 000000000..3c96f9e19 --- /dev/null +++ b/tstest/natlab/vmtest/web.go @@ -0,0 +1,198 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package vmtest + +//go:generate go run fetch-htmx.go + +import ( + "embed" + "flag" + "fmt" + "hash/crc32" + "html/template" + "io" + "io/fs" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/coder/websocket" + "github.com/robert-nix/ansihtml" +) + +var vmtestWeb = flag.String("vmtest-web", "", "listen address for vmtest web UI (e.g. :0, localhost:0, :8080)") + +//go:embed assets/*.html +var templatesSrc embed.FS + +//go:embed assets/*.css assets/*.min.js.gz +var staticAssets embed.FS + +var tmpl = sync.OnceValue(func() *template.Template { + d, err := fs.Sub(templatesSrc, "assets") + if err != nil { + panic(fmt.Errorf("getting vmtest web templates subdir: %w", err)) + } + return template.Must(template.New("").Funcs(template.FuncMap{ + "formatDuration": formatDuration, + "ansi": ansiToHTML, + }).ParseFS(d, "*")) +}) + +// ansiToHTML converts a string with ANSI escape sequences to HTML with +// inline styles. Returns template.HTML so html/template doesn't double-escape it. +func ansiToHTML(s string) template.HTML { + return template.HTML(ansihtml.ConvertToHTML([]byte(s))) +} + +// formatDuration returns a human-readable duration like "1.2s" or "45.3s". +func formatDuration(d time.Duration) string { + if d < time.Second { + return fmt.Sprintf("%dms", d.Milliseconds()) + } + return fmt.Sprintf("%.1fs", d.Seconds()) +} + +// deterministicPort returns a deterministic port in the range [20000, 40000) +// based on the test name, so re-running the same test gets the same URL. +func deterministicPort(testName string) int { + return int(crc32.ChecksumIEEE([]byte(testName)))%20000 + 20000 +} + +// listenWeb listens on the given address. If the port is 0, it first tries a +// deterministic port based on the test name so re-runs get the same URL. +// Falls back to :0 (OS-assigned) on any listen error. +func (e *Env) listenWeb(addr string) (net.Listener, error) { + host, port, _ := net.SplitHostPort(addr) + if port == "0" { + detPort := deterministicPort(e.t.Name()) + detAddr := net.JoinHostPort(host, fmt.Sprintf("%d", detPort)) + if ln, err := net.Listen("tcp", detAddr); err == nil { + return ln, nil + } + // Deterministic port busy; fall back to OS-assigned. + } + return net.Listen("tcp", addr) +} + +// maybeStartWebServer starts the web UI if --vmtest-web is set. +// Called at the very top of Env.Start(), before compilation or image downloads. +func (e *Env) maybeStartWebServer() { + addr := *vmtestWeb + if addr == "" { + return + } + + ln, err := e.listenWeb(addr) + if err != nil { + e.t.Fatalf("vmtest-web listen: %v", err) + } + e.t.Cleanup(func() { ln.Close() }) + + actualAddr := ln.Addr().(*net.TCPAddr) + + host, _, _ := net.SplitHostPort(addr) + if host == "" || host == "0.0.0.0" || host == "::" { + hostname, err := os.Hostname() + if err != nil { + hostname = "localhost" + } + e.t.Logf("Status at http://%s:%d/", hostname, actualAddr.Port) + } else { + e.t.Logf("Status at http://%s/", actualAddr.String()) + } + + mux := http.NewServeMux() + mux.HandleFunc("GET /", e.serveIndex) + mux.HandleFunc("GET /ws", e.serveWebSocket) + mux.HandleFunc("GET /style.css", serveStaticAsset("style.css")) + mux.HandleFunc("GET /htmx.min.js", serveStaticAsset("htmx.min.js.gz")) + mux.HandleFunc("GET /htmx-websocket.min.js", serveStaticAsset("htmx-websocket.min.js.gz")) + + srv := &http.Server{Handler: mux} + go srv.Serve(ln) + e.t.Cleanup(func() { srv.Close() }) +} + +func serveStaticAsset(name string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(name, ".css"): + w.Header().Set("Content-Type", "text/css") + case strings.HasSuffix(name, ".min.js.gz"): + w.Header().Set("Content-Type", "text/javascript") + w.Header().Set("Content-Encoding", "gzip") + default: + http.Error(w, "not found", http.StatusNotFound) + return + } + f, err := staticAssets.Open(filepath.Join("assets", name)) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + defer f.Close() + io.Copy(w, f) + } +} + +func (e *Env) serveIndex(w http.ResponseWriter, r *http.Request) { + type indexData struct { + TestName string + TestStatus *TestStatus + Steps []*Step + Nodes []NodeStatus + } + + data := indexData{ + TestName: e.t.Name(), + TestStatus: e.testStatus, + Steps: e.Steps(), + } + for _, n := range e.nodes { + data.Nodes = append(data.Nodes, e.getNodeStatus(n.name)) + } + + w.Header().Set("Content-Type", "text/html; charset=utf-8") + if err := tmpl().ExecuteTemplate(w, "index.html", data); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } +} + +func (e *Env) serveWebSocket(w http.ResponseWriter, r *http.Request) { + conn, err := websocket.Accept(w, r, nil) + if err != nil { + return + } + defer conn.CloseNow() + wsCtx := conn.CloseRead(r.Context()) + + sub := e.eventBus.Subscribe() + defer sub.Close() + + for { + select { + case <-wsCtx.Done(): + return + case <-sub.Done(): + return + case ev := <-sub.Events(): + msg, err := conn.Writer(r.Context(), websocket.MessageText) + if err != nil { + return + } + if err := tmpl().ExecuteTemplate(msg, "event.html", ev); err != nil { + msg.Close() + return + } + if err := msg.Close(); err != nil { + return + } + } + } +} diff --git a/tstest/natlab/vnet/vnet.go b/tstest/natlab/vnet/vnet.go index 43256dafe..cb9922a58 100644 --- a/tstest/natlab/vnet/vnet.go +++ b/tstest/natlab/vnet/vnet.go @@ -750,6 +750,10 @@ type Server struct { cloudInitData map[int]*CloudInitData // node num → cloud-init config fileContents map[string][]byte // filename → file bytes + + // onDHCPEvent, if non-nil, is called when DHCP messages are processed. + // Parameters are: source MAC, node number, DHCP message type, assigned IP. + onDHCPEvent func(nodeMAC MAC, nodeNum int, msgType layers.DHCPMsgType, assignedIP netip.Addr) } func (s *Server) logf(format string, args ...any) { @@ -764,6 +768,13 @@ func (s *Server) SetLoggerForTest(logf func(format string, args ...any)) { s.optLogf = logf } +// SetDHCPCallback registers a function to be called when DHCP messages are +// processed. The callback receives the source MAC, node number, DHCP message +// type (Discover, Offer, Request, Ack), and the assigned IP address. +func (s *Server) SetDHCPCallback(fn func(MAC, int, layers.DHCPMsgType, netip.Addr)) { + s.onDHCPEvent = fn +} + var derpMap = &tailcfg.DERPMap{ Regions: map[int]*tailcfg.DERPRegion{ 1: { @@ -1804,6 +1815,10 @@ func (s *Server) createDHCPResponse(request gopacket.Packet) ([]byte, error) { Length: 4, }, ) + if s.onDHCPEvent != nil { + s.onDHCPEvent(srcMAC, node.num, layers.DHCPMsgTypeDiscover, clientIP) + s.onDHCPEvent(srcMAC, node.num, layers.DHCPMsgTypeOffer, clientIP) + } case layers.DHCPMsgTypeRequest: response.Options = append(response.Options, layers.DHCPOption{ @@ -1832,6 +1847,10 @@ func (s *Server) createDHCPResponse(request gopacket.Packet) ([]byte, error) { Length: 4, }, ) + if s.onDHCPEvent != nil { + s.onDHCPEvent(srcMAC, node.num, layers.DHCPMsgTypeRequest, clientIP) + s.onDHCPEvent(srcMAC, node.num, layers.DHCPMsgTypeAck, clientIP) + } } eth := &layers.Ethernet{ |