1 files changed, 28 insertions, 2 deletions
diff --git a/cmd/testwrapper/testwrapper.go b/cmd/testwrapper/testwrapper.go
index 1df1ef11f..53c1b1d05 100644
--- a/cmd/testwrapper/testwrapper.go
+++ b/cmd/testwrapper/testwrapper.go
@@ -141,7 +141,7 @@ func runTests(ctx context.Context, attempt int, pt *packageTests, goTestArgs, te
 				}
 				outcome := goOutput.Action
 				if outcome == "build-fail" {
-					outcome = "FAIL"
+					outcome = "fail"
 				}
 				pkgTests[""].logs.WriteString(goOutput.Output)
 				ch <- &testAttempt{
@@ -152,7 +152,15 @@ func runTests(ctx context.Context, attempt int, pt *packageTests, goTestArgs, te
 					logs:        pkgTests[""].logs,
 					pkgFinished: true,
 				}
+			case "output":
+				// Capture all output from the package except for the final
+				// "FAIL    tailscale.io/control    0.684s" line, as
+				// printPkgOutcome will output a similar line
+				if !strings.HasPrefix(goOutput.Output, fmt.Sprintf("FAIL\t%s\t", goOutput.Package)) {
+					pkgTests[""].logs.WriteString(goOutput.Output)
+				}
 			}
+
 			continue
 		}
 		testName := goOutput.Test
@@ -251,6 +259,7 @@ func main() {
 			fmt.Printf("\n\nAttempt #%d: Retrying flaky tests:\n\nflakytest failures JSON: %s\n\n", thisRun.attempt, j)
 		}
 
+		fatalFailures := make(map[string]struct{}) // pkg.Test key
 		toRetry := make(map[string][]*testAttempt) // pkg -> tests to retry
 		for _, pt := range thisRun.tests {
 			ch := make(chan *testAttempt)
@@ -276,7 +285,11 @@ func main() {
 						// when a package times out.
 						failed = true
 					}
-					os.Stdout.ReadFrom(&tr.logs)
+					if testingVerbose || tr.outcome == "fail" {
+						// Output package-level output which is where e.g.
+						// panics outside tests will be printed
+						io.Copy(os.Stdout, &tr.logs)
+					}
 					printPkgOutcome(tr.pkg, tr.outcome, thisRun.attempt, tr.end.Sub(tr.start))
 					continue
 				}
@@ -289,11 +302,24 @@ func main() {
 				if tr.isMarkedFlaky {
 					toRetry[tr.pkg] = append(toRetry[tr.pkg], tr)
 				} else {
+					fatalFailures[tr.pkg+"."+tr.testName] = struct{}{}
 					failed = true
 				}
 			}
 			if failed {
 				fmt.Println("\n\nNot retrying flaky tests because non-flaky tests failed.")
+
+				// Print the list of non-flakytest failures.
+				// We will later analyze the retried GitHub Action runs to see
+				// if non-flakytest failures succeeded upon retry. This will
+				// highlight tests which are flaky but not yet flagged as such.
+				if len(fatalFailures) > 0 {
+					tests := slicesx.MapKeys(fatalFailures)
+					sort.Strings(tests)
+					j, _ := json.Marshal(tests)
+					fmt.Printf("non-flakytest failures: %s\n", j)
+				}
+				fmt.Println()
 				os.Exit(1)
 			}