summaryrefslogtreecommitdiffhomepage
path: root/logtail/example/logreprocess
diff options
context:
space:
mode:
Diffstat (limited to 'logtail/example/logreprocess')
-rwxr-xr-xlogtail/example/logreprocess/demo.sh172
-rw-r--r--logtail/example/logreprocess/logreprocess.go230
2 files changed, 201 insertions, 201 deletions
diff --git a/logtail/example/logreprocess/demo.sh b/logtail/example/logreprocess/demo.sh
index 4ec819a67..eaec706a3 100755
--- a/logtail/example/logreprocess/demo.sh
+++ b/logtail/example/logreprocess/demo.sh
@@ -1,86 +1,86 @@
-#!/bin/bash
-# Copyright (c) Tailscale Inc & AUTHORS
-# SPDX-License-Identifier: BSD-3-Clause
-
-#
-# This shell script demonstrates writing logs from machines
-# and then reprocessing those logs to amalgamate python tracebacks
-# into a single log entry in a new collection.
-#
-# To run this demo, first install the example applications:
-#
-# go install tailscale.com/logtail/example/...
-#
-# Then generate a LOGTAIL_API_KEY and two test collections by visiting:
-#
-# https://log.tailscale.io
-#
-# Then set the three variables below.
-trap 'rv=$?; [ "$rv" = 0 ] || echo "-- exiting with code $rv"; exit $rv' EXIT
-set -e
-
-LOG_TEXT='server starting
-config file loaded
-answering queries
-Traceback (most recent call last):
- File "/Users/crawshaw/junk.py", line 6, in <module>
- main()
- File "/Users/crawshaw/junk.py", line 4, in main
- raise Exception("oops")
-Exception: oops'
-
-die() {
- echo "$0: $*" >&2
- exit 1
-}
-
-msg() {
- echo "-- $*" >&2
-}
-
-if [ -z "$LOGTAIL_API_KEY" ]; then
- die "LOGTAIL_API_KEY is not set"
-fi
-
-if [ -z "$COLLECTION_IN" ]; then
- die "COLLECTION_IN is not set"
-fi
-
-if [ -z "$COLLECTION_OUT" ]; then
- die "COLLECTION_OUT is not set"
-fi
-
-# Private IDs are 32-bytes of random hex.
-# Normally you'd keep the same private IDs from one run to the next, but
-# this is just an example.
-msg "Generating keys..."
-privateid1=$(hexdump -n 32 -e '8/4 "%08X"' /dev/urandom)
-privateid2=$(hexdump -n 32 -e '8/4 "%08X"' /dev/urandom)
-privateid3=$(hexdump -n 32 -e '8/4 "%08X"' /dev/urandom)
-
-# Public IDs are the SHA-256 of the private ID.
-publicid1=$(echo -n $privateid1 | xxd -r -p - | shasum -a 256 | sed 's/ -//')
-publicid2=$(echo -n $privateid2 | xxd -r -p - | shasum -a 256 | sed 's/ -//')
-publicid3=$(echo -n $privateid3 | xxd -r -p - | shasum -a 256 | sed 's/ -//')
-
-# Write the machine logs to the input collection.
-# Notice that this doesn't require an API key.
-msg "Producing new logs..."
-echo "$LOG_TEXT" | logtail -c $COLLECTION_IN -k $privateid1 >/dev/null
-echo "$LOG_TEXT" | logtail -c $COLLECTION_IN -k $privateid2 >/dev/null
-
-# Adopt the logs, so they will be kept and are readable.
-msg "Adopting logs..."
-logadopt -p "$LOGTAIL_API_KEY" -c "$COLLECTION_IN" -m $publicid1
-logadopt -p "$LOGTAIL_API_KEY" -c "$COLLECTION_IN" -m $publicid2
-
-# Reprocess the logs, amalgamating python tracebacks.
-#
-# We'll take that reprocessed output and write it to a separate collection,
-# again via logtail.
-#
-# Time out quickly because all our "interesting" logs (generated
-# above) have already been processed.
-msg "Reprocessing logs..."
-logreprocess -t 3s -c "$COLLECTION_IN" -p "$LOGTAIL_API_KEY" 2>&1 |
- logtail -c "$COLLECTION_OUT" -k $privateid3
+#!/bin/bash
+# Copyright (c) Tailscale Inc & AUTHORS
+# SPDX-License-Identifier: BSD-3-Clause
+
+#
+# This shell script demonstrates writing logs from machines
+# and then reprocessing those logs to amalgamate python tracebacks
+# into a single log entry in a new collection.
+#
+# To run this demo, first install the example applications:
+#
+# go install tailscale.com/logtail/example/...
+#
+# Then generate a LOGTAIL_API_KEY and two test collections by visiting:
+#
+# https://log.tailscale.io
+#
+# Then set the three variables below.
+trap 'rv=$?; [ "$rv" = 0 ] || echo "-- exiting with code $rv"; exit $rv' EXIT
+set -e
+
+LOG_TEXT='server starting
+config file loaded
+answering queries
+Traceback (most recent call last):
+ File "/Users/crawshaw/junk.py", line 6, in <module>
+ main()
+ File "/Users/crawshaw/junk.py", line 4, in main
+ raise Exception("oops")
+Exception: oops'
+
+die() {
+ echo "$0: $*" >&2
+ exit 1
+}
+
+msg() {
+ echo "-- $*" >&2
+}
+
+if [ -z "$LOGTAIL_API_KEY" ]; then
+ die "LOGTAIL_API_KEY is not set"
+fi
+
+if [ -z "$COLLECTION_IN" ]; then
+ die "COLLECTION_IN is not set"
+fi
+
+if [ -z "$COLLECTION_OUT" ]; then
+ die "COLLECTION_OUT is not set"
+fi
+
+# Private IDs are 32-bytes of random hex.
+# Normally you'd keep the same private IDs from one run to the next, but
+# this is just an example.
+msg "Generating keys..."
+privateid1=$(hexdump -n 32 -e '8/4 "%08X"' /dev/urandom)
+privateid2=$(hexdump -n 32 -e '8/4 "%08X"' /dev/urandom)
+privateid3=$(hexdump -n 32 -e '8/4 "%08X"' /dev/urandom)
+
+# Public IDs are the SHA-256 of the private ID.
+publicid1=$(echo -n $privateid1 | xxd -r -p - | shasum -a 256 | sed 's/ -//')
+publicid2=$(echo -n $privateid2 | xxd -r -p - | shasum -a 256 | sed 's/ -//')
+publicid3=$(echo -n $privateid3 | xxd -r -p - | shasum -a 256 | sed 's/ -//')
+
+# Write the machine logs to the input collection.
+# Notice that this doesn't require an API key.
+msg "Producing new logs..."
+echo "$LOG_TEXT" | logtail -c $COLLECTION_IN -k $privateid1 >/dev/null
+echo "$LOG_TEXT" | logtail -c $COLLECTION_IN -k $privateid2 >/dev/null
+
+# Adopt the logs, so they will be kept and are readable.
+msg "Adopting logs..."
+logadopt -p "$LOGTAIL_API_KEY" -c "$COLLECTION_IN" -m $publicid1
+logadopt -p "$LOGTAIL_API_KEY" -c "$COLLECTION_IN" -m $publicid2
+
+# Reprocess the logs, amalgamating python tracebacks.
+#
+# We'll take that reprocessed output and write it to a separate collection,
+# again via logtail.
+#
+# Time out quickly because all our "interesting" logs (generated
+# above) have already been processed.
+msg "Reprocessing logs..."
+logreprocess -t 3s -c "$COLLECTION_IN" -p "$LOGTAIL_API_KEY" 2>&1 |
+ logtail -c "$COLLECTION_OUT" -k $privateid3
diff --git a/logtail/example/logreprocess/logreprocess.go b/logtail/example/logreprocess/logreprocess.go
index 5dbf76578..e88d5b485 100644
--- a/logtail/example/logreprocess/logreprocess.go
+++ b/logtail/example/logreprocess/logreprocess.go
@@ -1,115 +1,115 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-// The logreprocess program tails a log and reprocesses it.
-package main
-
-import (
- "bufio"
- "encoding/json"
- "flag"
- "io"
- "log"
- "net/http"
- "os"
- "strings"
- "time"
-
- "tailscale.com/types/logid"
-)
-
-func main() {
- collection := flag.String("c", "", "logtail collection name to read")
- apiKey := flag.String("p", "", "logtail API key")
- timeout := flag.Duration("t", 0, "timeout after which logreprocess quits")
- flag.Parse()
- if len(flag.Args()) != 0 {
- flag.Usage()
- os.Exit(1)
- }
- log.SetFlags(0)
-
- if *timeout != 0 {
- go func() {
- <-time.After(*timeout)
- log.Printf("logreprocess: timeout reached, quitting")
- os.Exit(1)
- }()
- }
-
- req, err := http.NewRequest("GET", "https://log.tailscale.io/c/"+*collection+"?stream=true", nil)
- if err != nil {
- log.Fatal(err)
- }
- req.SetBasicAuth(*apiKey, "")
- resp, err := http.DefaultClient.Do(req)
- if err != nil {
- log.Fatal(err)
- }
- defer resp.Body.Close()
-
- if resp.StatusCode != 200 {
- b, err := io.ReadAll(resp.Body)
- if err != nil {
- log.Fatalf("logreprocess: read error %d: %v", resp.StatusCode, err)
- }
- log.Fatalf("logreprocess: read error %d: %s", resp.StatusCode, string(b))
- }
-
- tracebackCache := make(map[logid.PublicID]*ProcessedMsg)
-
- scanner := bufio.NewScanner(resp.Body)
- for scanner.Scan() {
- var msg Msg
- if err := json.Unmarshal(scanner.Bytes(), &msg); err != nil {
- log.Fatalf("logreprocess of %q: %v", string(scanner.Bytes()), err)
- }
- var pMsg *ProcessedMsg
- if pMsg = tracebackCache[msg.Logtail.Instance]; pMsg != nil {
- pMsg.Text += "\n" + msg.Text
- if strings.HasPrefix(msg.Text, "Exception: ") {
- delete(tracebackCache, msg.Logtail.Instance)
- } else {
- continue // write later
- }
- } else {
- pMsg = &ProcessedMsg{
- OrigInstance: msg.Logtail.Instance,
- Text: msg.Text,
- }
- pMsg.Logtail.ClientTime = msg.Logtail.ClientTime
- }
-
- if strings.HasPrefix(msg.Text, "Traceback (most recent call last):") {
- tracebackCache[msg.Logtail.Instance] = pMsg
- continue // write later
- }
-
- b, err := json.Marshal(pMsg)
- if err != nil {
- log.Fatal(err)
- }
- log.Printf("%s", b)
- }
- if err := scanner.Err(); err != nil {
- log.Fatal(err)
- }
-}
-
-type Msg struct {
- Logtail struct {
- Instance logid.PublicID `json:"instance"`
- ClientTime time.Time `json:"client_time"`
- } `json:"logtail"`
-
- Text string `json:"text"`
-}
-
-type ProcessedMsg struct {
- Logtail struct {
- ClientTime time.Time `json:"client_time"`
- } `json:"logtail"`
-
- OrigInstance logid.PublicID `json:"orig_instance"`
- Text string `json:"text"`
-}
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// The logreprocess program tails a log and reprocesses it.
+package main
+
+import (
+ "bufio"
+ "encoding/json"
+ "flag"
+ "io"
+ "log"
+ "net/http"
+ "os"
+ "strings"
+ "time"
+
+ "tailscale.com/types/logid"
+)
+
+func main() {
+ collection := flag.String("c", "", "logtail collection name to read")
+ apiKey := flag.String("p", "", "logtail API key")
+ timeout := flag.Duration("t", 0, "timeout after which logreprocess quits")
+ flag.Parse()
+ if len(flag.Args()) != 0 {
+ flag.Usage()
+ os.Exit(1)
+ }
+ log.SetFlags(0)
+
+ if *timeout != 0 {
+ go func() {
+ <-time.After(*timeout)
+ log.Printf("logreprocess: timeout reached, quitting")
+ os.Exit(1)
+ }()
+ }
+
+ req, err := http.NewRequest("GET", "https://log.tailscale.io/c/"+*collection+"?stream=true", nil)
+ if err != nil {
+ log.Fatal(err)
+ }
+ req.SetBasicAuth(*apiKey, "")
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ log.Fatal(err)
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode != 200 {
+ b, err := io.ReadAll(resp.Body)
+ if err != nil {
+ log.Fatalf("logreprocess: read error %d: %v", resp.StatusCode, err)
+ }
+ log.Fatalf("logreprocess: read error %d: %s", resp.StatusCode, string(b))
+ }
+
+ tracebackCache := make(map[logid.PublicID]*ProcessedMsg)
+
+ scanner := bufio.NewScanner(resp.Body)
+ for scanner.Scan() {
+ var msg Msg
+ if err := json.Unmarshal(scanner.Bytes(), &msg); err != nil {
+ log.Fatalf("logreprocess of %q: %v", string(scanner.Bytes()), err)
+ }
+ var pMsg *ProcessedMsg
+ if pMsg = tracebackCache[msg.Logtail.Instance]; pMsg != nil {
+ pMsg.Text += "\n" + msg.Text
+ if strings.HasPrefix(msg.Text, "Exception: ") {
+ delete(tracebackCache, msg.Logtail.Instance)
+ } else {
+ continue // write later
+ }
+ } else {
+ pMsg = &ProcessedMsg{
+ OrigInstance: msg.Logtail.Instance,
+ Text: msg.Text,
+ }
+ pMsg.Logtail.ClientTime = msg.Logtail.ClientTime
+ }
+
+ if strings.HasPrefix(msg.Text, "Traceback (most recent call last):") {
+ tracebackCache[msg.Logtail.Instance] = pMsg
+ continue // write later
+ }
+
+ b, err := json.Marshal(pMsg)
+ if err != nil {
+ log.Fatal(err)
+ }
+ log.Printf("%s", b)
+ }
+ if err := scanner.Err(); err != nil {
+ log.Fatal(err)
+ }
+}
+
+type Msg struct {
+ Logtail struct {
+ Instance logid.PublicID `json:"instance"`
+ ClientTime time.Time `json:"client_time"`
+ } `json:"logtail"`
+
+ Text string `json:"text"`
+}
+
+type ProcessedMsg struct {
+ Logtail struct {
+ ClientTime time.Time `json:"client_time"`
+ } `json:"logtail"`
+
+ OrigInstance logid.PublicID `json:"orig_instance"`
+ Text string `json:"text"`
+}
Copyright © 2025 - 2026 Wayne Cole. WTFPL. Attribution appreciated. No warranty. Not liable for bodily damages.