diff options
| author | Nick Khyl <nickk@tailscale.com> | 2024-12-05 13:16:48 -0600 |
|---|---|---|
| committer | Nick Khyl <nickk@tailscale.com> | 2024-12-05 13:16:48 -0600 |
| commit | 0267fe83b200f1702a2fa0a395442c02a053fadb (patch) | |
| tree | 63654c55225eeb834de59a5a0bc8d19033c6145b /smallzstd/zstd.go | |
| parent | 87546a5edf6b6503a87eeb2d666baba57398a066 (diff) | |
| download | tailscale-1.78.0.tar.xz tailscale-1.78.0.zip | |
VERSION.txt: this is v1.78.0v1.78.0
Signed-off-by: Nick Khyl <nickk@tailscale.com>
Diffstat (limited to 'smallzstd/zstd.go')
| -rw-r--r-- | smallzstd/zstd.go | 156 |
1 files changed, 78 insertions, 78 deletions
diff --git a/smallzstd/zstd.go b/smallzstd/zstd.go index 1d8085422..d91afeb67 100644 --- a/smallzstd/zstd.go +++ b/smallzstd/zstd.go @@ -1,78 +1,78 @@ -// Copyright (c) Tailscale Inc & AUTHORS -// SPDX-License-Identifier: BSD-3-Clause - -// Package smallzstd produces zstd encoders and decoders optimized for -// low memory usage, at the expense of compression efficiency. -// -// This package is optimized primarily for the memory cost of -// compressing and decompressing data. We reduce this cost in two -// major ways: disable parallelism within the library (i.e. don't use -// multiple CPU cores to decompress), and drop the compression window -// down from the defaults of 4-16MiB, to 8kiB. -// -// Decompressors cost 2x the window size in RAM to run, so by using an -// 8kiB window, we can run ~1000 more decompressors per unit of memory -// than with the defaults. -// -// Depending on context, the benefit is either being able to run more -// decoders (e.g. in our logs processing system), or having a lower -// memory footprint when using compression in network protocols -// (e.g. in tailscaled, which should have a minimal RAM cost). -package smallzstd - -import ( - "io" - - "github.com/klauspost/compress/zstd" -) - -// WindowSize is the window size used for zstd compression. Decoder -// memory usage scales linearly with WindowSize. -const WindowSize = 8 << 10 // 8kiB - -// NewDecoder returns a zstd.Decoder configured for low memory usage, -// at the expense of decompression performance. -func NewDecoder(r io.Reader, options ...zstd.DOption) (*zstd.Decoder, error) { - defaults := []zstd.DOption{ - // Default is GOMAXPROCS, which costs many KiB in stacks. - zstd.WithDecoderConcurrency(1), - // Default is to allocate more upfront for performance. We - // prefer lower memory use and a bit of GC load. - zstd.WithDecoderLowmem(true), - // You might expect to see zstd.WithDecoderMaxMemory - // here. However, it's not terribly safe to use if you're - // doing stateless decoding, because it sets the maximum - // amount of memory the decompressed data can occupy, rather - // than the window size of the zstd stream. This means a very - // compressible piece of data might violate the max memory - // limit here, even if the window size (and thus total memory - // required to decompress the data) is small. - // - // As a result, we don't set a decoder limit here, and rely on - // the encoder below producing "cheap" streams. Callers are - // welcome to set their own max memory setting, if - // contextually there is a clearly correct value (e.g. it's - // known from the upper layer protocol that the decoded data - // can never be more than 1MiB). - } - - return zstd.NewReader(r, append(defaults, options...)...) -} - -// NewEncoder returns a zstd.Encoder configured for low memory usage, -// both during compression and at decompression time, at the expense -// of performance and compression efficiency. -func NewEncoder(w io.Writer, options ...zstd.EOption) (*zstd.Encoder, error) { - defaults := []zstd.EOption{ - // Default is GOMAXPROCS, which costs many KiB in stacks. - zstd.WithEncoderConcurrency(1), - // Default is several MiB, which bloats both encoders and - // their corresponding decoders. - zstd.WithWindowSize(WindowSize), - // Encode zero-length inputs in a way that the `zstd` utility - // can read, because interoperability is handy. - zstd.WithZeroFrames(true), - } - - return zstd.NewWriter(w, append(defaults, options...)...) -} +// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// Package smallzstd produces zstd encoders and decoders optimized for
+// low memory usage, at the expense of compression efficiency.
+//
+// This package is optimized primarily for the memory cost of
+// compressing and decompressing data. We reduce this cost in two
+// major ways: disable parallelism within the library (i.e. don't use
+// multiple CPU cores to decompress), and drop the compression window
+// down from the defaults of 4-16MiB, to 8kiB.
+//
+// Decompressors cost 2x the window size in RAM to run, so by using an
+// 8kiB window, we can run ~1000 more decompressors per unit of memory
+// than with the defaults.
+//
+// Depending on context, the benefit is either being able to run more
+// decoders (e.g. in our logs processing system), or having a lower
+// memory footprint when using compression in network protocols
+// (e.g. in tailscaled, which should have a minimal RAM cost).
+package smallzstd
+
+import (
+ "io"
+
+ "github.com/klauspost/compress/zstd"
+)
+
+// WindowSize is the window size used for zstd compression. Decoder
+// memory usage scales linearly with WindowSize.
+const WindowSize = 8 << 10 // 8kiB
+
+// NewDecoder returns a zstd.Decoder configured for low memory usage,
+// at the expense of decompression performance.
+func NewDecoder(r io.Reader, options ...zstd.DOption) (*zstd.Decoder, error) {
+ defaults := []zstd.DOption{
+ // Default is GOMAXPROCS, which costs many KiB in stacks.
+ zstd.WithDecoderConcurrency(1),
+ // Default is to allocate more upfront for performance. We
+ // prefer lower memory use and a bit of GC load.
+ zstd.WithDecoderLowmem(true),
+ // You might expect to see zstd.WithDecoderMaxMemory
+ // here. However, it's not terribly safe to use if you're
+ // doing stateless decoding, because it sets the maximum
+ // amount of memory the decompressed data can occupy, rather
+ // than the window size of the zstd stream. This means a very
+ // compressible piece of data might violate the max memory
+ // limit here, even if the window size (and thus total memory
+ // required to decompress the data) is small.
+ //
+ // As a result, we don't set a decoder limit here, and rely on
+ // the encoder below producing "cheap" streams. Callers are
+ // welcome to set their own max memory setting, if
+ // contextually there is a clearly correct value (e.g. it's
+ // known from the upper layer protocol that the decoded data
+ // can never be more than 1MiB).
+ }
+
+ return zstd.NewReader(r, append(defaults, options...)...)
+}
+
+// NewEncoder returns a zstd.Encoder configured for low memory usage,
+// both during compression and at decompression time, at the expense
+// of performance and compression efficiency.
+func NewEncoder(w io.Writer, options ...zstd.EOption) (*zstd.Encoder, error) {
+ defaults := []zstd.EOption{
+ // Default is GOMAXPROCS, which costs many KiB in stacks.
+ zstd.WithEncoderConcurrency(1),
+ // Default is several MiB, which bloats both encoders and
+ // their corresponding decoders.
+ zstd.WithWindowSize(WindowSize),
+ // Encode zero-length inputs in a way that the `zstd` utility
+ // can read, because interoperability is handy.
+ zstd.WithZeroFrames(true),
+ }
+
+ return zstd.NewWriter(w, append(defaults, options...)...)
+}
|
