diff --git a/cmd/go-getter/go.sum b/cmd/go-getter/go.sum index 83d5a06a6..d341d736f 100644 --- a/cmd/go-getter/go.sum +++ b/cmd/go-getter/go.sum @@ -437,6 +437,8 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfC github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.11.2 h1:MiK62aErc3gIiVEtyzKfeOHgW7atJb5g/KNX5m3c2nQ= +github.com/klauspost/compress v1.11.2/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= diff --git a/decompress.go b/decompress.go index 17d8b7c63..eb8cdc7aa 100644 --- a/decompress.go +++ b/decompress.go @@ -27,6 +27,7 @@ func init() { tbzDecompressor := new(TarBzip2Decompressor) tgzDecompressor := new(TarGzipDecompressor) txzDecompressor := new(TarXzDecompressor) + tzstDecompressor := new(TarZstdDecompressor) Decompressors = map[string]Decompressor{ "bz2": new(Bzip2Decompressor), @@ -35,11 +36,14 @@ func init() { "tar.bz2": tbzDecompressor, "tar.gz": tgzDecompressor, "tar.xz": txzDecompressor, + "tar.zst": tzstDecompressor, "tbz2": tbzDecompressor, "tgz": tgzDecompressor, "txz": txzDecompressor, + "tzst": tzstDecompressor, "zip": new(ZipDecompressor), "tar": tarDecompressor, + "zst": new(ZstdDecompressor), } } diff --git a/decompress_bzip2.go b/decompress_bzip2.go index a5373e4e1..6db0b3577 100644 --- a/decompress_bzip2.go +++ b/decompress_bzip2.go @@ -9,7 +9,12 @@ import ( // Bzip2Decompressor is an implementation of Decompressor that can // decompress bz2 files. -type Bzip2Decompressor struct{} +type Bzip2Decompressor struct { + // FileSizeLimit limits the size of a decompressed file. + // + // The zero value means no limit. + FileSizeLimit int64 +} func (d *Bzip2Decompressor) Decompress(dst, src string, dir bool, umask os.FileMode) error { // Directory isn't supported at all @@ -33,5 +38,5 @@ func (d *Bzip2Decompressor) Decompress(dst, src string, dir bool, umask os.FileM bzipR := bzip2.NewReader(f) // Copy it out - return copyReader(dst, bzipR, 0622, umask) + return copyReader(dst, bzipR, 0622, umask, d.FileSizeLimit) } diff --git a/decompress_gzip.go b/decompress_gzip.go index 669e5eafd..f94f2bcff 100644 --- a/decompress_gzip.go +++ b/decompress_gzip.go @@ -9,7 +9,12 @@ import ( // GzipDecompressor is an implementation of Decompressor that can // decompress gzip files. -type GzipDecompressor struct{} +type GzipDecompressor struct { + // FileSizeLimit limits the size of a decompressed file. + // + // The zero value means no limit. + FileSizeLimit int64 +} func (d *GzipDecompressor) Decompress(dst, src string, dir bool, umask os.FileMode) error { // Directory isn't supported at all @@ -37,5 +42,5 @@ func (d *GzipDecompressor) Decompress(dst, src string, dir bool, umask os.FileMo defer gzipR.Close() // Copy it out - return copyReader(dst, gzipR, 0622, umask) + return copyReader(dst, gzipR, 0622, umask, d.FileSizeLimit) } diff --git a/decompress_tar.go b/decompress_tar.go index ee1d29ca9..b5188c0e5 100644 --- a/decompress_tar.go +++ b/decompress_tar.go @@ -11,12 +11,25 @@ import ( // untar is a shared helper for untarring an archive. The reader should provide // an uncompressed view of the tar archive. -func untar(input io.Reader, dst, src string, dir bool, umask os.FileMode) error { +func untar(input io.Reader, dst, src string, dir bool, umask os.FileMode, fileSizeLimit int64, filesLimit int) error { tarR := tar.NewReader(input) done := false dirHdrs := []*tar.Header{} now := time.Now() + + var ( + fileSize int64 + filesCount int + ) + for { + if filesLimit > 0 { + filesCount++ + if filesCount > filesLimit { + return fmt.Errorf("tar archive contains too many files: %d > %d", filesCount, filesLimit) + } + } + hdr, err := tarR.Next() if err == io.EOF { if !done { @@ -45,7 +58,15 @@ func untar(input io.Reader, dst, src string, dir bool, umask os.FileMode) error path = filepath.Join(path, hdr.Name) } - if hdr.FileInfo().IsDir() { + fileInfo := hdr.FileInfo() + + fileSize += fileInfo.Size() + + if fileSizeLimit > 0 && fileSize > fileSizeLimit { + return fmt.Errorf("tar archive larger than limit: %d", fileSizeLimit) + } + + if fileInfo.IsDir() { if !dir { return fmt.Errorf("expected a single file: %s", src) } @@ -81,8 +102,8 @@ func untar(input io.Reader, dst, src string, dir bool, umask os.FileMode) error // Mark that we're done so future in single file mode errors done = true - // Open the file for writing - err = copyReader(path, tarR, hdr.FileInfo().Mode(), umask) + // Size limit is tracked using the returned file info. + err = copyReader(path, tarR, hdr.FileInfo().Mode(), umask, 0) if err != nil { return err } @@ -127,7 +148,19 @@ func untar(input io.Reader, dst, src string, dir bool, umask os.FileMode) error // TarDecompressor is an implementation of Decompressor that can // unpack tar files. -type TarDecompressor struct{} +type TarDecompressor struct { + // FileSizeLimit limits the total size of all + // decompressed files. + // + // The zero value means no limit. + FileSizeLimit int64 + + // FilesLimit limits the number of files that are + // allowed to be decompressed. + // + // The zero value means no limit. + FilesLimit int +} func (d *TarDecompressor) Decompress(dst, src string, dir bool, umask os.FileMode) error { // If we're going into a directory we should make that first @@ -146,5 +179,5 @@ func (d *TarDecompressor) Decompress(dst, src string, dir bool, umask os.FileMod } defer f.Close() - return untar(f, dst, src, dir, umask) + return untar(f, dst, src, dir, umask, d.FileSizeLimit, d.FilesLimit) } diff --git a/decompress_tar_test.go b/decompress_tar_test.go index cfc842019..39d8f5a71 100644 --- a/decompress_tar_test.go +++ b/decompress_tar_test.go @@ -1,10 +1,13 @@ package getter import ( + "archive/tar" + "bytes" "io/ioutil" "os" "path/filepath" "runtime" + "strings" "testing" "time" ) @@ -45,6 +48,86 @@ func TestTar(t *testing.T) { TestDecompressor(t, new(TarDecompressor), cases) } +func TestTarLimits(t *testing.T) { + b := bytes.NewBuffer(nil) + + tw := tar.NewWriter(b) + + var files = []struct { + Name, Body string + }{ + {"readme.txt", "This archive contains some text files."}, + {"gopher.txt", "Gopher names:\nCharlie\nRonald\nGlenn"}, + {"todo.txt", "Get animal handling license."}, + } + + for _, file := range files { + hdr := &tar.Header{ + Name: file.Name, + Mode: 0600, + Size: int64(len(file.Body)), + } + if err := tw.WriteHeader(hdr); err != nil { + t.Fatal(err) + } + if _, err := tw.Write([]byte(file.Body)); err != nil { + t.Fatal(err) + } + } + + if err := tw.Close(); err != nil { + t.Fatal(err) + } + + td, err := ioutil.TempDir("", "getter") + if err != nil { + t.Fatalf("err: %s", err) + } + + tarFilePath := filepath.Join(td, "input.tar") + + err = ioutil.WriteFile(tarFilePath, b.Bytes(), 0666) + if err != nil { + t.Fatalf("err: %s", err) + } + + t.Run("file size limit", func(t *testing.T) { + d := new(TarDecompressor) + + d.FileSizeLimit = 35 + + dst := filepath.Join(td, "subdir", "file-size-limit-result") + + err = d.Decompress(dst, tarFilePath, true, 0022) + + if err == nil { + t.Fatal("expected file size limit to error") + } + + if !strings.Contains(err.Error(), "tar archive larger than limit: 35") { + t.Fatalf("unexpected error: %q", err.Error()) + } + }) + + t.Run("files limit", func(t *testing.T) { + d := new(TarDecompressor) + + d.FilesLimit = 2 + + dst := filepath.Join(td, "subdir", "files-limit-result") + + err = d.Decompress(dst, tarFilePath, true, 0022) + + if err == nil { + t.Fatal("expected files limit to error") + } + + if !strings.Contains(err.Error(), "tar archive contains too many files: 3 > 2") { + t.Fatalf("unexpected error: %q", err.Error()) + } + }) +} + // testDecompressPermissions decompresses a directory and checks the permissions of the expanded files func testDecompressorPermissions(t *testing.T, d Decompressor, input string, expected map[string]int, umask os.FileMode) { td, err := ioutil.TempDir("", "getter") diff --git a/decompress_tbz2.go b/decompress_tbz2.go index e2e5458c9..78609c9ff 100644 --- a/decompress_tbz2.go +++ b/decompress_tbz2.go @@ -8,7 +8,19 @@ import ( // TarBzip2Decompressor is an implementation of Decompressor that can // decompress tar.bz2 files. -type TarBzip2Decompressor struct{} +type TarBzip2Decompressor struct { + // FileSizeLimit limits the total size of all + // decompressed files. + // + // The zero value means no limit. + FileSizeLimit int64 + + // FilesLimit limits the number of files that are + // allowed to be decompressed. + // + // The zero value means no limit. + FilesLimit int +} func (d *TarBzip2Decompressor) Decompress(dst, src string, dir bool, umask os.FileMode) error { // If we're going into a directory we should make that first @@ -29,5 +41,5 @@ func (d *TarBzip2Decompressor) Decompress(dst, src string, dir bool, umask os.Fi // Bzip2 compression is second bzipR := bzip2.NewReader(f) - return untar(bzipR, dst, src, dir, umask) + return untar(bzipR, dst, src, dir, umask, d.FileSizeLimit, d.FilesLimit) } diff --git a/decompress_tgz.go b/decompress_tgz.go index 84c4aa33d..848f5e372 100644 --- a/decompress_tgz.go +++ b/decompress_tgz.go @@ -9,7 +9,19 @@ import ( // TarGzipDecompressor is an implementation of Decompressor that can // decompress tar.gzip files. -type TarGzipDecompressor struct{} +type TarGzipDecompressor struct { + // FileSizeLimit limits the total size of all + // decompressed files. + // + // The zero value means no limit. + FileSizeLimit int64 + + // FilesLimit limits the number of files that are + // allowed to be decompressed. + // + // The zero value means no limit. + FilesLimit int +} func (d *TarGzipDecompressor) Decompress(dst, src string, dir bool, umask os.FileMode) error { // If we're going into a directory we should make that first @@ -35,5 +47,5 @@ func (d *TarGzipDecompressor) Decompress(dst, src string, dir bool, umask os.Fil } defer gzipR.Close() - return untar(gzipR, dst, src, dir, umask) + return untar(gzipR, dst, src, dir, umask, d.FileSizeLimit, d.FilesLimit) } diff --git a/decompress_txz.go b/decompress_txz.go index 24686f454..42f6179a8 100644 --- a/decompress_txz.go +++ b/decompress_txz.go @@ -10,7 +10,19 @@ import ( // TarXzDecompressor is an implementation of Decompressor that can // decompress tar.xz files. -type TarXzDecompressor struct{} +type TarXzDecompressor struct { + // FileSizeLimit limits the total size of all + // decompressed files. + // + // The zero value means no limit. + FileSizeLimit int64 + + // FilesLimit limits the number of files that are + // allowed to be decompressed. + // + // The zero value means no limit. + FilesLimit int +} func (d *TarXzDecompressor) Decompress(dst, src string, dir bool, umask os.FileMode) error { // If we're going into a directory we should make that first @@ -35,5 +47,5 @@ func (d *TarXzDecompressor) Decompress(dst, src string, dir bool, umask os.FileM return fmt.Errorf("Error opening an xz reader for %s: %s", src, err) } - return untar(txzR, dst, src, dir, umask) + return untar(txzR, dst, src, dir, umask, d.FileSizeLimit, d.FilesLimit) } diff --git a/decompress_tzst.go b/decompress_tzst.go new file mode 100644 index 000000000..3b086ced2 --- /dev/null +++ b/decompress_tzst.go @@ -0,0 +1,52 @@ +package getter + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/klauspost/compress/zstd" +) + +// TarZstdDecompressor is an implementation of Decompressor that can +// decompress tar.zstd files. +type TarZstdDecompressor struct { + // FileSizeLimit limits the total size of all + // decompressed files. + // + // The zero value means no limit. + FileSizeLimit int64 + + // FilesLimit limits the number of files that are + // allowed to be decompressed. + // + // The zero value means no limit. + FilesLimit int +} + +func (d *TarZstdDecompressor) Decompress(dst, src string, dir bool, umask os.FileMode) error { + // If we're going into a directory we should make that first + mkdir := dst + if !dir { + mkdir = filepath.Dir(dst) + } + if err := os.MkdirAll(mkdir, mode(0755, umask)); err != nil { + return err + } + + // File first + f, err := os.Open(src) + if err != nil { + return err + } + defer f.Close() + + // Zstd compression is second + zstdR, err := zstd.NewReader(f) + if err != nil { + return fmt.Errorf("Error opening a zstd reader for %s: %s", src, err) + } + defer zstdR.Close() + + return untar(zstdR, dst, src, dir, umask, d.FileSizeLimit, d.FilesLimit) +} diff --git a/decompress_tzst_test.go b/decompress_tzst_test.go new file mode 100644 index 000000000..9787c7267 --- /dev/null +++ b/decompress_tzst_test.go @@ -0,0 +1,95 @@ +package getter + +import ( + "path/filepath" + "testing" +) + +func TestTarZstdDecompressor(t *testing.T) { + + multiplePaths := []string{"dir/", "dir/test2", "test1"} + orderingPaths := []string{"workers/", "workers/mq/", "workers/mq/__init__.py"} + + cases := []TestDecompressCase{ + { + "empty.tar.zst", + false, + true, + nil, + "", + nil, + }, + + { + "single.tar.zst", + false, + false, + nil, + "d3b07384d113edec49eaa6238ad5ff00", + nil, + }, + + { + "single.tar.zst", + true, + false, + []string{"file"}, + "", + nil, + }, + + { + "multiple.tar.zst", + true, + false, + []string{"file1", "file2"}, + "", + nil, + }, + + { + "multiple.tar.zst", + false, + true, + nil, + "", + nil, + }, + + { + "multiple_dir.tar.zst", + true, + false, + multiplePaths, + "", + nil, + }, + + // Tests when the file is listed before the parent folder + { + "ordering.tar.zst", + true, + false, + orderingPaths, + "", + nil, + }, + + // Tests that a tar.zst can't contain references with "..". + // GNU `tar` also disallows this. + { + "outside_parent.tar.zst", + true, + true, + nil, + "", + nil, + }, + } + + for i, tc := range cases { + cases[i].Input = filepath.Join("./testdata", "decompress-tzst", tc.Input) + } + + TestDecompressor(t, new(TarZstdDecompressor), cases) +} diff --git a/decompress_xz.go b/decompress_xz.go index de5d6ce2b..89fafd6bc 100644 --- a/decompress_xz.go +++ b/decompress_xz.go @@ -10,7 +10,12 @@ import ( // XzDecompressor is an implementation of Decompressor that can // decompress xz files. -type XzDecompressor struct{} +type XzDecompressor struct { + // FileSizeLimit limits the size of a decompressed file. + // + // The zero value means no limit. + FileSizeLimit int64 +} func (d *XzDecompressor) Decompress(dst, src string, dir bool, umask os.FileMode) error { // Directory isn't supported at all @@ -36,6 +41,6 @@ func (d *XzDecompressor) Decompress(dst, src string, dir bool, umask os.FileMode return err } - // Copy it out - return copyReader(dst, xzR, 0622, umask) + // Copy it out, potentially using a file size limit. + return copyReader(dst, xzR, 0622, umask, d.FileSizeLimit) } diff --git a/decompress_zip.go b/decompress_zip.go index 8a672ce8b..dc96fbea9 100644 --- a/decompress_zip.go +++ b/decompress_zip.go @@ -7,9 +7,21 @@ import ( "path/filepath" ) -// ZipDecompressor is an implementation of Decompressor that can decompress zip -// files. -type ZipDecompressor struct{} +// ZipDecompressor is an implementation of Decompressor that can +// decompress zip files. +type ZipDecompressor struct { + // FileSizeLimit limits the total size of all + // decompressed files. + // + // The zero value means no limit. + FileSizeLimit int64 + + // FilesLimit limits the number of files that are + // allowed to be decompressed. + // + // The zero value means no limit. + FilesLimit int +} func (d *ZipDecompressor) Decompress(dst, src string, dir bool, umask os.FileMode) error { // If we're going into a directory we should make that first @@ -37,6 +49,12 @@ func (d *ZipDecompressor) Decompress(dst, src string, dir bool, umask os.FileMod return fmt.Errorf("expected a single file: %s", src) } + if d.FilesLimit > 0 && len(zipR.File) > d.FilesLimit { + return fmt.Errorf("zip archive contains too many files: %d > %d", len(zipR.File), d.FilesLimit) + } + + var fileSizeTotal int64 + // Go through and unarchive for _, f := range zipR.File { path := dst @@ -49,7 +67,15 @@ func (d *ZipDecompressor) Decompress(dst, src string, dir bool, umask os.FileMod path = filepath.Join(path, f.Name) } - if f.FileInfo().IsDir() { + fileInfo := f.FileInfo() + + fileSizeTotal += fileInfo.Size() + + if d.FileSizeLimit > 0 && fileSizeTotal > d.FileSizeLimit { + return fmt.Errorf("zip archive larger than limit: %d", d.FileSizeLimit) + } + + if fileInfo.IsDir() { if !dir { return fmt.Errorf("expected a single file: %s", src) } @@ -78,7 +104,8 @@ func (d *ZipDecompressor) Decompress(dst, src string, dir bool, umask os.FileMod return err } - err = copyReader(path, srcF, f.Mode(), umask) + // Size limit is tracked using the returned file info. + err = copyReader(path, srcF, f.Mode(), umask, 0) srcF.Close() if err != nil { return err diff --git a/decompress_zip_test.go b/decompress_zip_test.go index d898da158..b2298231e 100644 --- a/decompress_zip_test.go +++ b/decompress_zip_test.go @@ -4,6 +4,7 @@ import ( "os" "path/filepath" "runtime" + "strings" "testing" ) @@ -129,3 +130,25 @@ func TestDecompressZipPermissions(t *testing.T) { expected["directory/setuid"] = masked testDecompressorPermissions(t, d, input, expected, os.FileMode(060000000)) } + +func TestDecompressZipBomb(t *testing.T) { + // If the zip decompression bomb protection fails, this can fill up disk space on the entire + // computer. + if os.Getenv("GO_GETTER_TEST_ZIP_BOMB") != "true" { + t.Skip("skipping potentially dangerous test without GO_GETTER_TEST_ZIP_BOMB=true") + } + + // https://www.bamsoftware.com/hacks/zipbomb/zblg.zip + srcPath := filepath.Join("./testdata", "decompress-zip", "bomb.zip") + + d := new(ZipDecompressor) + d.FileSizeLimit = 512 + + err := d.Decompress(t.TempDir(), srcPath, true, 0644) + if err == nil { + t.FailNow() + } + if !strings.Contains(err.Error(), "zip archive larger than limit: 512") { + t.Fatalf("unexpected error: %q", err.Error()) + } +} diff --git a/decompress_zstd.go b/decompress_zstd.go new file mode 100644 index 000000000..3922d27bd --- /dev/null +++ b/decompress_zstd.go @@ -0,0 +1,46 @@ +package getter + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/klauspost/compress/zstd" +) + +// ZstdDecompressor is an implementation of Decompressor that +// can decompress .zst files. +type ZstdDecompressor struct { + // FileSizeLimit limits the size of a decompressed file. + // + // The zero value means no limit. + FileSizeLimit int64 +} + +func (d *ZstdDecompressor) Decompress(dst, src string, dir bool, umask os.FileMode) error { + if dir { + return fmt.Errorf("zstd-compressed files can only unarchive to a single file") + } + + // If we're going into a directory we should make that first + if err := os.MkdirAll(filepath.Dir(dst), mode(0755, umask)); err != nil { + return err + } + + // File first + f, err := os.Open(src) + if err != nil { + return err + } + defer f.Close() + + // zstd compression is second + zstdR, err := zstd.NewReader(f) + if err != nil { + return err + } + defer zstdR.Close() + + // Copy it out, potentially using a file size limit. + return copyReader(dst, zstdR, 0622, umask, d.FileSizeLimit) +} diff --git a/decompress_zstd_test.go b/decompress_zstd_test.go new file mode 100644 index 000000000..6a0683589 --- /dev/null +++ b/decompress_zstd_test.go @@ -0,0 +1,34 @@ +package getter + +import ( + "path/filepath" + "testing" +) + +func TestZstdDecompressor(t *testing.T) { + cases := []TestDecompressCase{ + { + "single.zst", + false, + false, + nil, + "d3b07384d113edec49eaa6238ad5ff00", + nil, + }, + + { + "single.zst", + true, + true, + nil, + "", + nil, + }, + } + + for i, tc := range cases { + cases[i].Input = filepath.Join("./testdata", "decompress-zst", tc.Input) + } + + TestDecompressor(t, new(ZstdDecompressor), cases) +} diff --git a/gcs/get_gcs.go b/gcs/get_gcs.go index 5adf6f843..888f9afc1 100644 --- a/gcs/get_gcs.go +++ b/gcs/get_gcs.go @@ -164,7 +164,8 @@ func (g *Getter) getObject( return err } - return req.CopyReader(dst, rc, 0666) + // There is no limit set for the size of an object from GCS + return req.CopyReader(dst, rc, 0666, 0) } func (g *Getter) parseURL(u *url.URL) (bucket, path string, err error) { diff --git a/gcs/go.sum b/gcs/go.sum index 2c5e72bf4..d75475c9d 100644 --- a/gcs/go.sum +++ b/gcs/go.sum @@ -426,6 +426,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1: github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.11.2 h1:MiK62aErc3gIiVEtyzKfeOHgW7atJb5g/KNX5m3c2nQ= +github.com/klauspost/compress v1.11.2/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= diff --git a/get_file_copy.go b/get_file_copy.go index be44ff35e..3aa35e055 100644 --- a/get_file_copy.go +++ b/get_file_copy.go @@ -31,13 +31,17 @@ func Copy(ctx context.Context, dst io.Writer, src io.Reader) (int64, error) { } // copyReader copies from an io.Reader into a file, using umask to create the dst file -func copyReader(dst string, src io.Reader, fmode, umask os.FileMode) error { +func copyReader(dst string, src io.Reader, fmode, umask os.FileMode, fileSizeLimit int64) error { dstF, err := os.OpenFile(dst, os.O_RDWR|os.O_CREATE|os.O_TRUNC, fmode) if err != nil { return err } defer dstF.Close() + if fileSizeLimit > 0 { + src = io.LimitReader(src, fileSizeLimit) + } + _, err = io.Copy(dstF, src) if err != nil { return err diff --git a/go.mod b/go.mod index 30bb133b5..08cecf8d7 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/hashicorp/go-multierror v1.1.0 github.com/hashicorp/go-safetemp v1.0.0 github.com/hashicorp/go-version v1.1.0 + github.com/klauspost/compress v1.11.2 github.com/mitchellh/go-homedir v1.0.0 github.com/mitchellh/go-testing-interface v1.0.0 github.com/ulikunitz/xz v0.5.8 diff --git a/go.sum b/go.sum index f4cd79dea..f1e596ed4 100644 --- a/go.sum +++ b/go.sum @@ -12,6 +12,8 @@ github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhE github.com/hashicorp/go-safetemp v1.0.0/go.mod h1:oaerMy3BhqiTbVye6QuFhFtIceqFoDHxNAB65b+Rj1I= github.com/hashicorp/go-version v1.1.0 h1:bPIoEKD27tNdebFGGxxYwcL4nepeY4j1QP23PFRGzg0= github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/klauspost/compress v1.11.2 h1:MiK62aErc3gIiVEtyzKfeOHgW7atJb5g/KNX5m3c2nQ= +github.com/klauspost/compress v1.11.2/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnGC8aR0= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= diff --git a/request.go b/request.go index 5e0ee6e9e..1d1356498 100644 --- a/request.go +++ b/request.go @@ -79,8 +79,8 @@ func (req *Request) umask() os.FileMode { return req.Umask } -func (req *Request) CopyReader(dst string, src io.Reader, fmode os.FileMode) error { - return copyReader(dst, src, fmode, req.Umask) +func (req *Request) CopyReader(dst string, src io.Reader, fmode os.FileMode, fileSizeLimit int64) error { + return copyReader(dst, src, fmode, req.Umask, fileSizeLimit) } // Mode returns file Mode umasked by the Request umask diff --git a/s3/get_s3.go b/s3/get_s3.go index 827a292f4..851a8f48d 100644 --- a/s3/get_s3.go +++ b/s3/get_s3.go @@ -24,7 +24,9 @@ import ( type Getter struct { // Timeout sets a deadline which all S3 operations should - // complete within. Zero value means no timeout. + // complete within. + // + // The zero value means no timeout. Timeout time.Duration } @@ -197,7 +199,8 @@ func (g *Getter) getObject(ctx context.Context, client *s3.S3, req *getter.Reque return err } - return req.CopyReader(dst, resp.Body, 0666) + // There is no limit set for the size of an object from S3 + return req.CopyReader(dst, resp.Body, 0666, 0) } func (g *Getter) getAWSConfig(region string, url *url.URL, creds *credentials.Credentials) *aws.Config { diff --git a/s3/go.sum b/s3/go.sum index 9c560d911..694ce6023 100644 --- a/s3/go.sum +++ b/s3/go.sum @@ -20,6 +20,8 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/klauspost/compress v1.11.2 h1:MiK62aErc3gIiVEtyzKfeOHgW7atJb5g/KNX5m3c2nQ= +github.com/klauspost/compress v1.11.2/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnGC8aR0= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= diff --git a/testdata/decompress-tzst/empty.tar.zst b/testdata/decompress-tzst/empty.tar.zst new file mode 100644 index 000000000..d123f19df Binary files /dev/null and b/testdata/decompress-tzst/empty.tar.zst differ diff --git a/testdata/decompress-tzst/multiple.tar.zst b/testdata/decompress-tzst/multiple.tar.zst new file mode 100644 index 000000000..5eb8008bf Binary files /dev/null and b/testdata/decompress-tzst/multiple.tar.zst differ diff --git a/testdata/decompress-tzst/multiple_dir.tar.zst b/testdata/decompress-tzst/multiple_dir.tar.zst new file mode 100644 index 000000000..a3f417a59 Binary files /dev/null and b/testdata/decompress-tzst/multiple_dir.tar.zst differ diff --git a/testdata/decompress-tzst/ordering.tar.zst b/testdata/decompress-tzst/ordering.tar.zst new file mode 100644 index 000000000..d9cbbeeef Binary files /dev/null and b/testdata/decompress-tzst/ordering.tar.zst differ diff --git a/testdata/decompress-tzst/outside_parent.tar.zst b/testdata/decompress-tzst/outside_parent.tar.zst new file mode 100644 index 000000000..b8521bec7 Binary files /dev/null and b/testdata/decompress-tzst/outside_parent.tar.zst differ diff --git a/testdata/decompress-tzst/single.tar.zst b/testdata/decompress-tzst/single.tar.zst new file mode 100644 index 000000000..551371c38 Binary files /dev/null and b/testdata/decompress-tzst/single.tar.zst differ diff --git a/testdata/decompress-zip/bomb.zip b/testdata/decompress-zip/bomb.zip new file mode 100644 index 000000000..3bad671b7 Binary files /dev/null and b/testdata/decompress-zip/bomb.zip differ diff --git a/testdata/decompress-zst/single.zst b/testdata/decompress-zst/single.zst new file mode 100644 index 000000000..5a56316bf Binary files /dev/null and b/testdata/decompress-zst/single.zst differ