From 49e36679c5d9f81898ee87dcaf80dbda59e8ea7f Mon Sep 17 00:00:00 2001 From: Evan Jones Date: Wed, 7 Jul 2021 16:40:42 -0400 Subject: [PATCH 1/2] zstd.go: Set ZSTD_LEGACY_SUPPORT=4 to decompress legacy payloads The compressed output format was finalized with zstd version 0.8. To support decompression of previous versions, set the macro ZSTD_LEGACY_SUPPORT=4, which matches the documented default for the zstd command line tool: https://github.com/facebook/zstd/blob/dev/programs/README.md This made the compiled library about 20 kiB larger, but will allow this to be used with really old zstd payloads. .circleci/config.yml: Do not test with different versions of this macro, since the tests now include a check that it can decompress old payloads. --- .circleci/config.yml | 9 --------- zstd.go | 5 +++++ zstd_test.go | 25 +++++++++++++++++++++++++ 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3f54a9c..9a93946 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -41,15 +41,6 @@ jobs: - run: 'unzip mr.zip' - run: 'go build' - run: 'PAYLOAD=`pwd`/mr GODEBUG=efence=1 go test -v' - "golang-zstd-legacy-support": - docker: - - image: circleci/golang:latest - steps: - - checkout - - run: 'wget https://github.com/DataDog/zstd/files/2246767/mr.zip' - - run: 'unzip mr.zip' - - run: 'CGO_CFLAGS="-DZSTD_LEGACY_SUPPORT=1" go build' - - run: 'PAYLOAD=`pwd`/mr CGO_CFLAGS="-DZSTD_LEGACY_SUPPORT=1" go test -v' "golang-i386": docker: - image: 32bit/ubuntu:16.04 diff --git a/zstd.go b/zstd.go index 634ed65..a86eef5 100644 --- a/zstd.go +++ b/zstd.go @@ -1,6 +1,11 @@ package zstd /* +// support decoding of "legacy" zstd payloads from versions [0.4, 0.8], matching the +// default configuration of the zstd command line tool: +// https://github.com/facebook/zstd/blob/dev/programs/README.md +#cgo CFLAGS: -DZSTD_LEGACY_SUPPORT=4 + #define ZSTD_STATIC_LINKING_ONLY #include "zstd.h" */ diff --git a/zstd_test.go b/zstd_test.go index e5bb2d2..399b69a 100644 --- a/zstd_test.go +++ b/zstd_test.go @@ -6,6 +6,8 @@ import ( "fmt" "io/ioutil" "os" + "strconv" + "strings" "testing" ) @@ -259,6 +261,29 @@ func TestRealPayload(t *testing.T) { } } +func TestLegacy(t *testing.T) { + // payloads compressed with zstd v0.5 + // needs ZSTD_LEGACY_SUPPORT=5 or less + testCases := []struct { + input string + expected string + }{ + {"%\xb5/\xfd\x00@\x00\x1bcompressed with legacy zstd\xc0\x00\x00", "compressed with legacy zstd"}, + {"%\xb5/\xfd\x00\x00\x00A\x11\x007\x14\xb0\xb5\x01@\x1aR\xb6iI7[FH\x022u\xe0O-\x18\xe3G\x9e2\xab\xd9\xea\xca7؊\xee\x884\xbf\xe7\xdc\xe4@\xe1-\x9e\xac\xf0\xf2\x86\x0f\xf1r\xbb7\b\x81Z\x01\x00\x01\x00\xdf`\xfe\xc0\x00\x00", "compressed with legacy zstd"}, + } + for i, testCase := range testCases { + t.Run(strconv.Itoa(i), func(t *testing.T) { + out, err := Decompress(nil, []byte(testCase.input)) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(out), testCase.expected) { + t.Errorf("expected to find %#v; output=%#v", testCase.expected, string(out)) + } + }) + } +} + func BenchmarkCompression(b *testing.B) { if raw == nil { b.Fatal(ErrNoPayloadEnv) From 791d5e3e96fb64a7f922bab0590d52d4ef8611bc Mon Sep 17 00:00:00 2001 From: Dmytro Milinevskyi Date: Wed, 16 Mar 2022 13:39:31 +0100 Subject: [PATCH 2/2] ci:circle: remove golang-zstd-legacy-support from the workflows --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5a4f907..f341c3c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -109,4 +109,3 @@ workflows: - "golang-efence" - "golang-efence-external-libzstd" - "golang-i386" - - "golang-zstd-legacy-support"