Skip to content

Commit d549254

Browse files
paulcruz74paulcruz74
authored
Merge pull request #3 from facebook/dev
pulling changes
2 parents 5be0f55 + c16748b commit d549254

14 files changed

+1080
-48
lines changed

‎README.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,11 +67,11 @@ Previous charts provide results applicable to typical file and stream scenarios
6767
The smaller the amount of data to compress, the more difficult it is to compress. This problem is common to all compression algorithms, and reason is, compression algorithms learn from past data how to compress future data. But at the beginning of a new data set, there is no "past" to build upon.
6868

6969
To solve this situation, Zstd offers a __training mode__, which can be used to tune the algorithm for a selected type of data.
70-
Training Zstandard is achieved by provide it with a few samples (one file per sample). The result of this training is stored in a file called "dictionary", which must be loaded before compression and decompression.
70+
Training Zstandard is achieved by providing it with a few samples (one file per sample). The result of this training is stored in a file called "dictionary", which must be loaded before compression and decompression.
7171
Using this dictionary, the compression ratio achievable on small data improves dramatically.
7272

7373
The following example uses the `github-users` [sample set](https://github.com/facebook/zstd/releases/tag/v1.1.3), created from [github public API](https://developer.github.com/v3/users/#get-all-users).
74-
It consists of roughly 10K records weighting about 1KB each.
74+
It consists of roughly 10K records weighing about 1KB each.
7575

7676
Compression Ratio | Compression Speed | Decompression Speed
7777
------------------|-------------------|--------------------

‎contrib/linux-kernel/lib/zstd/compress.c

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1978,10 +1978,15 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx *ctx, const void *src, size_t src
19781978
break; /* nothing found : store previous solution */
19791979
}
19801980

1981+
/* NOTE:
1982+
* start[-offset+ZSTD_REP_MOVE-1] is undefined behavior.
1983+
* (-offset+ZSTD_REP_MOVE-1) is unsigned, and is added to start, which
1984+
* overflows the pointer, which is undefined behavior.
1985+
*/
19811986
/* catch up */
19821987
if (offset) {
19831988
while ((start > anchor) && (start > base + offset - ZSTD_REP_MOVE) &&
1984-
(start[-1] == start[-1 - offset + ZSTD_REP_MOVE])) /* only search for offset within prefix */
1989+
(start[-1] == (start-offset+ZSTD_REP_MOVE)[-1])) /* only search for offset within prefix */
19851990
{
19861991
start--;
19871992
matchLength++;

‎contrib/linux-kernel/lib/zstd/decompress.c

Lines changed: 20 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -2212,6 +2212,20 @@ ZSTD_DStream *ZSTD_initDStream(size_t maxWindowSize, void *workspace, size_t wor
22122212
zds->ddict = zds->ddictLocal;
22132213
zds->legacyVersion = 0;
22142214
zds->hostageByte = 0;
2215+
2216+
{
2217+
size_t const blockSize = MIN(zds->maxWindowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);
2218+
size_t const neededOutSize = zds->maxWindowSize + blockSize + WILDCOPY_OVERLENGTH * 2;
2219+
2220+
zds->inBuff = (char *)ZSTD_malloc(blockSize, zds->customMem);
2221+
zds->inBuffSize = blockSize;
2222+
zds->outBuff = (char *)ZSTD_malloc(neededOutSize, zds->customMem);
2223+
zds->outBuffSize = neededOutSize;
2224+
if (zds->inBuff == NULL || zds->outBuff == NULL) {
2225+
ZSTD_freeDStream(zds);
2226+
return NULL;
2227+
}
2228+
}
22152229
return zds;
22162230
}
22172231

@@ -2333,25 +2347,17 @@ size_t ZSTD_decompressStream(ZSTD_DStream *zds, ZSTD_outBuffer *output, ZSTD_inB
23332347
if (zds->fParams.windowSize > zds->maxWindowSize)
23342348
return ERROR(frameParameter_windowTooLarge);
23352349

2336-
/* Adapt buffer sizes to frame header instructions */
2350+
/* Buffers are preallocated, but double check */
23372351
{
2338-
size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);
2339-
size_t const neededOutSize = zds->fParams.windowSize + blockSize + WILDCOPY_OVERLENGTH * 2;
2340-
zds->blockSize = blockSize;
2352+
size_t const blockSize = MIN(zds->maxWindowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);
2353+
size_t const neededOutSize = zds->maxWindowSize + blockSize + WILDCOPY_OVERLENGTH * 2;
23412354
if (zds->inBuffSize < blockSize) {
2342-
ZSTD_free(zds->inBuff, zds->customMem);
2343-
zds->inBuffSize = blockSize;
2344-
zds->inBuff = (char *)ZSTD_malloc(blockSize, zds->customMem);
2345-
if (zds->inBuff == NULL)
2346-
return ERROR(memory_allocation);
2355+
return ERROR(GENERIC);
23472356
}
23482357
if (zds->outBuffSize < neededOutSize) {
2349-
ZSTD_free(zds->outBuff, zds->customMem);
2350-
zds->outBuffSize = neededOutSize;
2351-
zds->outBuff = (char *)ZSTD_malloc(neededOutSize, zds->customMem);
2352-
if (zds->outBuff == NULL)
2353-
return ERROR(memory_allocation);
2358+
return ERROR(GENERIC);
23542359
}
2360+
zds->blockSize = blockSize;
23552361
}
23562362
zds->stage = zdss_read;
23572363
}

‎contrib/linux-kernel/test/DecompressCrash.c

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
/**
2+
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
3+
* All rights reserved.
4+
*
5+
* This source code is licensed under the BSD-style license found in the
6+
* LICENSE file in the root directory of this source tree. An additional grant
7+
* of patent rights can be found in the PATENTS file in the same directory.
8+
*/
9+
10+
/*
11+
This program takes a file in input,
12+
performs a zstd round-trip test (compression - decompress)
13+
compares the result with original
14+
and generates a crash (double free) on corruption detection.
15+
*/
16+
17+
/*===========================================
18+
* Dependencies
19+
*==========================================*/
20+
#include <stddef.h> /* size_t */
21+
#include <stdlib.h> /* malloc, free, exit */
22+
#include <stdio.h> /* fprintf */
23+
#include <linux/zstd.h>
24+
25+
/*===========================================
26+
* Macros
27+
*==========================================*/
28+
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
29+
30+
static ZSTD_DCtx *dctx = NULL;
31+
void *dws = NULL;
32+
static void* rBuff = NULL;
33+
static size_t buffSize = 0;
34+
35+
static void crash(int errorCode){
36+
/* abort if AFL/libfuzzer, exit otherwise */
37+
#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION /* could also use __AFL_COMPILER */
38+
abort();
39+
#else
40+
exit(errorCode);
41+
#endif
42+
}
43+
44+
static void decompressCheck(const void* srcBuff, size_t srcBuffSize)
45+
{
46+
size_t const neededBuffSize = 20 * srcBuffSize;
47+
48+
/* Allocate all buffers and contexts if not already allocated */
49+
if (neededBuffSize > buffSize) {
50+
free(rBuff);
51+
buffSize = 0;
52+
53+
rBuff = malloc(neededBuffSize);
54+
if (!rBuff) {
55+
fprintf(stderr, "not enough memory ! \n");
56+
crash(1);
57+
}
58+
buffSize = neededBuffSize;
59+
}
60+
if (!dctx) {
61+
size_t const workspaceSize = ZSTD_DCtxWorkspaceBound();
62+
dws = malloc(workspaceSize);
63+
if (!dws) {
64+
fprintf(stderr, "not enough memory ! \n");
65+
crash(1);
66+
}
67+
dctx = ZSTD_initDCtx(dws, workspaceSize);
68+
if (!dctx) {
69+
fprintf(stderr, "not enough memory ! \n");
70+
crash(1);
71+
}
72+
}
73+
ZSTD_decompressDCtx(dctx, rBuff, buffSize, srcBuff, srcBuffSize);
74+
75+
#ifndef SKIP_FREE
76+
free(dws); dws = NULL; dctx = NULL;
77+
free(rBuff); rBuff = NULL;
78+
buffSize = 0;
79+
#endif
80+
}
81+
82+
int LLVMFuzzerTestOneInput(const unsigned char *srcBuff, size_t srcBuffSize) {
83+
decompressCheck(srcBuff, srcBuffSize);
84+
return 0;
85+
}

‎contrib/linux-kernel/test/Makefile

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,12 +12,24 @@ CPPFLAGS += $(IFLAGS)
1212
../lib/zstd/libzstd.a: $(OBJECTS)
1313
$(AR) $(ARFLAGS) $@ $^
1414

15+
DecompressCrash: DecompressCrash.o $(OBJECTS) libFuzzer.a
16+
$(CXX) $(TEST_CPPFLAGS) $(TEST_CXXFLAGS) $(LDFLAGS) $^ -o $@
17+
18+
RoundTripCrash: RoundTripCrash.o $(OBJECTS) ../lib/xxhash.o libFuzzer.a
19+
$(CXX) $(TEST_CPPFLAGS) $(TEST_CXXFLAGS) $(LDFLAGS) $^ -o $@
20+
1521
UserlandTest: UserlandTest.cpp ../lib/zstd/libzstd.a ../lib/xxhash.o
1622
$(CXX) $(CXXFLAGS) $(CFLAGS) $(CPPFLAGS) $^ googletest/build/googlemock/gtest/libgtest.a googletest/build/googlemock/gtest/libgtest_main.a -o $@
1723

1824
XXHashUserlandTest: XXHashUserlandTest.cpp ../lib/xxhash.o ../../../lib/common/xxhash.o
1925
$(CXX) $(CXXFLAGS) $(CFLAGS) $(CPPFLAGS) $^ googletest/build/googlemock/gtest/libgtest.a googletest/build/googlemock/gtest/libgtest_main.a -o $@
2026

27+
# Install libfuzzer
28+
libFuzzer.a:
29+
@$(RM) -rf Fuzzer
30+
@git clone https://chromium.googlesource.com/chromium/llvm-project/llvm/lib/Fuzzer
31+
@./Fuzzer/build.sh
32+
2133
# Install googletest
2234
.PHONY: googletest
2335
googletest:
@@ -28,3 +40,4 @@ googletest:
2840

2941
clean:
3042
$(RM) -f *.{o,a} ../lib/zstd/*.{o,a}
43+
$(RM) -f DecompressCrash RoundTripCrash UserlandTest XXHashUserlandTest

‎contrib/linux-kernel/test/RoundTripCrash.c

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
/**
2+
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
3+
* All rights reserved.
4+
*
5+
* This source code is licensed under the BSD-style license found in the
6+
* LICENSE file in the root directory of this source tree. An additional grant
7+
* of patent rights can be found in the PATENTS file in the same directory.
8+
*/
9+
10+
/*
11+
This program takes a file in input,
12+
performs a zstd round-trip test (compression - decompress)
13+
compares the result with original
14+
and generates a crash (double free) on corruption detection.
15+
*/
16+
17+
/*===========================================
18+
* Dependencies
19+
*==========================================*/
20+
#include <stddef.h> /* size_t */
21+
#include <stdlib.h> /* malloc, free, exit */
22+
#include <stdio.h> /* fprintf */
23+
#include <linux/xxhash.h>
24+
#include <linux/zstd.h>
25+
26+
/*===========================================
27+
* Macros
28+
*==========================================*/
29+
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
30+
31+
static const int kMaxClevel = 22;
32+
33+
static ZSTD_CCtx *cctx = NULL;
34+
void *cws = NULL;
35+
static ZSTD_DCtx *dctx = NULL;
36+
void *dws = NULL;
37+
static void* cBuff = NULL;
38+
static void* rBuff = NULL;
39+
static size_t buffSize = 0;
40+
41+
42+
/** roundTripTest() :
43+
* Compresses `srcBuff` into `compressedBuff`,
44+
* then decompresses `compressedBuff` into `resultBuff`.
45+
* Compression level used is derived from first content byte.
46+
* @return : result of decompression, which should be == `srcSize`
47+
* or an error code if either compression or decompression fails.
48+
* Note : `compressedBuffCapacity` should be `>= ZSTD_compressBound(srcSize)`
49+
* for compression to be guaranteed to work */
50+
static size_t roundTripTest(void* resultBuff, size_t resultBuffCapacity,
51+
void* compressedBuff, size_t compressedBuffCapacity,
52+
const void* srcBuff, size_t srcBuffSize)
53+
{
54+
size_t const hashLength = MIN(128, srcBuffSize);
55+
unsigned const h32 = xxh32(srcBuff, hashLength, 0);
56+
int const cLevel = h32 % kMaxClevel;
57+
ZSTD_parameters const params = ZSTD_getParams(cLevel, srcBuffSize, 0);
58+
size_t const cSize = ZSTD_compressCCtx(cctx, compressedBuff, compressedBuffCapacity, srcBuff, srcBuffSize, params);
59+
if (ZSTD_isError(cSize)) {
60+
fprintf(stderr, "Compression error : %u \n", ZSTD_getErrorCode(cSize));
61+
return cSize;
62+
}
63+
return ZSTD_decompressDCtx(dctx, resultBuff, resultBuffCapacity, compressedBuff, cSize);
64+
}
65+
66+
67+
static size_t checkBuffers(const void* buff1, const void* buff2, size_t buffSize)
68+
{
69+
const char* ip1 = (const char*)buff1;
70+
const char* ip2 = (const char*)buff2;
71+
size_t pos;
72+
73+
for (pos=0; pos<buffSize; pos++)
74+
if (ip1[pos]!=ip2[pos])
75+
break;
76+
77+
return pos;
78+
}
79+
80+
static void crash(int errorCode){
81+
/* abort if AFL/libfuzzer, exit otherwise */
82+
#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION /* could also use __AFL_COMPILER */
83+
abort();
84+
#else
85+
exit(errorCode);
86+
#endif
87+
}
88+
89+
static void roundTripCheck(const void* srcBuff, size_t srcBuffSize)
90+
{
91+
size_t const neededBuffSize = ZSTD_compressBound(srcBuffSize);
92+
93+
/* Allocate all buffers and contexts if not already allocated */
94+
if (neededBuffSize > buffSize) {
95+
free(cBuff);
96+
free(rBuff);
97+
buffSize = 0;
98+
99+
cBuff = malloc(neededBuffSize);
100+
rBuff = malloc(neededBuffSize);
101+
if (!cBuff || !rBuff) {
102+
fprintf(stderr, "not enough memory ! \n");
103+
crash(1);
104+
}
105+
buffSize = neededBuffSize;
106+
}
107+
if (!cctx) {
108+
ZSTD_compressionParameters const params = ZSTD_getCParams(kMaxClevel, 0, 0);
109+
size_t const workspaceSize = ZSTD_CCtxWorkspaceBound(params);
110+
cws = malloc(workspaceSize);
111+
if (!cws) {
112+
fprintf(stderr, "not enough memory ! \n");
113+
crash(1);
114+
}
115+
cctx = ZSTD_initCCtx(cws, workspaceSize);
116+
if (!cctx) {
117+
fprintf(stderr, "not enough memory ! \n");
118+
crash(1);
119+
}
120+
}
121+
if (!dctx) {
122+
size_t const workspaceSize = ZSTD_DCtxWorkspaceBound();
123+
dws = malloc(workspaceSize);
124+
if (!dws) {
125+
fprintf(stderr, "not enough memory ! \n");
126+
crash(1);
127+
}
128+
dctx = ZSTD_initDCtx(dws, workspaceSize);
129+
if (!dctx) {
130+
fprintf(stderr, "not enough memory ! \n");
131+
crash(1);
132+
}
133+
}
134+
135+
{ size_t const result = roundTripTest(rBuff, buffSize, cBuff, buffSize, srcBuff, srcBuffSize);
136+
if (ZSTD_isError(result)) {
137+
fprintf(stderr, "roundTripTest error : %u \n", ZSTD_getErrorCode(result));
138+
crash(1);
139+
}
140+
if (result != srcBuffSize) {
141+
fprintf(stderr, "Incorrect regenerated size : %u != %u\n", (unsigned)result, (unsigned)srcBuffSize);
142+
crash(1);
143+
}
144+
if (checkBuffers(srcBuff, rBuff, srcBuffSize) != srcBuffSize) {
145+
fprintf(stderr, "Silent decoding corruption !!!");
146+
crash(1);
147+
}
148+
}
149+
150+
#ifndef SKIP_FREE
151+
free(cws); cws = NULL; cctx = NULL;
152+
free(dws); dws = NULL; dctx = NULL;
153+
free(cBuff); cBuff = NULL;
154+
free(rBuff); rBuff = NULL;
155+
buffSize = 0;
156+
#endif
157+
}
158+
159+
int LLVMFuzzerTestOneInput(const unsigned char *srcBuff, size_t srcBuffSize) {
160+
roundTripCheck(srcBuff, srcBuffSize);
161+
return 0;
162+
}

‎contrib/linux-kernel/test/UserlandTest.cpp

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -280,9 +280,9 @@ TEST(Block, ContentSize) {
280280

281281
TEST(Block, CCtxLevelIncrease) {
282282
std::string c;
283-
auto cctx = createCCtx(6);
283+
auto cctx = createCCtx(22);
284284
auto dctx = createDCtx();
285-
for (int level = 1; level <= 6; ++level) {
285+
for (int level = 1; level <= 22; ++level) {
286286
auto compressed = compress(*cctx, kData, level);
287287
auto const decompressed = decompress(*dctx, compressed, kData.size());
288288
EXPECT_EQ(kData, decompressed);
@@ -478,6 +478,17 @@ TEST(Stream, Flush) {
478478
EXPECT_EQ(kData, decompressed);
479479
}
480480

481+
TEST(Stream, DStreamLevelIncrease) {
482+
auto zds = createDStream();
483+
for (int level = 1; level <= 22; ++level) {
484+
auto zcs = createCStream(level);
485+
auto compressed = compress(*zcs, kData);
486+
ZSTD_resetDStream(zds.get());
487+
auto const decompressed = decompress(*zds, compressed, kData.size());
488+
EXPECT_EQ(kData, decompressed);
489+
}
490+
}
491+
481492
#define TEST_SYMBOL(symbol) \
482493
do { \
483494
extern void *__##symbol; \

0 commit comments

Comments
 (0)