Skip to content

Dev #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jan 24, 2015
Merged

Dev #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
18a6871
Fixed : issue within fullbench, reported by AJ Baxter
Cyan4973 Jan 24, 2015
759433d
Updated fullbench tests
Cyan4973 Jan 24, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions lib/zstd.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1004,9 +1004,9 @@ size_t ZSTD_compressContinue(ZSTD_cctx_t cctx, void* dst, size_t maxDstSize, con
//U32 limit = 4 * BLOCKSIZE;
//const U32 updateRate = 2 * BLOCKSIZE;

// Init
/* Init */
if (ctx->base==NULL) ctx->base = src, ctx->current=0;
if (src != ctx->base + ctx->current) // not contiguous
if (src != ctx->base + ctx->current) /* not contiguous */
{
ZSTD_resetCCtx(ctx);
ctx->base = src;
Expand Down Expand Up @@ -1035,12 +1035,12 @@ size_t ZSTD_compressContinue(ZSTD_cctx_t cctx, void* dst, size_t maxDstSize, con
}
*/

// compress
/* compress */
if (maxDstSize < ZSTD_blockHeaderSize) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
cSize = ZSTD_compressBlock(ctx, op+ZSTD_blockHeaderSize, maxDstSize-ZSTD_blockHeaderSize, ip, blockSize);
if (cSize == 0)
{
cSize = ZSTD_noCompressBlock(op, maxDstSize, ip, blockSize);
cSize = ZSTD_noCompressBlock(op, maxDstSize, ip, blockSize); /* block is not compressible */
if (ZSTD_isError(cSize)) return cSize;
}
else
Expand Down Expand Up @@ -1084,23 +1084,23 @@ static size_t ZSTD_compressCCtx(void* ctx, void* dst, size_t maxDstSize, const v
BYTE* const ostart = dst;
BYTE* op = ostart;

// Header
/* Header */
{
size_t headerSize = ZSTD_compressBegin(ctx, dst, maxDstSize);
if(ZSTD_isError(headerSize)) return headerSize;
op += headerSize;
maxDstSize -= headerSize;
}

// Compression
/* Compression */
{
size_t cSize = ZSTD_compressContinue(ctx, op, maxDstSize, src, srcSize);
if (ZSTD_isError(cSize)) return cSize;
op += cSize;
maxDstSize -= cSize;
}

// Close frame
/* Close frame */
{
size_t endSize = ZSTD_compressEnd(ctx, op, maxDstSize);
if(ZSTD_isError(endSize)) return endSize;
Expand Down
10 changes: 8 additions & 2 deletions programs/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,11 +127,17 @@ test-zstd32: zstd32 datagen
./datagen -g256MB | ./zstd32 -v | ./zstd32 -d > $(VOID)
./datagen -g6GB | ./zstd32 -vq | ./zstd32 -d > $(VOID)

test-fullbench: fullbench
test-fullbench: fullbench datagen
./fullbench -i1
./datagen -P0 -g516K > tmp
./fullbench -i1 tmp
@rm tmp

test-fullbench32: fullbench32
test-fullbench32: fullbench32 datagen
./fullbench32 -i1
./datagen -P0 -g516K > tmp
./fullbench32 -i1 tmp
@rm tmp

test-fuzzer: fuzzer
./fuzzer
Expand Down
87 changes: 62 additions & 25 deletions programs/datagen.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,24 @@
#endif


/**************************************
* OS-specific Includes
**************************************/
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
# include <fcntl.h> /* _O_BINARY */
# include <io.h> /* _setmode, _isatty */
# ifdef __MINGW32__
int _fileno(FILE *stream); /* MINGW somehow forgets to include this windows declaration into <stdio.h> */
# endif
# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY)
# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
#else
# include <unistd.h> /* isatty */
# define SET_BINARY_MODE(file)
# define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
#endif


/**************************************
* Constants
**************************************/
Expand Down Expand Up @@ -85,7 +103,6 @@
* Local Parameters
**************************************/
static unsigned no_prompt = 0;
static char* programName;
static unsigned displayLevel = 2;


Expand All @@ -97,7 +114,7 @@ static unsigned int CDG_rand(U32* src)
{
U32 rand32 = *src;
rand32 *= PRIME1;
rand32 += PRIME2;
rand32 ^= PRIME2;
rand32 = CDG_rotl32(rand32, 13);
*src = rand32;
return rand32;
Expand All @@ -106,14 +123,20 @@ static unsigned int CDG_rand(U32* src)

#define LTSIZE 8192
#define LTMASK (LTSIZE-1)
static const char firstChar = '(';
static const char lastChar = '}';
static void* CDG_createLiteralDistrib(double ld)
{
char* lt = malloc(LTSIZE);
BYTE* lt = malloc(LTSIZE);
U32 i = 0;
char character = '0';
BYTE character = '0';
BYTE firstChar = '(';
BYTE lastChar = '}';

if (ld==0.0)
{
character = 0;
firstChar = 0;
lastChar =255;
}
while (i<LTSIZE)
{
U32 weight = (U32)((double)(LTSIZE - i) * ld) + 1;
Expand All @@ -137,18 +160,19 @@ static char CDG_genChar(U32* seed, const void* ltctx)
#define CDG_RAND15BITS ((CDG_rand(seed) >> 3) & 32767)
#define CDG_RANDLENGTH ( ((CDG_rand(seed) >> 7) & 7) ? (CDG_rand(seed) & 15) : (CDG_rand(seed) & 511) + 15)
#define CDG_DICTSIZE (32 KB)
static void CDG_generate(U64 size, U32* seed, double matchProba)
static void CDG_generate(U64 size, U32* seed, double matchProba, double litProba)
{
BYTE fullbuff[CDG_DICTSIZE + 128 KB + 1];
BYTE* buff = fullbuff + CDG_DICTSIZE;
U64 total=0;
U32 P32 = (U32)(32768 * matchProba);
U32 pos=1;
U32 genBlockSize = 128 KB;
double literalDistrib = 0.13;
void* ldctx = CDG_createLiteralDistrib(literalDistrib);
void* ldctx = CDG_createLiteralDistrib(litProba);
FILE* fout = stdout;

/* Build initial prefix */
/* init */
SET_BINARY_MODE(stdout);
fullbuff[0] = CDG_genChar(seed, ldctx);
while (pos<32 KB)
{
Expand Down Expand Up @@ -207,11 +231,8 @@ static void CDG_generate(U64 size, U32* seed, double matchProba)
}
}

/* output datagen */
pos=0;
for (;pos+512<=genBlockSize;pos+=512)
printf("%512.512s", buff+pos);
for (;pos<genBlockSize;pos++) printf("%c", buff[pos]);
/* output generated data */
fwrite(buff, 1, genBlockSize, fout);
/* Regenerate prefix */
memcpy(fullbuff, buff + 96 KB, 32 KB);
}
Expand All @@ -221,7 +242,7 @@ static void CDG_generate(U64 size, U32* seed, double matchProba)
/*********************************************************
* Command line
*********************************************************/
static int CDG_usage(void)
static int CDG_usage(char* programName)
{
DISPLAY( "Compressible data generator\n");
DISPLAY( "Usage :\n");
Expand All @@ -239,9 +260,11 @@ static int CDG_usage(void)
int main(int argc, char** argv)
{
int argNb;
int proba = CDG_COMPRESSIBILITY_DEFAULT;
double proba = (double)CDG_COMPRESSIBILITY_DEFAULT / 100;
double litProba = proba / 3.6;
U64 size = CDG_SIZE_DEFAULT;
U32 seed = CDG_SEED_DEFAULT;
char* programName;

/* Check command line */
programName = argv[0];
Expand All @@ -262,7 +285,7 @@ int main(int argc, char** argv)
switch(*argument)
{
case 'h':
return CDG_usage();
return CDG_usage(programName);
case 'g':
argument++;
size=0;
Expand All @@ -287,23 +310,37 @@ int main(int argc, char** argv)
argument++;
}
break;
case 'p':
case 'P':
argument++;
proba=0;
proba=0.0;
while ((*argument>='0') && (*argument<='9'))
{
proba *= 10;
proba += *argument - '0';
argument++;
}
if (proba<0) proba=0;
if (proba>100) proba=100;
if (proba>100.) proba=100.;
proba /= 100.;
litProba = proba / 4.;
break;
case 'L':
argument++;
litProba=0.;
while ((*argument>='0') && (*argument<='9'))
{
litProba *= 10;
litProba += *argument - '0';
argument++;
}
if (litProba>100.) litProba=100.;
litProba /= 100.;
break;
case 'v':
displayLevel = 4;
argument++;
break;
default: ;
default:
return CDG_usage(programName);
}
}

Expand All @@ -312,9 +349,9 @@ int main(int argc, char** argv)

DISPLAYLEVEL(4, "Data Generator %s \n", ZSTD_VERSION);
DISPLAYLEVEL(3, "Seed = %u \n", seed);
if (proba!=CDG_COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", proba);
if (proba!=CDG_COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", (U32)(proba*100));

CDG_generate(size, &seed, ((double)proba) / 100);
CDG_generate(size, &seed, proba, litProba);

return 0;
}
26 changes: 21 additions & 5 deletions programs/fullbench.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -383,7 +383,7 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
}

/* Allocation */
dstBuffSize = srcSize + 512;
dstBuffSize = ZSTD_compressBound(srcSize);
dstBuff = malloc(dstBuffSize);
buff2 = malloc(dstBuffSize);
if ((!dstBuff) || (!buff2))
Expand All @@ -403,6 +403,14 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
{
blockProperties_t bp;
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize);
ZSTD_getcBlockSize(dstBuff+4, dstBuffSize, &bp); // Get first block compressed size
if (bp.blockType != bt_compressed)
{
DISPLAY("ZSTD_decodeLiteralsBlock : impossible to test on this sample (not compressible)\n");
free(dstBuff);
free(buff2);
return 0;
}
g_cSize = ZSTD_getcBlockSize(dstBuff+7, dstBuffSize, &bp) + 3;
memcpy(buff2, dstBuff+7, g_cSize);
//srcSize = benchFunction(dstBuff, dstBuffSize, buff2, src, srcSize); // real speed
Expand All @@ -418,6 +426,13 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize);
ip += 4; // Jump magic Number
blockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); // Get first block compressed size
if (bp.blockType != bt_compressed)
{
DISPLAY("ZSTD_decodeSeqHeaders : impossible to test on this sample (not compressible)\n");
free(dstBuff);
free(buff2);
return 0;
}
iend = ip + 3 + blockSize; // Get end of first block
ip += 3; // jump first block header
ip += ZSTD_getcBlockSize(ip, iend - ip, &bp) + 3; // jump literal sub block and its header
Expand Down Expand Up @@ -450,14 +465,15 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
default : ;
}

{ size_t i; for (i=0; i<dstBuffSize; i++) dstBuff[i]=(BYTE)i; } /* warming up memory */

for (loopNb = 1; loopNb <= nbIterations; loopNb++)
{
double averageTime;
int milliTime;
U32 nbRounds=0;

DISPLAY("%2i- %-30.30s : \r", loopNb, benchName);
{ size_t i; for (i=0; i<dstBuffSize; i++) dstBuff[i]=(BYTE)i; } /* warming up memory */

milliTime = BMK_GetMilliStart();
while(BMK_GetMilliStart() == milliTime);
Expand Down Expand Up @@ -545,7 +561,7 @@ int benchFiles(char** fileNamesTable, int nbFiles, U32 benchNb)
DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20));
}

// Alloc
/* Alloc */
origBuff = (char*) malloc((size_t)benchedSize);
if(!origBuff)
{
Expand All @@ -554,7 +570,7 @@ int benchFiles(char** fileNamesTable, int nbFiles, U32 benchNb)
return 12;
}

// Fill input buffer
/* Fill input buffer */
DISPLAY("Loading %s... \r", inFileName);
readSize = fread(origBuff, 1, benchedSize, inFile);
fclose(inFile);
Expand All @@ -566,7 +582,7 @@ int benchFiles(char** fileNamesTable, int nbFiles, U32 benchNb)
return 13;
}

// bench
/* bench */
DISPLAY("\r%79s\r", "");
DISPLAY(" %s : \n", inFileName);
if (benchNb)
Expand Down