Skip to content

Commit 8202057

Browse files
paulcruz74paulcruz74
authored
Merge pull request #2 from paulcruz74/list
List
2 parents 37e1b14 + fc428ab commit 8202057

File tree

3 files changed

+208
-3
lines changed

3 files changed

+208
-3
lines changed

‎programs/fileio.c

Lines changed: 187 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -862,6 +862,193 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
862862
return result;
863863
}
864864

865+
typedef struct {
866+
int numActualFrames;
867+
int numSkippableFrames;
868+
unsigned long long decompressedSize;
869+
int canComputeDecompSize;
870+
unsigned long long compressedSize;
871+
int usesCheck;
872+
} fileInfo_t;
873+
874+
875+
int calcFrameHeaderSize(BYTE frameHeaderDescriptor){
876+
int frameContentSizeBytes = 0;
877+
int windowDescriptorBytes;
878+
int dictionaryIDBytes;
879+
int frameContentSizeFlag = frameHeaderDescriptor >> 6;
880+
int singleSegmentFlag = (frameHeaderDescriptor & (1 << 5)) >> 5;
881+
int dictionaryIDFlag = frameHeaderDescriptor & 3;
882+
if(frameContentSizeFlag!=0){
883+
frameContentSizeBytes = 1 << frameContentSizeFlag;
884+
}
885+
else if(singleSegmentFlag){
886+
frameContentSizeBytes = 1;
887+
}
888+
windowDescriptorBytes = singleSegmentFlag ? 0 : 1;
889+
dictionaryIDBytes = dictionaryIDFlag ? 1 << (dictionaryIDFlag - 1): 0;
890+
return 4 + 1 + windowDescriptorBytes + frameContentSizeBytes + dictionaryIDBytes;
891+
}
892+
893+
/*
894+
* Reads information from file, stores in *info
895+
* if successful, returns 0, otherwise returns 1
896+
*/
897+
int getFileInfo(fileInfo_t* info, const char* inFileName){
898+
FILE* srcFile = FIO_openSrcFile(inFileName);
899+
if(srcFile==NULL){
900+
return 1;
901+
}
902+
info->compressedSize = (unsigned long long)UTIL_getFileSize(inFileName);
903+
info->decompressedSize = 0;
904+
info->numActualFrames = 0;
905+
info-> numSkippableFrames = 0;
906+
info->canComputeDecompSize = 1;
907+
/* begin analyzing frame */
908+
while(1){
909+
BYTE magicNumberBuffer[4];
910+
size_t numBytesRead = fread(magicNumberBuffer, 1, 4, srcFile);
911+
U32 magicNumber;
912+
if(numBytesRead != 4) break;
913+
magicNumber = MEM_readLE32(magicNumberBuffer);
914+
if(magicNumber==ZSTD_MAGICNUMBER){
915+
BYTE frameHeaderDescriptor;
916+
int totalFrameHeaderBytes;
917+
BYTE* frameHeader;
918+
int lastBlock = 0;
919+
size_t readBytes = fread(&frameHeaderDescriptor, 1, 1, srcFile);
920+
info->numActualFrames++;
921+
if(readBytes != 1){
922+
DISPLAY("There was an error with reading frame header descriptor\n");
923+
exit(1);
924+
}
925+
/* calculate actual frame header size */
926+
totalFrameHeaderBytes = calcFrameHeaderSize(frameHeaderDescriptor);
927+
928+
/* reset to beginning of from and read entire header */
929+
fseek(srcFile, -5, SEEK_CUR);
930+
frameHeader = (BYTE*)malloc(totalFrameHeaderBytes);
931+
readBytes = fread(frameHeader, 1, totalFrameHeaderBytes, srcFile);
932+
if(readBytes != (size_t)totalFrameHeaderBytes){
933+
DISPLAY("There was an error reading the frame header\n");
934+
exit(1);
935+
}
936+
937+
/* get decompressed file size */
938+
{
939+
U64 additional = ZSTD_getFrameContentSize(frameHeader, totalFrameHeaderBytes);
940+
if(additional!=ZSTD_CONTENTSIZE_UNKNOWN && additional!=ZSTD_CONTENTSIZE_ERROR){
941+
info->decompressedSize += additional;
942+
}
943+
else{
944+
info->canComputeDecompSize = 0;
945+
}
946+
}
947+
948+
/* skip the rest of the blocks in the frame */
949+
do{
950+
BYTE blockHeaderBuffer[3];
951+
U32 blockHeader;
952+
int blockSize;
953+
readBytes = fread(blockHeaderBuffer, 1, 3, srcFile);
954+
if(readBytes != 3){
955+
DISPLAY("There was a problem reading the block header\n");
956+
exit(1);
957+
}
958+
blockHeader = MEM_readLE24(blockHeaderBuffer);
959+
lastBlock = blockHeader & 1;
960+
blockSize = (blockHeader - (blockHeader & 7)) >> 3;
961+
fseek(srcFile, blockSize, SEEK_CUR);
962+
}while(lastBlock != 1);
963+
{
964+
/* check if checksum is used */
965+
int contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2;
966+
if(contentChecksumFlag){
967+
info->usesCheck = 1;
968+
}
969+
if(contentChecksumFlag){
970+
fseek(srcFile, 4, SEEK_CUR);
971+
}
972+
}
973+
}
974+
else if(magicNumber==ZSTD_MAGIC_SKIPPABLE_START){
975+
BYTE frameSizeBuffer[4];
976+
long frameSize;
977+
size_t readBytes = fread(frameSizeBuffer, 1, 4, srcFile);
978+
info->numSkippableFrames++;
979+
if(readBytes != 4){
980+
DISPLAY("There was an error reading skippable frame size");
981+
exit(1);
982+
}
983+
frameSize = MEM_readLE32(frameSizeBuffer);
984+
fseek(srcFile, frameSize, SEEK_CUR);
985+
}
986+
987+
}
988+
return 0;
989+
}
990+
void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){
991+
double compressedSizeMB = (double)info->compressedSize/(1 MB);
992+
double decompressedSizeMB = (double)info->decompressedSize/(1 MB);
993+
994+
if(displayLevel<=2){
995+
if(info->usesCheck && info->canComputeDecompSize){
996+
DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n");
997+
DISPLAY("%9d %13d %7.2f MB %7.2f MB %5.3f XXH64 %s\n",
998+
info->numSkippableFrames, info->numActualFrames, compressedSizeMB, decompressedSizeMB,
999+
compressedSizeMB/decompressedSizeMB, inFileName);
1000+
}
1001+
else if(!info->usesCheck){
1002+
DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n");
1003+
DISPLAY("%9d %13d %7.2f MB %7.2f MB %5.3f %s\n",
1004+
info->numSkippableFrames, info->numActualFrames, compressedSizeMB, decompressedSizeMB,
1005+
compressedSizeMB/decompressedSizeMB, inFileName);
1006+
}
1007+
else if(!info->canComputeDecompSize){
1008+
DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n");
1009+
DISPLAY("%9d %13d %7.2f MB XXH64 %s\n",
1010+
info->numSkippableFrames, info->numActualFrames, compressedSizeMB, inFileName);
1011+
}
1012+
else{
1013+
DISPLAY("Skippable Non-Skippable Filename\n");
1014+
DISPLAY("%9d %13d %7.2f MB %s\n",
1015+
info->numSkippableFrames, info->numActualFrames, compressedSizeMB, inFileName);
1016+
}
1017+
}
1018+
else{
1019+
DISPLAY("# Zstandard Frames: %d\n", info->numActualFrames);
1020+
DISPLAY("# Skippable Frames: %d\n", info->numSkippableFrames);
1021+
DISPLAY("Compressed Size: %.2f MB (%llu B)\n", compressedSizeMB, info->compressedSize);
1022+
if(info->canComputeDecompSize){
1023+
DISPLAY("Decompressed Size: %.2f MB (%llu B)\n", decompressedSizeMB, info->decompressedSize);
1024+
DISPLAY("Ratio: %.4f\n", compressedSizeMB/decompressedSizeMB);
1025+
}
1026+
if(info->usesCheck){
1027+
DISPLAY("Check: XXH64\n");
1028+
}
1029+
}
1030+
1031+
}
1032+
1033+
int FIO_listFile(const char* inFileName, int displayLevel){
1034+
const char* const suffixPtr = strrchr(inFileName, '.');
1035+
DISPLAY("File: %s\n", inFileName);
1036+
if(!suffixPtr || strcmp(suffixPtr, ZSTD_EXTENSION)){
1037+
DISPLAYLEVEL(1, "file %s was not compressed with zstd -- ignoring\n\n", inFileName);
1038+
return 1;
1039+
}
1040+
else{
1041+
fileInfo_t* info = (fileInfo_t*)malloc(sizeof(fileInfo_t));
1042+
int error = getFileInfo(info, inFileName);
1043+
if(error==1){
1044+
DISPLAY("An error occurred with getting file info\n");
1045+
exit(1);
1046+
}
1047+
displayInfo(inFileName, info, displayLevel);
1048+
}
1049+
DISPLAY("\n");
1050+
return 0;
1051+
}
8651052

8661053
int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFiles,
8671054
const char* suffix,

‎programs/fileio.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ int FIO_compressFilename (const char* outfilename, const char* infilename, const
7070
@return : 0 == ok; 1 == pb with src file. */
7171
int FIO_decompressFilename (const char* outfilename, const char* infilename, const char* dictFileName);
7272

73+
int FIO_listFile(const char* infilename, int displayLevel);
7374

7475
/*-*************************************
7576
* Multiple File functions

‎programs/zstdcli.c

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,7 @@ static int usage_advanced(const char* programName)
117117
DISPLAY( " -v : verbose mode; specify multiple times to increase verbosity\n");
118118
DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n");
119119
DISPLAY( " -c : force write to standard output, even if it is the console\n");
120+
DISPLAY( " -l : print information about zstd compressed files.\n");
120121
#ifndef ZSTD_NOCOMPRESS
121122
DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
122123
#ifdef ZSTD_MULTITHREAD
@@ -310,7 +311,7 @@ static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressi
310311
}
311312

312313

313-
typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train } zstd_operation_mode;
314+
typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train, zom_list } zstd_operation_mode;
314315

315316
#define CLEAN_RETURN(i) { operationResult = (i); goto _end; }
316317

@@ -531,7 +532,7 @@ int main(int argCount, const char* argv[])
531532
argument++;
532533
memLimit = readU32FromChar(&argument);
533534
break;
534-
535+
case 'l': operation=zom_list; argument++; break;
535536
#ifdef UTIL_HAS_CREATEFILELIST
536537
/* recursive */
537538
case 'r': recursive=1; argument++; break;
@@ -672,7 +673,23 @@ int main(int argCount, const char* argv[])
672673
}
673674
}
674675
#endif
675-
676+
if(operation==zom_list){
677+
if(filenameIdx==0){
678+
DISPLAY("No files given\n");
679+
CLEAN_RETURN(0);
680+
}
681+
DISPLAY("===========================================\n");
682+
DISPLAY("Printing information about compressed files\n");
683+
DISPLAY("===========================================\n");
684+
DISPLAY("Number of files listed: %d\n", filenameIdx);
685+
{
686+
unsigned u;
687+
for(u=0; u<filenameIdx;u++){
688+
FIO_listFile(filenameTable[u],g_displayLevel);
689+
}
690+
}
691+
CLEAN_RETURN(0);
692+
}
676693
/* Check if benchmark is selected */
677694
if (operation==zom_bench) {
678695
#ifndef ZSTD_NOBENCH

0 commit comments

Comments
 (0)