Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Refactor progress bar & summary line logic #2984

Merged
merged 2 commits into from
Dec 14, 2022
Merged

Refactor progress bar & summary line logic #2984

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0f7105c
Allow tests to fake stdin/stdout/stderr is a console
terrelln Jan 14, 2022
26c3e42
Refactor progress bar & summary line logic
terrelln Jan 7, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
125 changes: 72 additions & 53 deletions programs/fileio.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -249,6 +249,18 @@ struct FIO_ctx_s {
size_t totalBytesOutput;
};

static int FIO_shouldDisplayFileSummary(FIO_ctx_t const* fCtx)
{
return fCtx->nbFilesTotal <= 1 || g_display_prefs.displayLevel >= 3;
}

static int FIO_shouldDisplayMultipleFileSummary(FIO_ctx_t const* fCtx)
{
int const shouldDisplay = (fCtx->nbFilesProcessed >= 1 && fCtx->nbFilesTotal > 1);
assert(shouldDisplay || FIO_shouldDisplayFileSummary(fCtx) || fCtx->nbFilesProcessed == 0);
return shouldDisplay;
}


/*-*************************************
* Parameters: Initialization
Expand Down Expand Up @@ -1044,14 +1056,16 @@ FIO_compressGzFrame(const cRess_t* ress, /* buffers & handlers are used, but no
strm.avail_out = (uInt)writeJob->bufferSize;
} }
if (srcFileSize == UTIL_FILESIZE_UNKNOWN) {
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ",
(unsigned)(inFileSize>>20),
(double)outFileSize/inFileSize*100)
DISPLAYUPDATE_PROGRESS(
"\rRead : %u MB ==> %.2f%% ",
(unsigned)(inFileSize>>20),
(double)outFileSize/inFileSize*100)
} else {
DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%% ",
(unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
(double)outFileSize/inFileSize*100);
} }
DISPLAYUPDATE_PROGRESS(
"\rRead : %u / %u MB ==> %.2f%% ",
(unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
(double)outFileSize/inFileSize*100);
} }

while (1) {
int const ret = deflate(&strm, Z_FINISH);
Expand Down Expand Up @@ -1141,11 +1155,11 @@ FIO_compressLzmaFrame(cRess_t* ress,
strm.avail_out = writeJob->bufferSize;
} }
if (srcFileSize == UTIL_FILESIZE_UNKNOWN)
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%",
DISPLAYUPDATE_PROGRESS("\rRead : %u MB ==> %.2f%%",
(unsigned)(inFileSize>>20),
(double)outFileSize/inFileSize*100)
else
DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%",
DISPLAYUPDATE_PROGRESS("\rRead : %u / %u MB ==> %.2f%%",
(unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
(double)outFileSize/inFileSize*100);
if (ret == LZMA_STREAM_END) break;
Expand Down Expand Up @@ -1225,11 +1239,11 @@ FIO_compressLz4Frame(cRess_t* ress,
srcFileName, LZ4F_getErrorName(outSize));
outFileSize += outSize;
if (srcFileSize == UTIL_FILESIZE_UNKNOWN) {
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%",
DISPLAYUPDATE_PROGRESS("\rRead : %u MB ==> %.2f%%",
(unsigned)(inFileSize>>20),
(double)outFileSize/inFileSize*100)
} else {
DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%",
DISPLAYUPDATE_PROGRESS("\rRead : %u / %u MB ==> %.2f%%",
(unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
(double)outFileSize/inFileSize*100);
}
Expand Down Expand Up @@ -1364,32 +1378,34 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx,
UTIL_HumanReadableSize_t const produced_hrs = UTIL_makeHumanReadableSize(zfp.produced);

/* display progress notifications */
DISPLAY_PROGRESS("\r%79s\r", ""); /* Clear out the current displayed line */
if (g_display_prefs.displayLevel >= 3) {
DISPLAYUPDATE(3, "\r(L%i) Buffered:%5.*f%s - Consumed:%5.*f%s - Compressed:%5.*f%s => %.2f%% ",
compressionLevel,
buffered_hrs.precision, buffered_hrs.value, buffered_hrs.suffix,
consumed_hrs.precision, consumed_hrs.value, consumed_hrs.suffix,
produced_hrs.precision, produced_hrs.value, produced_hrs.suffix,
cShare );
} else if (g_display_prefs.displayLevel >= 2 || g_display_prefs.progressSetting == FIO_ps_always) {
/* Verbose progress update */
DISPLAY_PROGRESS(
"(L%i) Buffered:%5.*f%s - Consumed:%5.*f%s - Compressed:%5.*f%s => %.2f%% ",
compressionLevel,
buffered_hrs.precision, buffered_hrs.value, buffered_hrs.suffix,
consumed_hrs.precision, consumed_hrs.value, consumed_hrs.suffix,
produced_hrs.precision, produced_hrs.value, produced_hrs.suffix,
cShare );
} else {
/* Require level 2 or forcibly displayed progress counter for summarized updates */
DISPLAYLEVEL(1, "\r%79s\r", ""); /* Clear out the current displayed line */
if (fCtx->nbFilesTotal > 1) {
size_t srcFileNameSize = strlen(srcFileName);
/* Ensure that the string we print is roughly the same size each time */
if (srcFileNameSize > 18) {
const char* truncatedSrcFileName = srcFileName + srcFileNameSize - 15;
DISPLAYLEVEL(1, "Compress: %u/%u files. Current: ...%s ",
DISPLAY_PROGRESS("Compress: %u/%u files. Current: ...%s ",
fCtx->currFileIdx+1, fCtx->nbFilesTotal, truncatedSrcFileName);
} else {
DISPLAYLEVEL(1, "Compress: %u/%u files. Current: %*s ",
DISPLAY_PROGRESS("Compress: %u/%u files. Current: %*s ",
fCtx->currFileIdx+1, fCtx->nbFilesTotal, (int)(18-srcFileNameSize), srcFileName);
}
}
DISPLAYLEVEL(1, "Read:%6.*f%4s ", consumed_hrs.precision, consumed_hrs.value, consumed_hrs.suffix);
DISPLAY_PROGRESS("Read:%6.*f%4s ", consumed_hrs.precision, consumed_hrs.value, consumed_hrs.suffix);
if (fileSize != UTIL_FILESIZE_UNKNOWN)
DISPLAYLEVEL(2, "/%6.*f%4s", file_hrs.precision, file_hrs.value, file_hrs.suffix);
DISPLAYLEVEL(1, " ==> %2.f%%", cShare);
DISPLAY_PROGRESS("/%6.*f%4s", file_hrs.precision, file_hrs.value, file_hrs.suffix);
DISPLAY_PROGRESS(" ==> %2.f%%", cShare);
DELAY_NEXT_UPDATE();
}

Expand Down Expand Up @@ -1555,20 +1571,18 @@ FIO_compressFilename_internal(FIO_ctx_t* const fCtx,
/* Status */
fCtx->totalBytesInput += (size_t)readsize;
fCtx->totalBytesOutput += (size_t)compressedfilesize;
DISPLAYLEVEL(2, "\r%79s\r", "");
if (g_display_prefs.displayLevel >= 2 &&
!fCtx->hasStdoutOutput &&
(g_display_prefs.displayLevel >= 3 || fCtx->nbFilesTotal <= 1)) {
DISPLAY_PROGRESS("\r%79s\r", "");
if (FIO_shouldDisplayFileSummary(fCtx)) {
UTIL_HumanReadableSize_t hr_isize = UTIL_makeHumanReadableSize((U64) readsize);
UTIL_HumanReadableSize_t hr_osize = UTIL_makeHumanReadableSize((U64) compressedfilesize);
if (readsize == 0) {
DISPLAYLEVEL(2,"%-20s : (%6.*f%s => %6.*f%s, %s) \n",
DISPLAY_SUMMARY("%-20s : (%6.*f%s => %6.*f%s, %s) \n",
srcFileName,
hr_isize.precision, hr_isize.value, hr_isize.suffix,
hr_osize.precision, hr_osize.value, hr_osize.suffix,
dstFileName);
} else {
DISPLAYLEVEL(2,"%-20s :%6.2f%% (%6.*f%s => %6.*f%s, %s) \n",
DISPLAY_SUMMARY("%-20s :%6.2f%% (%6.*f%s => %6.*f%s, %s) \n",
srcFileName,
(double)compressedfilesize / (double)readsize * 100,
hr_isize.precision, hr_isize.value, hr_isize.suffix,
Expand Down Expand Up @@ -1917,16 +1931,23 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx,
FIO_checkFilenameCollisions(inFileNamesTable , (unsigned)fCtx->nbFilesTotal);
}

if (fCtx->nbFilesProcessed >= 1 && fCtx->nbFilesTotal > 1 && fCtx->totalBytesInput != 0) {
if (FIO_shouldDisplayMultipleFileSummary(fCtx)) {
UTIL_HumanReadableSize_t hr_isize = UTIL_makeHumanReadableSize((U64) fCtx->totalBytesInput);
UTIL_HumanReadableSize_t hr_osize = UTIL_makeHumanReadableSize((U64) fCtx->totalBytesOutput);

DISPLAYLEVEL(2, "\r%79s\r", "");
DISPLAYLEVEL(2, "%3d files compressed :%.2f%% (%6.*f%4s => %6.*f%4s)\n",
fCtx->nbFilesProcessed,
(double)fCtx->totalBytesOutput/((double)fCtx->totalBytesInput)*100,
hr_isize.precision, hr_isize.value, hr_isize.suffix,
hr_osize.precision, hr_osize.value, hr_osize.suffix);
DISPLAY_PROGRESS("\r%79s\r", "");
if (fCtx->totalBytesInput == 0) {
DISPLAY_SUMMARY("%3d files compressed : (%6.*f%4s => %6.*f%4s)\n",
fCtx->nbFilesProcessed,
hr_isize.precision, hr_isize.value, hr_isize.suffix,
hr_osize.precision, hr_osize.value, hr_osize.suffix);
} else {
DISPLAY_SUMMARY("%3d files compressed : %.2f%% (%6.*f%4s => %6.*f%4s)\n",
fCtx->nbFilesProcessed,
(double)fCtx->totalBytesOutput/((double)fCtx->totalBytesInput)*100,
hr_isize.precision, hr_isize.value, hr_isize.suffix,
hr_osize.precision, hr_osize.value, hr_osize.suffix);
}
}

FIO_freeCResources(&ress);
Expand Down Expand Up @@ -2067,7 +2088,6 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress,
ZSTD_inBuffer inBuff = { ress->readCtx->srcBuffer, ress->readCtx->srcBufferLoaded, 0 };
ZSTD_outBuffer outBuff= { writeJob->buffer, writeJob->bufferSize, 0 };
size_t const readSizeHint = ZSTD_decompressStream(ress->dctx, &outBuff, &inBuff);
const int displayLevel = (g_display_prefs.progressSetting == FIO_ps_always) ? 1 : 2;
UTIL_HumanReadableSize_t const hrs = UTIL_makeHumanReadableSize(alreadyDecoded+frameSize);
if (ZSTD_isError(readSizeHint)) {
DISPLAYLEVEL(1, "%s : Decoding error (36) : %s \n",
Expand All @@ -2085,14 +2105,15 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress,
size_t srcFileNameSize = strlen(srcFileName);
if (srcFileNameSize > 18) {
const char* truncatedSrcFileName = srcFileName + srcFileNameSize - 15;
DISPLAYUPDATE(displayLevel, "\rDecompress: %2u/%2u files. Current: ...%s : %.*f%s... ",
fCtx->currFileIdx+1, fCtx->nbFilesTotal, truncatedSrcFileName, hrs.precision, hrs.value, hrs.suffix);
DISPLAYUPDATE_PROGRESS(
"\rDecompress: %2u/%2u files. Current: ...%s : %.*f%s... ",
fCtx->currFileIdx+1, fCtx->nbFilesTotal, truncatedSrcFileName, hrs.precision, hrs.value, hrs.suffix);
} else {
DISPLAYUPDATE(displayLevel, "\rDecompress: %2u/%2u files. Current: %s : %.*f%s... ",
DISPLAYUPDATE_PROGRESS("\rDecompress: %2u/%2u files. Current: %s : %.*f%s... ",
fCtx->currFileIdx+1, fCtx->nbFilesTotal, srcFileName, hrs.precision, hrs.value, hrs.suffix);
}
} else {
DISPLAYUPDATE(displayLevel, "\r%-20.20s : %.*f%s... ",
DISPLAYUPDATE_PROGRESS("\r%-20.20s : %.*f%s... ",
srcFileName, hrs.precision, hrs.value, hrs.suffix);
}

Expand Down Expand Up @@ -2307,7 +2328,7 @@ FIO_decompressLz4Frame(dRess_t* ress, const char* srcFileName)
AIO_WritePool_enqueueAndReacquireWriteJob(&writeJob);
filesize += decodedBytes;
hrs = UTIL_makeHumanReadableSize(filesize);
DISPLAYUPDATE(2, "\rDecompressed : %.*f%s ", hrs.precision, hrs.value, hrs.suffix);
DISPLAYUPDATE_PROGRESS("\rDecompressed : %.*f%s ", hrs.precision, hrs.value, hrs.suffix);
}

if (!nextToLoad) break;
Expand Down Expand Up @@ -2415,13 +2436,9 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx,

/* Final Status */
fCtx->totalBytesOutput += (size_t)filesize;
DISPLAYLEVEL(2, "\r%79s\r", "");
/* No status message in pipe mode (stdin - stdout) or multi-files mode */
if ((g_display_prefs.displayLevel >= 2 && fCtx->nbFilesTotal <= 1) ||
g_display_prefs.displayLevel >= 3 ||
g_display_prefs.progressSetting == FIO_ps_always) {
DISPLAYLEVEL(1, "\r%-20s: %llu bytes \n", srcFileName, filesize);
}
DISPLAY_PROGRESS("\r%79s\r", "");
if (FIO_shouldDisplayFileSummary(fCtx))
DISPLAY_SUMMARY("%-20s: %llu bytes \n", srcFileName, filesize);

return 0;
}
Expand Down Expand Up @@ -2730,8 +2747,10 @@ FIO_decompressMultipleFilenames(FIO_ctx_t* const fCtx,
FIO_checkFilenameCollisions(srcNamesTable , (unsigned)fCtx->nbFilesTotal);
}

if (fCtx->nbFilesProcessed >= 1 && fCtx->nbFilesTotal > 1 && fCtx->totalBytesOutput != 0)
DISPLAYLEVEL(2, "%d files decompressed : %6zu bytes total \n", fCtx->nbFilesProcessed, fCtx->totalBytesOutput);
if (FIO_shouldDisplayMultipleFileSummary(fCtx)) {
DISPLAY_PROGRESS("\r%79s\r", "");
DISPLAY_SUMMARY("%d files decompressed : %6zu bytes total \n", fCtx->nbFilesProcessed, fCtx->totalBytesOutput);
}

FIO_freeDResources(ress);
return error;
Expand Down Expand Up @@ -3010,7 +3029,7 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis
} }

if (numFiles == 0) {
if (!IS_CONSOLE(stdin)) {
if (!UTIL_isConsole(stdin)) {
DISPLAYLEVEL(1, "zstd: --list does not support reading from standard input \n");
}
DISPLAYLEVEL(1, "No files given \n");
Expand Down
8 changes: 8 additions & 0 deletions programs/fileio_common.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,14 @@ extern UTIL_time_t g_displayClock;
if (g_display_prefs.displayLevel>=4) fflush(stderr); \
} } }

#define SHOULD_DISPLAY_SUMMARY() \
(g_display_prefs.displayLevel >= 2 || g_display_prefs.progressSetting == FIO_ps_always)
#define SHOULD_DISPLAY_PROGRESS() \
(g_display_prefs.progressSetting != FIO_ps_never && SHOULD_DISPLAY_SUMMARY())
#define DISPLAY_PROGRESS(...) { if (SHOULD_DISPLAY_PROGRESS()) { DISPLAYLEVEL(1, __VA_ARGS__); }}
#define DISPLAYUPDATE_PROGRESS(...) { if (SHOULD_DISPLAY_PROGRESS()) { DISPLAYUPDATE(1, __VA_ARGS__); }}
#define DISPLAY_SUMMARY(...) { if (SHOULD_DISPLAY_SUMMARY()) { DISPLAYLEVEL(1, __VA_ARGS__); } }

#undef MIN /* in case it would be already defined */
#define MIN(a,b) ((a) < (b) ? (a) : (b))

Expand Down
4 changes: 4 additions & 0 deletions programs/platform.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,10 @@ extern "C" {

/*-*********************************************
* Detect if isatty() and fileno() are available
*
* Note: Use UTIL_isConsole() for the zstd CLI
* instead, as it allows faking is console for
* testing.
************************************************/
#if (defined(__linux__) && (PLATFORM_POSIX_VERSION > 1)) \
|| (PLATFORM_POSIX_VERSION >= 200112L) \
Expand Down
28 changes: 28 additions & 0 deletions programs/util.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -288,6 +288,34 @@ int UTIL_isLink(const char* infilename)
return 0;
}

static int g_fakeStdinIsConsole = 0;
static int g_fakeStderrIsConsole = 0;
static int g_fakeStdoutIsConsole = 0;

int UTIL_isConsole(FILE* file)
{
if (file == stdin && g_fakeStdinIsConsole)
return 1;
if (file == stderr && g_fakeStderrIsConsole)
return 1;
if (file == stdout && g_fakeStdoutIsConsole)
return 1;
return IS_CONSOLE(file);
}

void UTIL_fakeStdinIsConsole(void)
{
g_fakeStdinIsConsole = 1;
}
void UTIL_fakeStdoutIsConsole(void)
{
g_fakeStdoutIsConsole = 1;
}
void UTIL_fakeStderrIsConsole(void)
{
g_fakeStderrIsConsole = 1;
}

U64 UTIL_getFileSize(const char* infilename)
{
stat_t statbuf;
Expand Down
14 changes: 14 additions & 0 deletions programs/util.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,20 @@ int UTIL_isCompressedFile(const char* infilename, const char *extensionList[]);
int UTIL_isLink(const char* infilename);
int UTIL_isFIFO(const char* infilename);

/**
* Returns with the given file descriptor is a console.
* Allows faking whether stdin/stdout/stderr is a console
* using UTIL_fake*IsConsole().
*/
int UTIL_isConsole(FILE* file);

/**
* Pretends that stdin/stdout/stderr is a console for testing.
*/
void UTIL_fakeStdinIsConsole(void);
void UTIL_fakeStdoutIsConsole(void);
void UTIL_fakeStderrIsConsole(void);

#define UTIL_FILESIZE_UNKNOWN ((U64)(-1))
U64 UTIL_getFileSize(const char* infilename);
U64 UTIL_getTotalFileSize(const char* const * fileNamesTable, unsigned nbFiles);
Expand Down
13 changes: 8 additions & 5 deletions programs/zstdcli.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@
/*-************************************
* Dependencies
**************************************/
#include "platform.h" /* IS_CONSOLE, PLATFORM_POSIX_VERSION */
#include "util.h" /* UTIL_HAS_CREATEFILELIST, UTIL_createFileList */
#include "platform.h" /* PLATFORM_POSIX_VERSION */
#include "util.h" /* UTIL_HAS_CREATEFILELIST, UTIL_createFileList, UTIL_isConsole */
#include <stdlib.h> /* getenv */
#include <string.h> /* strcmp, strlen */
#include <stdio.h> /* fprintf(), stdin, stdout, stderr */
Expand Down Expand Up @@ -987,6 +987,9 @@ int main(int argCount, const char* argv[])
if (!strcmp(argument, "--no-progress")) { FIO_setProgressSetting(FIO_ps_never); continue; }
if (!strcmp(argument, "--progress")) { FIO_setProgressSetting(FIO_ps_always); continue; }
if (!strcmp(argument, "--exclude-compressed")) { FIO_setExcludeCompressedFile(prefs, 1); continue; }
if (!strcmp(argument, "--fake-stdin-is-console")) { UTIL_fakeStdinIsConsole(); continue; }
if (!strcmp(argument, "--fake-stdout-is-console")) { UTIL_fakeStdoutIsConsole(); continue; }
if (!strcmp(argument, "--fake-stderr-is-console")) { UTIL_fakeStderrIsConsole(); continue; }

/* long commands with arguments */
#ifndef ZSTD_NODICT
Expand Down Expand Up @@ -1437,12 +1440,12 @@ int main(int argCount, const char* argv[])
/* Check if input/output defined as console; trigger an error in this case */
if (!forceStdin
&& (UTIL_searchFileNamesTable(filenames, stdinmark) != -1)
&& IS_CONSOLE(stdin) ) {
&& UTIL_isConsole(stdin) ) {
DISPLAYLEVEL(1, "stdin is a console, aborting\n");
CLEAN_RETURN(1);
}
if ( (!outFileName || !strcmp(outFileName, stdoutmark))
&& IS_CONSOLE(stdout)
&& UTIL_isConsole(stdout)
&& (UTIL_searchFileNamesTable(filenames, stdinmark) != -1)
&& !forceStdout
&& operation!=zom_decompress ) {
Expand Down Expand Up @@ -1479,7 +1482,7 @@ int main(int argCount, const char* argv[])
/* No status message in pipe mode (stdin - stdout) */
hasStdout = outFileName && !strcmp(outFileName,stdoutmark);

if ((hasStdout || !IS_CONSOLE(stderr)) && (g_displayLevel==2)) g_displayLevel=1;
if ((hasStdout || !UTIL_isConsole(stderr)) && (g_displayLevel==2)) g_displayLevel=1;

/* IO Stream/File */
FIO_setHasStdoutOutput(fCtx, hasStdout);
Expand Down
Loading