Skip to content

Commit

Permalink
Refactor progress bar & summary line logic
Browse files Browse the repository at this point in the history
* Centralize the logic about whether to print the progress bar or not in
  the `*_PROGRESS()` macros.
* Centralize the logc about whether to print the summary line or not in
  `FIO_shouldDisplayFileSummary()` and
  `FIO_shouldDisplayMultipleFileSummary()`.
* Make `--progress` work for non-zstd (de)compressors.
* Clean up several edge cases in compression and decompression progress
  printing along the way. E.g. wrong log level, or missing summary line.

One thing I don't like about stdout mode, which sets the display level
to 1, is that warnings aren't displayed. After this PR, we could change
stdout mode from lowering the display level, to defaulting to implied
`--no-progress`. But, I think that deserves a separate PR.
  • Loading branch information
terrelln committed Dec 14, 2022
1 parent e58a39f commit fbff782
Show file tree
Hide file tree
Showing 6 changed files with 318 additions and 52 deletions.
123 changes: 71 additions & 52 deletions programs/fileio.c
Original file line number Diff line number Diff line change
Expand Up @@ -249,6 +249,18 @@ struct FIO_ctx_s {
size_t totalBytesOutput;
};

static int FIO_shouldDisplayFileSummary(FIO_ctx_t const* fCtx)
{
return fCtx->nbFilesTotal <= 1 || g_display_prefs.displayLevel >= 3;
}

static int FIO_shouldDisplayMultipleFileSummary(FIO_ctx_t const* fCtx)
{
int const shouldDisplay = (fCtx->nbFilesProcessed >= 1 && fCtx->nbFilesTotal > 1);
assert(shouldDisplay || FIO_shouldDisplayFileSummary(fCtx) || fCtx->nbFilesProcessed == 0);
return shouldDisplay;
}


/*-*************************************
* Parameters: Initialization
Expand Down Expand Up @@ -1044,14 +1056,16 @@ FIO_compressGzFrame(const cRess_t* ress, /* buffers & handlers are used, but no
strm.avail_out = (uInt)writeJob->bufferSize;
} }
if (srcFileSize == UTIL_FILESIZE_UNKNOWN) {
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ",
(unsigned)(inFileSize>>20),
(double)outFileSize/inFileSize*100)
DISPLAYUPDATE_PROGRESS(
"\rRead : %u MB ==> %.2f%% ",
(unsigned)(inFileSize>>20),
(double)outFileSize/inFileSize*100)
} else {
DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%% ",
(unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
(double)outFileSize/inFileSize*100);
} }
DISPLAYUPDATE_PROGRESS(
"\rRead : %u / %u MB ==> %.2f%% ",
(unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
(double)outFileSize/inFileSize*100);
} }

while (1) {
int const ret = deflate(&strm, Z_FINISH);
Expand Down Expand Up @@ -1141,11 +1155,11 @@ FIO_compressLzmaFrame(cRess_t* ress,
strm.avail_out = writeJob->bufferSize;
} }
if (srcFileSize == UTIL_FILESIZE_UNKNOWN)
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%",
DISPLAYUPDATE_PROGRESS("\rRead : %u MB ==> %.2f%%",
(unsigned)(inFileSize>>20),
(double)outFileSize/inFileSize*100)
else
DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%",
DISPLAYUPDATE_PROGRESS("\rRead : %u / %u MB ==> %.2f%%",
(unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
(double)outFileSize/inFileSize*100);
if (ret == LZMA_STREAM_END) break;
Expand Down Expand Up @@ -1225,11 +1239,11 @@ FIO_compressLz4Frame(cRess_t* ress,
srcFileName, LZ4F_getErrorName(outSize));
outFileSize += outSize;
if (srcFileSize == UTIL_FILESIZE_UNKNOWN) {
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%",
DISPLAYUPDATE_PROGRESS("\rRead : %u MB ==> %.2f%%",
(unsigned)(inFileSize>>20),
(double)outFileSize/inFileSize*100)
} else {
DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%",
DISPLAYUPDATE_PROGRESS("\rRead : %u / %u MB ==> %.2f%%",
(unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
(double)outFileSize/inFileSize*100);
}
Expand Down Expand Up @@ -1364,32 +1378,34 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx,
UTIL_HumanReadableSize_t const produced_hrs = UTIL_makeHumanReadableSize(zfp.produced);

/* display progress notifications */
DISPLAY_PROGRESS("\r%79s\r", ""); /* Clear out the current displayed line */
if (g_display_prefs.displayLevel >= 3) {
DISPLAYUPDATE(3, "\r(L%i) Buffered:%5.*f%s - Consumed:%5.*f%s - Compressed:%5.*f%s => %.2f%% ",
compressionLevel,
buffered_hrs.precision, buffered_hrs.value, buffered_hrs.suffix,
consumed_hrs.precision, consumed_hrs.value, consumed_hrs.suffix,
produced_hrs.precision, produced_hrs.value, produced_hrs.suffix,
cShare );
} else if (g_display_prefs.displayLevel >= 2 || g_display_prefs.progressSetting == FIO_ps_always) {
/* Verbose progress update */
DISPLAY_PROGRESS(
"(L%i) Buffered:%5.*f%s - Consumed:%5.*f%s - Compressed:%5.*f%s => %.2f%% ",
compressionLevel,
buffered_hrs.precision, buffered_hrs.value, buffered_hrs.suffix,
consumed_hrs.precision, consumed_hrs.value, consumed_hrs.suffix,
produced_hrs.precision, produced_hrs.value, produced_hrs.suffix,
cShare );
} else {
/* Require level 2 or forcibly displayed progress counter for summarized updates */
DISPLAYLEVEL(1, "\r%79s\r", ""); /* Clear out the current displayed line */
if (fCtx->nbFilesTotal > 1) {
size_t srcFileNameSize = strlen(srcFileName);
/* Ensure that the string we print is roughly the same size each time */
if (srcFileNameSize > 18) {
const char* truncatedSrcFileName = srcFileName + srcFileNameSize - 15;
DISPLAYLEVEL(1, "Compress: %u/%u files. Current: ...%s ",
DISPLAY_PROGRESS("Compress: %u/%u files. Current: ...%s ",
fCtx->currFileIdx+1, fCtx->nbFilesTotal, truncatedSrcFileName);
} else {
DISPLAYLEVEL(1, "Compress: %u/%u files. Current: %*s ",
DISPLAY_PROGRESS("Compress: %u/%u files. Current: %*s ",
fCtx->currFileIdx+1, fCtx->nbFilesTotal, (int)(18-srcFileNameSize), srcFileName);
}
}
DISPLAYLEVEL(1, "Read:%6.*f%4s ", consumed_hrs.precision, consumed_hrs.value, consumed_hrs.suffix);
DISPLAY_PROGRESS("Read:%6.*f%4s ", consumed_hrs.precision, consumed_hrs.value, consumed_hrs.suffix);
if (fileSize != UTIL_FILESIZE_UNKNOWN)
DISPLAYLEVEL(2, "/%6.*f%4s", file_hrs.precision, file_hrs.value, file_hrs.suffix);
DISPLAYLEVEL(1, " ==> %2.f%%", cShare);
DISPLAY_PROGRESS("/%6.*f%4s", file_hrs.precision, file_hrs.value, file_hrs.suffix);
DISPLAY_PROGRESS(" ==> %2.f%%", cShare);
DELAY_NEXT_UPDATE();
}

Expand Down Expand Up @@ -1555,20 +1571,18 @@ FIO_compressFilename_internal(FIO_ctx_t* const fCtx,
/* Status */
fCtx->totalBytesInput += (size_t)readsize;
fCtx->totalBytesOutput += (size_t)compressedfilesize;
DISPLAYLEVEL(2, "\r%79s\r", "");
if (g_display_prefs.displayLevel >= 2 &&
!fCtx->hasStdoutOutput &&
(g_display_prefs.displayLevel >= 3 || fCtx->nbFilesTotal <= 1)) {
DISPLAY_PROGRESS("\r%79s\r", "");
if (FIO_shouldDisplayFileSummary(fCtx)) {
UTIL_HumanReadableSize_t hr_isize = UTIL_makeHumanReadableSize((U64) readsize);
UTIL_HumanReadableSize_t hr_osize = UTIL_makeHumanReadableSize((U64) compressedfilesize);
if (readsize == 0) {
DISPLAYLEVEL(2,"%-20s : (%6.*f%s => %6.*f%s, %s) \n",
DISPLAY_SUMMARY("%-20s : (%6.*f%s => %6.*f%s, %s) \n",
srcFileName,
hr_isize.precision, hr_isize.value, hr_isize.suffix,
hr_osize.precision, hr_osize.value, hr_osize.suffix,
dstFileName);
} else {
DISPLAYLEVEL(2,"%-20s :%6.2f%% (%6.*f%s => %6.*f%s, %s) \n",
DISPLAY_SUMMARY("%-20s :%6.2f%% (%6.*f%s => %6.*f%s, %s) \n",
srcFileName,
(double)compressedfilesize / (double)readsize * 100,
hr_isize.precision, hr_isize.value, hr_isize.suffix,
Expand Down Expand Up @@ -1917,16 +1931,23 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx,
FIO_checkFilenameCollisions(inFileNamesTable , (unsigned)fCtx->nbFilesTotal);
}

if (fCtx->nbFilesProcessed >= 1 && fCtx->nbFilesTotal > 1 && fCtx->totalBytesInput != 0) {
if (FIO_shouldDisplayMultipleFileSummary(fCtx)) {
UTIL_HumanReadableSize_t hr_isize = UTIL_makeHumanReadableSize((U64) fCtx->totalBytesInput);
UTIL_HumanReadableSize_t hr_osize = UTIL_makeHumanReadableSize((U64) fCtx->totalBytesOutput);

DISPLAYLEVEL(2, "\r%79s\r", "");
DISPLAYLEVEL(2, "%3d files compressed :%.2f%% (%6.*f%4s => %6.*f%4s)\n",
fCtx->nbFilesProcessed,
(double)fCtx->totalBytesOutput/((double)fCtx->totalBytesInput)*100,
hr_isize.precision, hr_isize.value, hr_isize.suffix,
hr_osize.precision, hr_osize.value, hr_osize.suffix);
DISPLAY_PROGRESS("\r%79s\r", "");
if (fCtx->totalBytesInput == 0) {
DISPLAY_SUMMARY("%3d files compressed : (%6.*f%4s => %6.*f%4s)\n",
fCtx->nbFilesProcessed,
hr_isize.precision, hr_isize.value, hr_isize.suffix,
hr_osize.precision, hr_osize.value, hr_osize.suffix);
} else {
DISPLAY_SUMMARY("%3d files compressed : %.2f%% (%6.*f%4s => %6.*f%4s)\n",
fCtx->nbFilesProcessed,
(double)fCtx->totalBytesOutput/((double)fCtx->totalBytesInput)*100,
hr_isize.precision, hr_isize.value, hr_isize.suffix,
hr_osize.precision, hr_osize.value, hr_osize.suffix);
}
}

FIO_freeCResources(&ress);
Expand Down Expand Up @@ -2067,7 +2088,6 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress,
ZSTD_inBuffer inBuff = { ress->readCtx->srcBuffer, ress->readCtx->srcBufferLoaded, 0 };
ZSTD_outBuffer outBuff= { writeJob->buffer, writeJob->bufferSize, 0 };
size_t const readSizeHint = ZSTD_decompressStream(ress->dctx, &outBuff, &inBuff);
const int displayLevel = (g_display_prefs.progressSetting == FIO_ps_always) ? 1 : 2;
UTIL_HumanReadableSize_t const hrs = UTIL_makeHumanReadableSize(alreadyDecoded+frameSize);
if (ZSTD_isError(readSizeHint)) {
DISPLAYLEVEL(1, "%s : Decoding error (36) : %s \n",
Expand All @@ -2085,14 +2105,15 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress,
size_t srcFileNameSize = strlen(srcFileName);
if (srcFileNameSize > 18) {
const char* truncatedSrcFileName = srcFileName + srcFileNameSize - 15;
DISPLAYUPDATE(displayLevel, "\rDecompress: %2u/%2u files. Current: ...%s : %.*f%s... ",
fCtx->currFileIdx+1, fCtx->nbFilesTotal, truncatedSrcFileName, hrs.precision, hrs.value, hrs.suffix);
DISPLAYUPDATE_PROGRESS(
"\rDecompress: %2u/%2u files. Current: ...%s : %.*f%s... ",
fCtx->currFileIdx+1, fCtx->nbFilesTotal, truncatedSrcFileName, hrs.precision, hrs.value, hrs.suffix);
} else {
DISPLAYUPDATE(displayLevel, "\rDecompress: %2u/%2u files. Current: %s : %.*f%s... ",
DISPLAYUPDATE_PROGRESS("\rDecompress: %2u/%2u files. Current: %s : %.*f%s... ",
fCtx->currFileIdx+1, fCtx->nbFilesTotal, srcFileName, hrs.precision, hrs.value, hrs.suffix);
}
} else {
DISPLAYUPDATE(displayLevel, "\r%-20.20s : %.*f%s... ",
DISPLAYUPDATE_PROGRESS("\r%-20.20s : %.*f%s... ",
srcFileName, hrs.precision, hrs.value, hrs.suffix);
}

Expand Down Expand Up @@ -2307,7 +2328,7 @@ FIO_decompressLz4Frame(dRess_t* ress, const char* srcFileName)
AIO_WritePool_enqueueAndReacquireWriteJob(&writeJob);
filesize += decodedBytes;
hrs = UTIL_makeHumanReadableSize(filesize);
DISPLAYUPDATE(2, "\rDecompressed : %.*f%s ", hrs.precision, hrs.value, hrs.suffix);
DISPLAYUPDATE_PROGRESS("\rDecompressed : %.*f%s ", hrs.precision, hrs.value, hrs.suffix);
}

if (!nextToLoad) break;
Expand Down Expand Up @@ -2415,13 +2436,9 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx,

/* Final Status */
fCtx->totalBytesOutput += (size_t)filesize;
DISPLAYLEVEL(2, "\r%79s\r", "");
/* No status message in pipe mode (stdin - stdout) or multi-files mode */
if ((g_display_prefs.displayLevel >= 2 && fCtx->nbFilesTotal <= 1) ||
g_display_prefs.displayLevel >= 3 ||
g_display_prefs.progressSetting == FIO_ps_always) {
DISPLAYLEVEL(1, "\r%-20s: %llu bytes \n", srcFileName, filesize);
}
DISPLAY_PROGRESS("\r%79s\r", "");
if (FIO_shouldDisplayFileSummary(fCtx))
DISPLAY_SUMMARY("%-20s: %llu bytes \n", srcFileName, filesize);

return 0;
}
Expand Down Expand Up @@ -2730,8 +2747,10 @@ FIO_decompressMultipleFilenames(FIO_ctx_t* const fCtx,
FIO_checkFilenameCollisions(srcNamesTable , (unsigned)fCtx->nbFilesTotal);
}

if (fCtx->nbFilesProcessed >= 1 && fCtx->nbFilesTotal > 1 && fCtx->totalBytesOutput != 0)
DISPLAYLEVEL(2, "%d files decompressed : %6zu bytes total \n", fCtx->nbFilesProcessed, fCtx->totalBytesOutput);
if (FIO_shouldDisplayMultipleFileSummary(fCtx)) {
DISPLAY_PROGRESS("\r%79s\r", "");
DISPLAY_SUMMARY("%d files decompressed : %6zu bytes total \n", fCtx->nbFilesProcessed, fCtx->totalBytesOutput);
}

FIO_freeDResources(ress);
return error;
Expand Down
8 changes: 8 additions & 0 deletions programs/fileio_common.h
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,14 @@ extern UTIL_time_t g_displayClock;
if (g_display_prefs.displayLevel>=4) fflush(stderr); \
} } }

#define SHOULD_DISPLAY_SUMMARY() \
(g_display_prefs.displayLevel >= 2 || g_display_prefs.progressSetting == FIO_ps_always)
#define SHOULD_DISPLAY_PROGRESS() \
(g_display_prefs.progressSetting != FIO_ps_never && SHOULD_DISPLAY_SUMMARY())
#define DISPLAY_PROGRESS(...) { if (SHOULD_DISPLAY_PROGRESS()) { DISPLAYLEVEL(1, __VA_ARGS__); }}
#define DISPLAYUPDATE_PROGRESS(...) { if (SHOULD_DISPLAY_PROGRESS()) { DISPLAYUPDATE(1, __VA_ARGS__); }}
#define DISPLAY_SUMMARY(...) { if (SHOULD_DISPLAY_SUMMARY()) { DISPLAYLEVEL(1, __VA_ARGS__); } }

#undef MIN /* in case it would be already defined */
#define MIN(a,b) ((a) < (b) ? (a) : (b))

Expand Down
46 changes: 46 additions & 0 deletions tests/cli-tests/progress/no-progress.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
#!/bin/sh

#!/bin/sh

. "$COMMON/platform.sh"

set -e

echo hello > hello
echo world > world

zstd -q hello world

println >&2 "Tests cases where progress information should not be printed"

for args in \
"" \
"--fake-stderr-is-console -q" \
"--fake-stderr-is-console -qq --progress" \
"--no-progress --fake-stderr-is-console" \
"--no-progress --fake-stderr-is-console -v"
do
println >&2 "args = $args"
println >&2 "compress file to file"
zstd $args -f hello
println >&2 "compress pipe to pipe"
zstd $args < hello > $INTOVOID
println >&2 "compress pipe to file"
zstd $args < hello -fo hello.zst
println >&2 "compress file to pipe"
zstd $args hello -c > $INTOVOID
println >&2 "compress 2 files"
zstd $args -f hello world

println >&2 "decompress file to file"
zstd $args -d -f hello.zst
println >&2 "decompress pipe to pipe"
zstd $args -d < hello.zst > $INTOVOID
println >&2 "decompress pipe to file"
zstd $args -d < hello.zst -fo hello
println >&2 "decompress file to pipe"
zstd $args -d hello.zst -c > $INTOVOID
println >&2 "decompress 2 files"
zstd $args -d -f hello.zst world.zst
println >&2 ""
done
90 changes: 90 additions & 0 deletions tests/cli-tests/progress/no-progress.sh.stderr.glob
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
Tests cases where progress information should not be printed
args =
compress file to file
compress pipe to pipe
compress pipe to file
compress file to pipe
compress 2 files
decompress file to file
decompress pipe to pipe
decompress pipe to file
decompress file to pipe
decompress 2 files

args = --fake-stderr-is-console -q
compress file to file
compress pipe to pipe
compress pipe to file
compress file to pipe
compress 2 files
decompress file to file
decompress pipe to pipe
decompress pipe to file
decompress file to pipe
decompress 2 files

args = --fake-stderr-is-console -qq --progress
compress file to file
compress pipe to pipe
compress pipe to file
compress file to pipe
compress 2 files
decompress file to file
decompress pipe to pipe
decompress pipe to file
decompress file to pipe
decompress 2 files

args = --no-progress --fake-stderr-is-console
compress file to file
hello*hello.zst*
compress pipe to pipe
compress pipe to file
*stdin*hello.zst*
compress file to pipe
compress 2 files
2 files compressed*
decompress file to file
hello.zst*
decompress pipe to pipe
decompress pipe to file
*stdin*
decompress file to pipe
decompress 2 files
2 files decompressed*

args = --no-progress --fake-stderr-is-console -v
compress file to file
*zstd*
hello*hello.zst*
compress pipe to pipe
*zstd*
*stdin*stdout*
compress pipe to file
*zstd*
*stdin*hello.zst*
compress file to pipe
*zstd*
*hello*stdout*
compress 2 files
*zstd*
*hello*hello.zst*
*world*world.zst*
2 files compressed*
decompress file to file
*zstd*
hello.zst*
decompress pipe to pipe
*zstd*
*stdin*
decompress pipe to file
*zstd*
*stdin*
decompress file to pipe
*zstd*
hello.zst*
decompress 2 files
*zstd*
hello.zst*
world.zst*
2 files decompressed*
Loading

0 comments on commit fbff782

Please sign in to comment.