Skip to content

Commit

Permalink
Merge pull request #2983 from facebook/minLitPricev2
Browse files Browse the repository at this point in the history 
[opt] minor compression ratio improvement
  • Loading branch information
@Cyan4973
Cyan4973 authored Jan 21, 2022
2 parents 330c97d + 5595aec commit 71921e5
Show file tree
Hide file tree
Showing 2 changed files with 16 additions and 13 deletions.
9 changes: 6 additions & 3 deletions lib/compress/zstd_opt.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -255,11 +255,14 @@ static U32 ZSTD_rawLiteralsCost(const BYTE* const literals, U32 const litLength,
return (litLength*6) * BITCOST_MULTIPLIER; /* 6 bit per literal - no statistic used */

/* dynamic statistics */
{ U32 price = litLength * optPtr->litSumBasePrice;
{ U32 price = optPtr->litSumBasePrice * litLength;
U32 const litPriceMax = optPtr->litSumBasePrice - BITCOST_MULTIPLIER;
U32 u;
assert(optPtr->litSumBasePrice >= BITCOST_MULTIPLIER);
for (u=0; u < litLength; u++) {
assert(WEIGHT(optPtr->litFreq[literals[u]], optLevel) <= optPtr->litSumBasePrice); /* literal cost should never be negative */
price -= WEIGHT(optPtr->litFreq[literals[u]], optLevel);
U32 litPrice = WEIGHT(optPtr->litFreq[literals[u]], optLevel);
if (UNLIKELY(litPrice > litPriceMax)) litPrice = litPriceMax;
price -= litPrice;
}
return price;
}
Expand Down
20 changes: 10 additions & 10 deletions tests/regression/results.csv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ silesia, level 7, compress
silesia, level 9, compress cctx, 4543018
silesia, level 13, compress cctx, 4493990
silesia, level 16, compress cctx, 4359864
silesia, level 19, compress cctx, 4296880
silesia, level 19, compress cctx, 4296686
silesia, long distance mode, compress cctx, 4842075
silesia, multithreaded, compress cctx, 4842075
silesia, multithreaded long distance mode, compress cctx, 4842075
Expand All @@ -55,7 +55,7 @@ silesia, small hash log, compress
silesia, small chain log, compress cctx, 4912197
silesia, explicit params, compress cctx, 4794052
silesia, uncompressed literals, compress cctx, 4842075
silesia, uncompressed literals optimal, compress cctx, 4296880
silesia, uncompressed literals optimal, compress cctx, 4296686
silesia, huffman literals, compress cctx, 6172178
silesia, multithreaded with advanced params, compress cctx, 4842075
github, level -5, compress cctx, 204411
Expand Down Expand Up @@ -110,7 +110,7 @@ silesia, level 7, zstdcli,
silesia, level 9, zstdcli, 4543066
silesia, level 13, zstdcli, 4494038
silesia, level 16, zstdcli, 4359912
silesia, level 19, zstdcli, 4296928
silesia, level 19, zstdcli, 4296734
silesia, long distance mode, zstdcli, 4833785
silesia, multithreaded, zstdcli, 4842123
silesia, multithreaded long distance mode, zstdcli, 4833785
Expand Down Expand Up @@ -249,7 +249,7 @@ silesia, level 12 row 1, advanced
silesia, level 12 row 2, advanced one pass, 4503116
silesia, level 13, advanced one pass, 4493990
silesia, level 16, advanced one pass, 4359864
silesia, level 19, advanced one pass, 4296880
silesia, level 19, advanced one pass, 4296686
silesia, no source size, advanced one pass, 4842075
silesia, long distance mode, advanced one pass, 4833710
silesia, multithreaded, advanced one pass, 4842075
Expand Down Expand Up @@ -567,7 +567,7 @@ silesia, level 12 row 1, advanced
silesia, level 12 row 2, advanced one pass small out, 4503116
silesia, level 13, advanced one pass small out, 4493990
silesia, level 16, advanced one pass small out, 4359864
silesia, level 19, advanced one pass small out, 4296880
silesia, level 19, advanced one pass small out, 4296686
silesia, no source size, advanced one pass small out, 4842075
silesia, long distance mode, advanced one pass small out, 4833710
silesia, multithreaded, advanced one pass small out, 4842075
Expand Down Expand Up @@ -885,7 +885,7 @@ silesia, level 12 row 1, advanced
silesia, level 12 row 2, advanced streaming, 4503116
silesia, level 13, advanced streaming, 4493990
silesia, level 16, advanced streaming, 4359864
silesia, level 19, advanced streaming, 4296880
silesia, level 19, advanced streaming, 4296686
silesia, no source size, advanced streaming, 4842039
silesia, long distance mode, advanced streaming, 4833710
silesia, multithreaded, advanced streaming, 4842075
Expand Down Expand Up @@ -1195,10 +1195,10 @@ silesia, level 7, old stre
silesia, level 9, old streaming, 4543018
silesia, level 13, old streaming, 4493990
silesia, level 16, old streaming, 4359864
silesia, level 19, old streaming, 4296880
silesia, level 19, old streaming, 4296686
silesia, no source size, old streaming, 4842039
silesia, uncompressed literals, old streaming, 4842075
silesia, uncompressed literals optimal, old streaming, 4296880
silesia, uncompressed literals optimal, old streaming, 4296686
silesia, huffman literals, old streaming, 6179294
silesia.tar, level -5, old streaming, 7043687
silesia.tar, level -3, old streaming, 6671317
Expand Down Expand Up @@ -1297,7 +1297,7 @@ silesia, level 7, old stre
silesia, level 9, old streaming advanced, 4543018
silesia, level 13, old streaming advanced, 4493990
silesia, level 16, old streaming advanced, 4359864
silesia, level 19, old streaming advanced, 4296880
silesia, level 19, old streaming advanced, 4296686
silesia, no source size, old streaming advanced, 4842039
silesia, long distance mode, old streaming advanced, 4842075
silesia, multithreaded, old streaming advanced, 4842075
Expand All @@ -1307,7 +1307,7 @@ silesia, small hash log, old stre
silesia, small chain log, old streaming advanced, 4912197
silesia, explicit params, old streaming advanced, 4795452
silesia, uncompressed literals, old streaming advanced, 4842075
silesia, uncompressed literals optimal, old streaming advanced, 4296880
silesia, uncompressed literals optimal, old streaming advanced, 4296686
silesia, huffman literals, old streaming advanced, 6179294
silesia, multithreaded with advanced params, old streaming advanced, 4842075
silesia.tar, level -5, old streaming advanced, 7043687
Expand Down

0 comments on commit 71921e5

Please sign in to comment.