Skip to content

Commit

Permalink
Merge pull request #219 from Blosc/pr/217
Browse files Browse the repository at this point in the history 
Refuse to read future format versions
  • Loading branch information
@FrancescAlted
FrancescAlted authored Feb 22, 2018
2 parents b569be2 + 5066fc9 commit 759eb80
Show file tree
Hide file tree
Showing 3 changed files with 58 additions and 36 deletions.
2 changes: 1 addition & 1 deletion README_HEADER.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ All entries are little endian.
:bit 2 (``0x04``):
Whether the bit-shuffle filter has been applied or not.
:bit 3 (``0x08``):
Reserved
Reserved, must be zero.
:bit 4 (``0x10``):
If set, the blocks will not be split in sub-blocks during compression.
:bit 5 (``0x20``):
Expand Down
83 changes: 51 additions & 32 deletions blosc/blosc.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,10 +90,8 @@ struct blosc_context {

const uint8_t* src;
uint8_t* dest; /* The current pos in the destination buffer */
uint8_t* header_flags; /* Flags for header. Currently booked:
- 0: byte-shuffled?
- 1: memcpy'ed?
- 2: bit-shuffled? */
uint8_t* header_flags; /* Flags for header */
int compversion; /* Compressor version byte, only used during decompression */
int32_t sourcesize; /* Number of bytes in source buffer (or uncompressed bytes in compressed file) */
int32_t nblocks; /* Number of total blocks in buffer */
int32_t leftover; /* Extra bytes at end of buffer */
Expand Down Expand Up @@ -691,6 +689,7 @@ static int blosc_d(struct blosc_context* context, int32_t blocksize,
int doshuffle = (header_flags & BLOSC_DOSHUFFLE) && (typesize > 1);
int dobitshuffle = ((header_flags & BLOSC_DOBITSHUFFLE) &&
(blocksize >= typesize));
int compversion = context->compversion;

if (doshuffle || dobitshuffle) {
_tmp = tmp;
Expand Down Expand Up @@ -719,28 +718,48 @@ static int blosc_d(struct blosc_context* context, int32_t blocksize,
}
else {
if (compformat == BLOSC_BLOSCLZ_FORMAT) {
if (compversion != BLOSC_BLOSCLZ_VERSION_FORMAT) {
fprintf(stderr, "Unrecognized BloscLZ version %d\n", compversion);
return -9;
}
nbytes = blosclz_decompress(src, cbytes, _tmp, neblock);
}
#if defined(HAVE_LZ4)
else if (compformat == BLOSC_LZ4_FORMAT) {
if (compversion != BLOSC_LZ4_VERSION_FORMAT) {
fprintf(stderr, "Unrecognized LZ4 version %d\n", compversion);
return -9;
}
nbytes = lz4_wrap_decompress((char *)src, (size_t)cbytes,
(char*)_tmp, (size_t)neblock);
}
#endif /* HAVE_LZ4 */
#if defined(HAVE_SNAPPY)
else if (compformat == BLOSC_SNAPPY_FORMAT) {
if (compversion != BLOSC_SNAPPY_VERSION_FORMAT) {
fprintf(stderr, "Unrecognized Snappy version %d\n", compversion);
return -9;
}
nbytes = snappy_wrap_decompress((char *)src, (size_t)cbytes,
(char*)_tmp, (size_t)neblock);
}
#endif /* HAVE_SNAPPY */
#if defined(HAVE_ZLIB)
else if (compformat == BLOSC_ZLIB_FORMAT) {
if (compversion != BLOSC_ZLIB_VERSION_FORMAT) {
fprintf(stderr, "Unrecognized Zlib version %d\n", compversion);
return -9;
}
nbytes = zlib_wrap_decompress((char *)src, (size_t)cbytes,
(char*)_tmp, (size_t)neblock);
}
#endif /* HAVE_ZLIB */
#if defined(HAVE_ZSTD)
else if (compformat == BLOSC_ZSTD_FORMAT) {
if (compversion != BLOSC_ZSTD_VERSION_FORMAT) {
fprintf(stderr, "Unrecognized Zstd version %d\n", compversion);
return -9;
}
nbytes = zstd_wrap_decompress((char*)src, (size_t)cbytes,
(char*)_tmp, (size_t)neblock);
}
Expand Down Expand Up @@ -1397,18 +1416,21 @@ int blosc_run_decompression_with_context(struct blosc_context* context,

/* Read the header block */
version = context->src[0]; /* blosc format version */
versionlz = context->src[1]; /* blosclz format version */
context->compversion = context->src[1];

context->header_flags = (uint8_t*)(context->src + 2); /* flags */
context->typesize = (int32_t)context->src[3]; /* typesize */
context->sourcesize = sw32_(context->src + 4); /* buffer size */
context->blocksize = sw32_(context->src + 8); /* block size */
ctbytes = sw32_(context->src + 12); /* compressed buffer size */

/* Unused values */
version += 0; /* shut up compiler warning */
versionlz += 0; /* shut up compiler warning */
ctbytes += 0; /* shut up compiler warning */
if (version != BLOSC_VERSION_FORMAT) {
/* Version from future */
return -1;
}
if (*context->header_flags & 0x08) {
/* compressor flags from the future */
return -1;
}

context->bstarts = (uint8_t*)(context->src + 16);
/* Compute some params */
Expand Down Expand Up @@ -1497,7 +1519,7 @@ int blosc_decompress(const void *src, void *dest, size_t destsize)
int blosc_getitem(const void *src, int start, int nitems, void *dest)
{
uint8_t *_src=NULL; /* current pos for source buffer */
uint8_t version, versionlz; /* versions for compressed header */
uint8_t version, compversion; /* versions for compressed header */
uint8_t flags; /* flags for header */
int32_t ntbytes = 0; /* the number of uncompressed bytes */
int32_t nblocks; /* number of total blocks in buffer */
Expand All @@ -1516,22 +1538,21 @@ int blosc_getitem(const void *src, int start, int nitems, void *dest)

/* Read the header block */
version = _src[0]; /* blosc format version */
versionlz = _src[1]; /* blosclz format version */
compversion = _src[1];
flags = _src[2]; /* flags */
typesize = (int32_t)_src[3]; /* typesize */
nbytes = sw32_(_src + 4); /* buffer size */
blocksize = sw32_(_src + 8); /* block size */
ctbytes = sw32_(_src + 12); /* compressed buffer size */

if (version != BLOSC_VERSION_FORMAT)
return -9;

ebsize = blocksize + typesize * (int32_t)sizeof(int32_t);
tmp = my_malloc(blocksize + ebsize + blocksize);
tmp2 = tmp + blocksize;
tmp3 = tmp + blocksize + ebsize;

version += 0; /* shut up compiler warning */
versionlz += 0; /* shut up compiler warning */
ctbytes += 0; /* shut up compiler warning */

_src += 16;
bstarts = _src;
/* Compute some params */
Expand Down Expand Up @@ -1582,10 +1603,11 @@ int blosc_getitem(const void *src, int start, int nitems, void *dest)
cbytes = bsize2;
}
else {
struct blosc_context context;
/* blosc_d only uses typesize and flags */
struct blosc_context context = {0};
/* Only initialize the fields blosc_d uses */
context.typesize = typesize;
context.header_flags = &flags;
context.compversion = compversion;

/* Regular decompression. Put results in tmp2. */
cbytes = blosc_d(&context, bsize, leftoverblock,
Expand Down Expand Up @@ -2009,14 +2031,12 @@ void blosc_cbuffer_sizes(const void *cbuffer, size_t *nbytes,
size_t *cbytes, size_t *blocksize)
{
uint8_t *_src = (uint8_t *)(cbuffer); /* current pos for source buffer */
uint8_t version, versionlz; /* versions for compressed header */
uint8_t version = _src[0]; /* version of header */

/* Read the version info (could be useful in the future) */
version = _src[0]; /* blosc format version */
versionlz = _src[1]; /* blosclz format version */

version += 0; /* shut up compiler warning */
versionlz += 0; /* shut up compiler warning */
if (version != BLOSC_VERSION_FORMAT) {
*nbytes = *blocksize = *cbytes = 0;
return;
}

/* Read the interesting values */
*nbytes = (size_t)sw32_(_src + 4); /* uncompressed buffer size */
Expand All @@ -2030,17 +2050,16 @@ void blosc_cbuffer_metainfo(const void *cbuffer, size_t *typesize,
int *flags)
{
uint8_t *_src = (uint8_t *)(cbuffer); /* current pos for source buffer */
uint8_t version, versionlz; /* versions for compressed header */

/* Read the version info (could be useful in the future) */
version = _src[0]; /* blosc format version */
versionlz = _src[1]; /* blosclz format version */
uint8_t version = _src[0]; /* version of header */

version += 0; /* shut up compiler warning */
versionlz += 0; /* shut up compiler warning */
if (version != BLOSC_VERSION_FORMAT) {
*flags = *typesize = 0;
return;
}

/* Read the interesting values */
*flags = (int)_src[2]; /* flags */
*flags = (int)_src[2] & 7; /* first three flags */
*typesize = (size_t)_src[3]; /* typesize */
}

Expand Down
9 changes: 6 additions & 3 deletions blosc/blosc.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -413,7 +413,8 @@ BLOSC_EXPORT int blosc_free_resources(void);
You only need to pass the first BLOSC_MIN_HEADER_LENGTH bytes of a
compressed buffer for this call to work.
This function should always succeed.
If the format is not supported by the library, all output arguments will be
filled with zeros.
*/
BLOSC_EXPORT void blosc_cbuffer_sizes(const void *cbuffer, size_t *nbytes,
size_t *cbytes, size_t *blocksize);
Expand All @@ -423,16 +424,18 @@ BLOSC_EXPORT void blosc_cbuffer_sizes(const void *cbuffer, size_t *nbytes,
Return information about a compressed buffer, namely the type size
(`typesize`), as well as some internal `flags`.
The `flags` is a set of bits, where the currently used ones are:
The `flags` is a set of bits, where the used ones are:
* bit 0: whether the shuffle filter has been applied or not
* bit 1: whether the internal buffer is a pure memcpy or not
* bit 2: whether the bit shuffle filter has been applied or not
You can use the `BLOSC_DOSHUFFLE`, `BLOSC_DOBITSHUFFLE` and
`BLOSC_MEMCPYED` symbols for extracting the interesting bits
(e.g. ``flags & BLOSC_DOSHUFFLE`` says whether the buffer is
byte-shuffled or not).
This function should always succeed.
If the format is not supported by the library, all output arguments will be
filled with zeros.
*/
BLOSC_EXPORT void blosc_cbuffer_metainfo(const void *cbuffer, size_t *typesize,
int *flags);
Expand Down

0 comments on commit 759eb80

Please sign in to comment.