diff --git a/README_HEADER.rst b/README_HEADER.rst index 5acfd7d9..0f97804d 100644 --- a/README_HEADER.rst +++ b/README_HEADER.rst @@ -31,7 +31,7 @@ All entries are little endian. :bit 2 (``0x04``): Whether the bit-shuffle filter has been applied or not. :bit 3 (``0x08``): - Reserved + Reserved, must be zero. :bit 4 (``0x10``): If set, the blocks will not be split in sub-blocks during compression. :bit 5 (``0x20``): diff --git a/blosc/blosc.c b/blosc/blosc.c index 4b3e6b55..23791c84 100644 --- a/blosc/blosc.c +++ b/blosc/blosc.c @@ -90,10 +90,8 @@ struct blosc_context { const uint8_t* src; uint8_t* dest; /* The current pos in the destination buffer */ - uint8_t* header_flags; /* Flags for header. Currently booked: - - 0: byte-shuffled? - - 1: memcpy'ed? - - 2: bit-shuffled? */ + uint8_t* header_flags; /* Flags for header */ + int compversion; /* Compressor version byte, only used during decompression */ int32_t sourcesize; /* Number of bytes in source buffer (or uncompressed bytes in compressed file) */ int32_t nblocks; /* Number of total blocks in buffer */ int32_t leftover; /* Extra bytes at end of buffer */ @@ -691,6 +689,7 @@ static int blosc_d(struct blosc_context* context, int32_t blocksize, int doshuffle = (header_flags & BLOSC_DOSHUFFLE) && (typesize > 1); int dobitshuffle = ((header_flags & BLOSC_DOBITSHUFFLE) && (blocksize >= typesize)); + int compversion = context->compversion; if (doshuffle || dobitshuffle) { _tmp = tmp; @@ -719,28 +718,48 @@ static int blosc_d(struct blosc_context* context, int32_t blocksize, } else { if (compformat == BLOSC_BLOSCLZ_FORMAT) { + if (compversion != BLOSC_BLOSCLZ_VERSION_FORMAT) { + fprintf(stderr, "Unrecognized BloscLZ version %d\n", compversion); + return -9; + } nbytes = blosclz_decompress(src, cbytes, _tmp, neblock); } #if defined(HAVE_LZ4) else if (compformat == BLOSC_LZ4_FORMAT) { + if (compversion != BLOSC_LZ4_VERSION_FORMAT) { + fprintf(stderr, "Unrecognized LZ4 version %d\n", compversion); + return -9; + } nbytes = lz4_wrap_decompress((char *)src, (size_t)cbytes, (char*)_tmp, (size_t)neblock); } #endif /* HAVE_LZ4 */ #if defined(HAVE_SNAPPY) else if (compformat == BLOSC_SNAPPY_FORMAT) { + if (compversion != BLOSC_SNAPPY_VERSION_FORMAT) { + fprintf(stderr, "Unrecognized Snappy version %d\n", compversion); + return -9; + } nbytes = snappy_wrap_decompress((char *)src, (size_t)cbytes, (char*)_tmp, (size_t)neblock); } #endif /* HAVE_SNAPPY */ #if defined(HAVE_ZLIB) else if (compformat == BLOSC_ZLIB_FORMAT) { + if (compversion != BLOSC_ZLIB_VERSION_FORMAT) { + fprintf(stderr, "Unrecognized Zlib version %d\n", compversion); + return -9; + } nbytes = zlib_wrap_decompress((char *)src, (size_t)cbytes, (char*)_tmp, (size_t)neblock); } #endif /* HAVE_ZLIB */ #if defined(HAVE_ZSTD) else if (compformat == BLOSC_ZSTD_FORMAT) { + if (compversion != BLOSC_ZSTD_VERSION_FORMAT) { + fprintf(stderr, "Unrecognized Zstd version %d\n", compversion); + return -9; + } nbytes = zstd_wrap_decompress((char*)src, (size_t)cbytes, (char*)_tmp, (size_t)neblock); } @@ -1397,18 +1416,21 @@ int blosc_run_decompression_with_context(struct blosc_context* context, /* Read the header block */ version = context->src[0]; /* blosc format version */ - versionlz = context->src[1]; /* blosclz format version */ + context->compversion = context->src[1]; context->header_flags = (uint8_t*)(context->src + 2); /* flags */ context->typesize = (int32_t)context->src[3]; /* typesize */ context->sourcesize = sw32_(context->src + 4); /* buffer size */ context->blocksize = sw32_(context->src + 8); /* block size */ - ctbytes = sw32_(context->src + 12); /* compressed buffer size */ - /* Unused values */ - version += 0; /* shut up compiler warning */ - versionlz += 0; /* shut up compiler warning */ - ctbytes += 0; /* shut up compiler warning */ + if (version != BLOSC_VERSION_FORMAT) { + /* Version from future */ + return -1; + } + if (*context->header_flags & 0x08) { + /* compressor flags from the future */ + return -1; + } context->bstarts = (uint8_t*)(context->src + 16); /* Compute some params */ @@ -1497,7 +1519,7 @@ int blosc_decompress(const void *src, void *dest, size_t destsize) int blosc_getitem(const void *src, int start, int nitems, void *dest) { uint8_t *_src=NULL; /* current pos for source buffer */ - uint8_t version, versionlz; /* versions for compressed header */ + uint8_t version, compversion; /* versions for compressed header */ uint8_t flags; /* flags for header */ int32_t ntbytes = 0; /* the number of uncompressed bytes */ int32_t nblocks; /* number of total blocks in buffer */ @@ -1516,22 +1538,21 @@ int blosc_getitem(const void *src, int start, int nitems, void *dest) /* Read the header block */ version = _src[0]; /* blosc format version */ - versionlz = _src[1]; /* blosclz format version */ + compversion = _src[1]; flags = _src[2]; /* flags */ typesize = (int32_t)_src[3]; /* typesize */ nbytes = sw32_(_src + 4); /* buffer size */ blocksize = sw32_(_src + 8); /* block size */ ctbytes = sw32_(_src + 12); /* compressed buffer size */ + if (version != BLOSC_VERSION_FORMAT) + return -9; + ebsize = blocksize + typesize * (int32_t)sizeof(int32_t); tmp = my_malloc(blocksize + ebsize + blocksize); tmp2 = tmp + blocksize; tmp3 = tmp + blocksize + ebsize; - version += 0; /* shut up compiler warning */ - versionlz += 0; /* shut up compiler warning */ - ctbytes += 0; /* shut up compiler warning */ - _src += 16; bstarts = _src; /* Compute some params */ @@ -1582,10 +1603,11 @@ int blosc_getitem(const void *src, int start, int nitems, void *dest) cbytes = bsize2; } else { - struct blosc_context context; - /* blosc_d only uses typesize and flags */ + struct blosc_context context = {0}; + /* Only initialize the fields blosc_d uses */ context.typesize = typesize; context.header_flags = &flags; + context.compversion = compversion; /* Regular decompression. Put results in tmp2. */ cbytes = blosc_d(&context, bsize, leftoverblock, @@ -2009,14 +2031,12 @@ void blosc_cbuffer_sizes(const void *cbuffer, size_t *nbytes, size_t *cbytes, size_t *blocksize) { uint8_t *_src = (uint8_t *)(cbuffer); /* current pos for source buffer */ - uint8_t version, versionlz; /* versions for compressed header */ + uint8_t version = _src[0]; /* version of header */ - /* Read the version info (could be useful in the future) */ - version = _src[0]; /* blosc format version */ - versionlz = _src[1]; /* blosclz format version */ - - version += 0; /* shut up compiler warning */ - versionlz += 0; /* shut up compiler warning */ + if (version != BLOSC_VERSION_FORMAT) { + *nbytes = *blocksize = *cbytes = 0; + return; + } /* Read the interesting values */ *nbytes = (size_t)sw32_(_src + 4); /* uncompressed buffer size */ @@ -2030,17 +2050,16 @@ void blosc_cbuffer_metainfo(const void *cbuffer, size_t *typesize, int *flags) { uint8_t *_src = (uint8_t *)(cbuffer); /* current pos for source buffer */ - uint8_t version, versionlz; /* versions for compressed header */ - /* Read the version info (could be useful in the future) */ - version = _src[0]; /* blosc format version */ - versionlz = _src[1]; /* blosclz format version */ + uint8_t version = _src[0]; /* version of header */ - version += 0; /* shut up compiler warning */ - versionlz += 0; /* shut up compiler warning */ + if (version != BLOSC_VERSION_FORMAT) { + *flags = *typesize = 0; + return; + } /* Read the interesting values */ - *flags = (int)_src[2]; /* flags */ + *flags = (int)_src[2] & 7; /* first three flags */ *typesize = (size_t)_src[3]; /* typesize */ } diff --git a/blosc/blosc.h b/blosc/blosc.h index 487faa6e..9ed25424 100644 --- a/blosc/blosc.h +++ b/blosc/blosc.h @@ -413,7 +413,8 @@ BLOSC_EXPORT int blosc_free_resources(void); You only need to pass the first BLOSC_MIN_HEADER_LENGTH bytes of a compressed buffer for this call to work. - This function should always succeed. + If the format is not supported by the library, all output arguments will be + filled with zeros. */ BLOSC_EXPORT void blosc_cbuffer_sizes(const void *cbuffer, size_t *nbytes, size_t *cbytes, size_t *blocksize); @@ -423,16 +424,18 @@ BLOSC_EXPORT void blosc_cbuffer_sizes(const void *cbuffer, size_t *nbytes, Return information about a compressed buffer, namely the type size (`typesize`), as well as some internal `flags`. - The `flags` is a set of bits, where the currently used ones are: + The `flags` is a set of bits, where the used ones are: * bit 0: whether the shuffle filter has been applied or not * bit 1: whether the internal buffer is a pure memcpy or not + * bit 2: whether the bit shuffle filter has been applied or not You can use the `BLOSC_DOSHUFFLE`, `BLOSC_DOBITSHUFFLE` and `BLOSC_MEMCPYED` symbols for extracting the interesting bits (e.g. ``flags & BLOSC_DOSHUFFLE`` says whether the buffer is byte-shuffled or not). - This function should always succeed. + If the format is not supported by the library, all output arguments will be + filled with zeros. */ BLOSC_EXPORT void blosc_cbuffer_metainfo(const void *cbuffer, size_t *typesize, int *flags);