From aa03f38e9b4ec6148a750c5cad9c1b0e3cb9629c Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Mon, 1 Nov 2021 02:44:03 -0400 Subject: [PATCH 01/28] Add bzip2, lz4, and zstd filters from HDF5Plugins.jl --- Project.toml | 8 +- src/HDF5.jl | 2 +- src/filters/H5Zbzip2.jl | 199 ++++++++++++++++++++++++++++ src/filters/H5Zlz4.jl | 221 ++++++++++++++++++++++++++++++++ src/filters/H5Zlz_LICENSE.txt | 43 +++++++ src/filters/H5Zzstd.jl | 120 +++++++++++++++++ src/filters/H5Zzstd_LICENSE.txt | 201 +++++++++++++++++++++++++++++ src/filters/H5bzip2_LICENSE.txt | 29 +++++ src/filters/TestUtils.jl | 80 ++++++++++++ src/filters/filters.jl | 29 ++++- 10 files changed, 929 insertions(+), 3 deletions(-) create mode 100644 src/filters/H5Zbzip2.jl create mode 100644 src/filters/H5Zlz4.jl create mode 100644 src/filters/H5Zlz_LICENSE.txt create mode 100644 src/filters/H5Zzstd.jl create mode 100644 src/filters/H5Zzstd_LICENSE.txt create mode 100644 src/filters/H5bzip2_LICENSE.txt create mode 100644 src/filters/TestUtils.jl diff --git a/Project.toml b/Project.toml index 40ecdeddf..9a9e17c71 100644 --- a/Project.toml +++ b/Project.toml @@ -1,9 +1,12 @@ name = "HDF5" uuid = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f" -version = "0.15.5" +version = "0.16.0" [deps] Blosc = "a74b3585-a348-5f62-a45c-50e91977d574" +CodecBzip2 = "523fee87-0ab8-5b00-afb7-3ecf72e48cfd" +CodecLz4 = "5ba52731-8f18-5e0d-9241-30f10d1ec561" +CodecZstd = "6b39b394-51ab-5f42-8807-6242bab2b4c2" Compat = "34da2185-b29b-5c13-b0c7-acf172513d20" HDF5_jll = "0234f1f7-429e-5d53-9886-15a909be8d59" Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" @@ -12,6 +15,9 @@ Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Requires = "ae029012-a4dd-5104-9daa-d747884805df" [compat] +CodecBzip2 = "0.7" +CodecLz4 = "0.4" +CodecZstd = "0.7" Blosc = "0.7.1" Compat = "3.1.0" HDF5_jll = "~1.10.5, ~1.12.0" diff --git a/src/HDF5.jl b/src/HDF5.jl index c0e647ab2..724ea89d1 100644 --- a/src/HDF5.jl +++ b/src/HDF5.jl @@ -1593,7 +1593,7 @@ function __init__() ENV["HDF5_USE_FILE_LOCKING"] = "FALSE" end - Filters.register_blosc() + Filters.register() # use our own error handling machinery (i.e. turn off automatic error printing) API.h5e_set_auto(API.H5E_DEFAULT, C_NULL, C_NULL) diff --git a/src/filters/H5Zbzip2.jl b/src/filters/H5Zbzip2.jl new file mode 100644 index 000000000..a207b8f9f --- /dev/null +++ b/src/filters/H5Zbzip2.jl @@ -0,0 +1,199 @@ +# The code below has been ported to Julia from the original C: +# https://github.com/nexusformat/HDF5-External-Filter-Plugins/blob/master/BZIP2/src/H5Zbzip2.c +# The filter function H5Z_filter_bzip2 was adopted from +# PyTables http://www.pytables.org. +# The plugin can be used with the HDF5 library vesrion 1.8.11+ to read +# HDF5 datasets compressed with bzip2 created by PyTables. +# See H5Zbzip2_LICENSE.txt for the license. + +# The following copyright and license applies to the Julia port itself. +# Copyright © 2021 Mark Kittisopikul, Howard Hughes Medical Institute +# Licensed under MIT License, see LICENSE +module H5Zbzip2 + +using ..API +using CodecBzip2 +import CodecBzip2: libbzip2 +import ..TestUtils: test_filter +import ..Filters: FILTERS, Filter, filterid + +export H5Z_FILTER_BZIP2, H5Z_filter_bzip2, register_bzip2 + + +const H5Z_FILTER_BZIP2 = API.H5Z_filter_t(307) +const bzip2_name = "HDF5 bzip2 filter; see http://www.hdfgroup.org/services/contributions.html" + +function H5Z_filter_bzip2(flags::Cuint, cd_nelmts::Csize_t, + cd_values::Ptr{Cuint}, nbytes::Csize_t, + buf_size::Ptr{Csize_t}, buf::Ptr{Ptr{Cvoid}}) + outbuf = C_NULL + outdatalen = Cuint(0) + + # Prepare the output buffer + + try + + if flags & API.H5Z_FLAG_REVERSE != 0 + # Decompress + + outbuflen = nbytes * 3 + 1 + outbuf = Libc.malloc(outbuflen) + if outbuf == C_NULL + error("H5Zbzip2: memory allocation failed for bzip2 decompression.") + end + + stream = CodecBzip2.BZStream() + # Just use default malloc and free + stream.bzalloc = C_NULL + stream.bzfree = C_NULL + # BZ2_bzDecompressInit + ret = CodecBzip2.decompress_init!(stream, 0, false) + if ret != CodecBzip2.BZ_OK + errror("H5Zbzip2: bzip2 decompress start failed with error $ret.") + end + + stream.next_out = outbuf + stream.avail_out = outbuflen + stream.next_in = unsafe_load(buf) + stream.avail_in = nbytes + + cont = true + + while cont + # BZ2_bzDecompress + ret = CodecBzip2.decompress!(stream) + if ret < 0 + error("H5Zbzip2: bzip2 decompression failed with error $ret.") + end + cont = ret != CodecBzip2.BZ_STREAM_END + if cont && stream.avail_out == 0 + # Grow the output buffer + newbuflen = outbuflen * 2 + newbuf = Libc.realloc(outbuf, newbuflen) + if newbuf == C_NULL + error("H5Zbzip2: memory allocation failed for bzip2 decompression.") + end + stream.next_out = newbuf + outbuflen + stream.avail_out = outbuflen + outbuf = newbuf + outbuflen = newbuflen + end + end + + outdatalen = stream.total_out_lo32 + # BZ2_bzDecompressEnd + ret = CodecBzip2.decompress_end!(stream) + if ret != CodecBzip2.BZ_OK + error("H5Zbzip2: bzip2 compression end failed with error $ret.") + end + else + # Compress data + + # Maybe not the same size as outdatalen + odatalen = Cuint(0) + blockSize100k = 9 + + # Get compression blocksize if present + if cd_nelmts > 0 + blockSize100k = unsafe_load(cd_values) + if blockSize100k < 1 || blockSize100k > 9 + error("H5Zbzip2: Invalid compression blocksize: $blockSize100k") + end + end + + # Prepare the output buffer + outbuflen = nbytes + nbytes ÷ 100 + 600 # worse case (bzip2 docs) + outbuf = Libc.malloc(outbuflen) + @debug "Allocated" outbuflen outbuf + if outbuf == C_NULL + error("H5Zbzip2: Memory allocation failed for bzip2 compression") + end + + # Compress data + odatalen = outbuflen + r_odatalen = Ref{Cuint}(odatalen) + ret = BZ2_bzBuffToBuffCompress(outbuf, r_odatalen, unsafe_load(buf), nbytes, + blockSize100k, 0, 0) + outdatalen = r_odatalen[] + if ret != CodecBzip2.BZ_OK + error("H5Zbzip2: bzip2 compression failed with error $ret.") + end + end # if flags & API.H5Z_FLAG_REVERSE != 0 + Libc.free(unsafe_load(buf)) + unsafe_store!(buf, outbuf) + unsafe_store!(buf_size, outbuflen) + + catch err + if outbuf != C_NULL + Libc.free(outbuf) + end + rethrow(err) + outdatalen = Csize_t(0) + end # try - catch + + return Csize_t(outdatalen) +end # function H5Z_filter_bzip2 + +function register_bzip2() + c_bzip2_filter = @cfunction(H5Z_filter_bzip2, Csize_t, + (Cuint, Csize_t, Ptr{Cuint}, Csize_t, + Ptr{Csize_t}, Ptr{Ptr{Cvoid}})) + API.h5z_register(API.H5Z_class_t( + API.H5Z_CLASS_T_VERS, + H5Z_FILTER_BZIP2, + 1, + 1, + pointer(bzip2_name), + C_NULL, + C_NULL, + c_bzip2_filter + )) + return nothing +end + + +# Need stdcall for 32-bit Windows? +function BZ2_bzBuffToBuffCompress(dest, destLen, source, sourceLen, blockSize100k, verbosity, workFactor) + return ccall( + (:BZ2_bzBuffToBuffCompress, libbzip2), + Cint, + (Ptr{Cchar}, Ptr{Cuint}, Ptr{Cchar}, Cuint, Cint, Cint, Cint), + dest, + destLen, + source, + sourceLen, + blockSize100k, + verbosity, + workFactor + ) +end + +function BZ2_bzBuffToBuffDecompress(dest, destLen, source, sourceLen, small, verbosity) + return ccall( + (:BZ2_bzBuffToBuffDecompress, libbzip2), + Cint, + (Ptr{Cchar}, Ptr{Cuint}, Ptr{Cchar}, Cuint, Cint, Cint), + dest, + destLen, + source, + sourceLen, + small, + verbosity + ) +end + +function test_bzip2_filter(data = ones(UInt8, 1024)) + cd_values = Cuint[8] + test_filter(H5Z_filter_bzip2; cd_values = cd_values, data = data) +end + +# Filters Module + +struct Bzip2Filter <: Filter + blockSize100k::Cuint +end + +filterid(::Type{Bzip2Filter}) = H5Z_FILTER_BZIP2 +FILTERS[H5Z_FILTER_BZIP2] = Bzip2Filter + +end # module H5Zbzip2 \ No newline at end of file diff --git a/src/filters/H5Zlz4.jl b/src/filters/H5Zlz4.jl new file mode 100644 index 000000000..af449218a --- /dev/null +++ b/src/filters/H5Zlz4.jl @@ -0,0 +1,221 @@ +#= +This is a port of H5Zlz4.c to Julia +https://github.com/HDFGroup/hdf5_plugins/blob/master/LZ4/src/H5Zlz4.c +https://github.com/nexusformat/HDF5-External-Filter-Plugins/blob/master/LZ4/src/H5Zlz4.c +https://github.com/silx-kit/hdf5plugin/blob/main/src/LZ4/H5Zlz4.c + +H5Zlz4 is originally a copyright of HDF Group. +See "LZ4 filter plugin license" in H5Zlz4_LICENSE.txt + +The following copyright and license applies to the Julia port itself. +Copyright © 2021 Mark Kittisopikul, Howard Hughes Medical Institute +Licensed under MIT License, see LICENSE +=# +module H5Zlz4 + +using ..API +using CodecLz4 +import ..TestUtils: test_filter +import ..Filters: FILTERS, Filter, filterid + +export H5Z_FILTER_LZ4, H5Z_filter_lz4, register_lz4 + +const H5Z_FILTER_LZ4 = API.H5Z_filter_t(32004) + +const DEFAULT_BLOCK_SIZE = 1 << 30; +const lz4_name = "HDF5 lz4 filter; see http://www.hdfgroup.org/services/contributions.html" + +const LZ4_AGGRESSION = Ref(1) + + + +# flags H5Z_FLAG_REVERSE or H5Z_FLAG_OPTIONAL +# cd_nelmts number of elements in cd_values +# cd_values +# nbytes - number of valid bytes of data +# buf_size - total size of buffer +# buf - pointer to pointer of data +function H5Z_filter_lz4(flags::Cuint, cd_nelmts::Csize_t, + cd_values::Ptr{Cuint}, nbytes::Csize_t, + buf_size::Ptr{Csize_t}, buf::Ptr{Ptr{Cvoid}}) + + outBuf = C_NULL + ret_value = Csize_t(0) + + try + + if (flags & API.H5Z_FLAG_REVERSE) != 0 # reverse filter, decompressing + #i32Buf = Ref{UInt32}() + blockSize = UInt32(0) + roBuf = Ref{UInt8}() + rpos = Ptr{UInt8}(unsafe_load(buf)) + #i64Buf = Ptr{UInt64}(rpos) + # Load the first 8 bytes from buffer as a big endian UInt64 + # This is the original size of the buffer + origSize = ntoh(unsafe_load(Ptr{UInt64}(rpos))) + rpos += 8 # advance the pointer + + # Next read the next four bytes from teh buffer as a big endian UInt32 + # This is the blocksize + #i32Buf[] = rpos + blockSize = ntoh(unsafe_load(Ptr{UInt32}(rpos))) + rpos += 4 + if blockSize > origSize + blockSize = origSize + end + + # malloc a byte buffer of origSize + # outBuf = Vector{UInt8}(undef, origSize) + @debug "OrigSize" origSize + outBuf = Libc.malloc(origSize) + # Julia should throw an error if it cannot allocate this + roBuf = Ptr{UInt8}(outBuf) + decompSize = 0 + # Start with the first blockSize + while decompSize < origSize + # compressedBlockSize = UInt32(0) + if origSize - decompSize < blockSize # the lsat block can be smaller than block size + blockSize = origSize - decompSize + end + + #i32Buf[] = rpos + compressedBlockSize = ntoh(unsafe_load(Ptr{UInt32}(rpos))) + rpos += 4 + + if compressedBlockSize == blockSize + # There was no compression + # memcpy(roBuf, rpos, blockSize) + unsafe_copyto!(roBuf, rpos, blockSize) + decompressedBytes = blockSize + else + # do the compression + # LZ4_decompress_fast, version number 10300 ? + @debug "decompress_safe" rpos roBuf compressedBlockSize (origSize - decompSize) + decompressedBytes = CodecLz4.LZ4_decompress_safe(rpos, roBuf, compressedBlockSize, origSize -decompSize) + @debug "decompressedBytes" decompressedBytes + end + + rpos += compressedBlockSize + roBuf += blockSize + decompSize += decompressedBytes + end + Libc.free(unsafe_load(buf)) + unsafe_store!(buf, outBuf) + outBuf = C_NULL + ret_value = Csize_t(origSize) + else + # forward filter + # compressing + #i64Buf = Ref{UInt64}() + #i32Buf = Ref{UInt32}() + + if nbytes > typemax(Int32) + error("Can only compress chunks up to 2GB") + end + blockSize = unsafe_load(cd_values) + if cd_nelmts > 0 && blockSize > 0 + else + blockSize = DEFAULT_BLOCK_SIZE + end + if blockSize > nbytes + blockSize = nbytes + end + nBlocks = (nbytes-1) ÷ blockSize + 1 + maxDestSize = nBlocks * CodecLz4.LZ4_compressBound(blockSize) + 4 + 8 + nBlocks*4 + outBuf = Libc.malloc(maxDestSize) + + rpos = Ptr{UInt8}(unsafe_load(buf)) + roBuf = Ptr{UInt8}(outBuf) + + # Header + unsafe_store!(Ptr{UInt64}(roBuf), hton(UInt64(nbytes))) + roBuf += 8 + + unsafe_store!(Ptr{UInt32}(roBuf), hton(UInt32(blockSize))) + roBuf += 4 + + outSize = 12 + + for block = 0:nBlocks-1 + # compBlockSize::UInt32 + origWritten = Csize_t(block*blockSize) + if nbytes - origWritten < blockSize # the last block may be < blockSize + blockSize = nbytes - origWritten + end + + # aggression = 1 is the same LZ4_compress_default + @debug "LZ4_compress_fast args" rpos outBuf roBuf roBuf+4 blockSize nBlocks CodecLz4.LZ4_compressBound(blockSize) + compBlockSize = UInt32(CodecLz4.LZ4_compress_fast(rpos, roBuf+4, blockSize, CodecLz4.LZ4_compressBound(blockSize), LZ4_AGGRESSION[])) + @debug "Compressed block size" compBlockSize + + if compBlockSize == 0 + error("Could not compress block $block") + end + + if compBlockSize >= blockSize # compression did not save any space, do a memcpy instead + compBlockSize = blockSize + unsafe_copyto!(roBuf+4, rpos, blockSize) + end + + unsafe_store!(Ptr{UInt32}(roBuf), hton(UInt32(compBlockSize))) # write blocksize + roBuf += 4 + + rpos += blockSize + roBuf += compBlockSize + outSize += compBlockSize + 4 + end + + Libc.free(unsafe_load(buf)) + unsafe_store!(buf, outBuf) + unsafe_store!(buf_size, outSize) + outBuf = C_NULL + ret_value = Csize_t(outSize) + end # (flags & API.H5Z_FLAG_REVERSE) != 0 + + catch err + println("ERROR: ", err) + rethrow() + #println(stacktrace(catch_backtrace())) + ret_value = Csize_t(0) + finally + if outBuf != C_NULL + Libc.free(outBuf) + end + end + return Csize_t(ret_value) +end + +function test_lz4_filter() + cd_values = Cuint[1024] + test_filter(H5Z_filter_lz4; cd_values = cd_values) +end + +function register_lz4(; aggression = 1) + LZ4_AGGRESSION[] = aggression + c_lz4_filter = @cfunction(H5Z_filter_lz4, Csize_t, + (Cuint, Csize_t, Ptr{Cuint}, Csize_t, + Ptr{Csize_t}, Ptr{Ptr{Cvoid}})) + API.h5z_register(API.H5Z_class_t( + API.H5Z_CLASS_T_VERS, + H5Z_FILTER_LZ4, + 1, + 1, + pointer(lz4_name), + C_NULL, + C_NULL, + c_lz4_filter + )) + + return nothing +end + +# Filters Module + +struct Lz4Filter <: Filter + blockSize::Cuint +end + +filterid(::Type{Lz4Filter}) = H5Z_FILTER_LZ4 +FILTERS[H5Z_FILTER_LZ4] = Lz4Filter + +end \ No newline at end of file diff --git a/src/filters/H5Zlz_LICENSE.txt b/src/filters/H5Zlz_LICENSE.txt new file mode 100644 index 000000000..3257bb370 --- /dev/null +++ b/src/filters/H5Zlz_LICENSE.txt @@ -0,0 +1,43 @@ +================================================================================ +LZ4 filter plugin license +================================================================================ + +Copyright Notice and License Terms for +HDF5 LZ4 compression filter plugin +----------------------------------------------------------------------------- + +HDF5 LZ4 compression filter plugin +Copyright 2013-2015 by The HDF Group. + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted for any purpose (including commercial purposes) +provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or materials provided with the distribution. + +3. In addition, redistributions of modified forms of the source or binary + code must carry prominent notices stating that the original code was + changed and the date of the change. + +4. All publications or advertising materials mentioning features or use of + this software are asked, but not required, to acknowledge that it was + developed by The HDF Group and credit the contributors. + +5. Neither the name of The HDF Group, nor the name of any Contributor may + be used to endorse or promote products derived from this software + without specific prior written permission from The HDF Group or the + Contributor, respectively. + +DISCLAIMER: +THIS SOFTWARE IS PROVIDED BY THE HDF GROUP AND THE CONTRIBUTORS +"AS IS" WITH NO WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED. In no +event shall The HDF Group or the Contributors be liable for any damages +suffered by the users arising out of the use of this software, even if +advised of the possibility of such damage. \ No newline at end of file diff --git a/src/filters/H5Zzstd.jl b/src/filters/H5Zzstd.jl new file mode 100644 index 000000000..bbdcba49b --- /dev/null +++ b/src/filters/H5Zzstd.jl @@ -0,0 +1,120 @@ +#= +Derived from https://github.com/aparamon/HDF5Plugin-Zstandard, zstd_h5plugin.c +Originally licensed under Apache License Version 2.0 +See H5Zzstd_LICENSE.txt + +The following copyright and license applies to the Julia port itself. +Copyright © 2021 Mark Kittisopikul, Howard Hughes Medical Institute +Licensed under MIT License, see LICENSE +=# +module H5Zzstd + +using ..API +using CodecZstd +import CodecZstd.LibZstd +import ..TestUtils: test_filter +import ..Filters: FILTERS, Filter, filterid + +const H5Z_FILTER_ZSTD = API.H5Z_filter_t(32015) +const zstd_name = "Zstandard compression: http://www.zstd.net" + +export H5Z_filter_zstd, H5Z_FILTER_ZSTD, register_zstd + + +function H5Z_filter_zstd(flags::Cuint, cd_nelmts::Csize_t, + cd_values::Ptr{Cuint}, nbytes::Csize_t, + buf_size::Ptr{Csize_t}, buf::Ptr{Ptr{Cvoid}}) + inbuf = unsafe_load(buf) + outbuf = C_NULL + origSize = nbytes + ret_value = Csize_t(0) + + try + + if flags & API.H5Z_FLAG_REVERSE != 0 + #decompresssion + + decompSize = LibZstd.ZSTD_getDecompressedSize(inbuf, origSize) + outbuf = Libc.malloc(decompSize) + if outbuf == C_NULL + error("zstd_h5plugin: Cannot allocate memory for outbuf during decompression.") + end + decompSize = LibZstd.ZSTD_decompress(outbuf, decompSize, inbuf, origSize) + Libc.free(inbuf) + unsafe_store!(buf, outbuf) + outbuf = C_NULL + ret_value = Csize_t(decompSize) + else + # compression + + if cd_nelmts > 0 + aggression = Cint(unsafe_load(cd_values)) + else + aggression = CodecZstd.LibZstd.ZSTD_CLEVEL_DEFAULT + end + + if aggression < 1 + aggression = 1 # ZSTD_minCLevel() + elseif aggression > LibZstd.ZSTD_maxCLevel() + aggression = LibZstd.ZSTD_maxCLevel() + end + + compSize = LibZstd.ZSTD_compressBound(origSize) + outbuf = Libc.malloc(compSize) + if outbuf == C_NULL + error("zstd_h5plugin: Cannot allocate memory for outbuf during compression.") + end + + compSize = LibZstd.ZSTD_compress(outbuf, compSize, inbuf, origSize, aggression) + + Libc.free(unsafe_load(buf)) + unsafe_store!(buf, outbuf) + unsafe_store!(buf_size, compSize) + outbuf = C_NULL + ret_value = compSize + end + catch err + rethrow(err) + finally + + if outbuf != C_NULL + free(outbuf) + end + + end + return ret_value + +end + +function test_zstd_filter() + cd_values = Cuint[3] # aggression + test_filter(H5Z_filter_zstd; cd_values = cd_values) +end + +function register_zstd() + c_zstd_filter = @cfunction(H5Z_filter_zstd, Csize_t, + (Cuint, Csize_t, Ptr{Cuint}, Csize_t, + Ptr{Csize_t}, Ptr{Ptr{Cvoid}})) + API.h5z_register(API.H5Z_class_t( + API.H5Z_CLASS_T_VERS, + H5Z_FILTER_ZSTD, + 1, + 1, + pointer(zstd_name), + C_NULL, + C_NULL, + c_zstd_filter + )) + return nothing +end + +# Filters Module + +struct ZstdFilter <: Filter + clevel::Cuint +end + +filterid(::Type{ZstdFilter}) = H5Z_FILTER_ZSTD +FILTERS[H5Z_FILTER_ZSTD] = ZstdFilter + +end # module H5Zzstd \ No newline at end of file diff --git a/src/filters/H5Zzstd_LICENSE.txt b/src/filters/H5Zzstd_LICENSE.txt new file mode 100644 index 000000000..9c8f3ea08 --- /dev/null +++ b/src/filters/H5Zzstd_LICENSE.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/src/filters/H5bzip2_LICENSE.txt b/src/filters/H5bzip2_LICENSE.txt new file mode 100644 index 000000000..46c548303 --- /dev/null +++ b/src/filters/H5bzip2_LICENSE.txt @@ -0,0 +1,29 @@ +Copyright Notice and Statement for PyTables Software Library and Utilities: +Copyright (c) 2002-2004 by Francesc Alted +Copyright (c) 2005-2007 by Carabos Coop. V. +Copyright (c) 2008-2010 by Francesc Alted +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +a. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +b. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the + distribution. +c. Neither the name of Francesc Alted nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/src/filters/TestUtils.jl b/src/filters/TestUtils.jl new file mode 100644 index 000000000..b5409066c --- /dev/null +++ b/src/filters/TestUtils.jl @@ -0,0 +1,80 @@ +module TestUtils + +import ..API + +export test_filter + +function test_filter_init(; cd_values = Cuint[], data = ones(UInt8, 1024)) + flags = Cuint(0) + nbytes = sizeof(data) + buf_size = Ref(Csize_t(sizeof(data))) + databuf = Libc.malloc(sizeof(data)) + data = reinterpret(UInt8, data) + unsafe_copyto!(Ptr{UInt8}(databuf), pointer(data), sizeof(data)) + buf = Ref(Ptr{Cvoid}(databuf)) + return flags, cd_values, nbytes, buf_size, buf +end + +function test_filter_compress!(filter_func, flags::Cuint, cd_values::Vector{Cuint}, nbytes::Integer, buf_size::Ref{Csize_t}, buf::Ref{Ptr{Cvoid}}) + nbytes = Csize_t(nbytes) + cd_nelmts = Csize_t(length(cd_values)) + GC.@preserve flags cd_nelmts cd_values nbytes buf_size buf begin + ret_code = filter_func( + flags, + cd_nelmts, + pointer(cd_values), + Csize_t(nbytes), + Base.unsafe_convert(Ptr{Csize_t}, buf_size), + Base.unsafe_convert(Ptr{Ptr{Cvoid}}, buf) + ) + @info "Compression:" ret_code buf_size[] + if ret_code <= 0 + error("Test compression failed: $ret_code.") + end + end + return ret_code +end + +function test_filter_decompress!(filter_func, flags::Cuint, cd_values::Vector{Cuint}, nbytes::Integer, buf_size::Ref{Csize_t}, buf::Ref{Ptr{Cvoid}}) + nbytes = Csize_t(nbytes) + cd_nelmts = Csize_t(length(cd_values)) + flags |= UInt32(API.H5Z_FLAG_REVERSE) + GC.@preserve flags cd_nelmts cd_values nbytes buf_size buf begin + ret_code = filter_func( + flags, + cd_nelmts, + pointer(cd_values), + Csize_t(nbytes), + Base.unsafe_convert(Ptr{Csize_t},buf_size), + Base.unsafe_convert(Ptr{Ptr{Cvoid}}, buf) + ) + @info "Decompression:" ret_code buf_size[] + end + return ret_code +end + +function test_filter_cleanup!(buf::Ref{Ptr{Cvoid}}) + Libc.free(buf[]) +end + +function test_filter(filter_func; cd_values::Vector{Cuint} = Cuint[], data = ones(UInt8, 1024)) + flags, cd_values, nbytes, buf_size, buf = test_filter_init(; cd_values = cd_values, data = data) + nbytes_compressed, nbytes_decompressed = 0, 0 + try + nbytes_compressed = test_filter_compress!(filter_func, flags, cd_values, nbytes, buf_size, buf) + nbytes_decompressed = test_filter_decompress!(filter_func, flags, cd_values, nbytes_compressed, buf_size, buf) + if nbytes_decompressed > 0 + # ret_code is the number of bytes out + round_trip_data = unsafe_wrap(Array,Ptr{UInt8}(buf[]), nbytes_decompressed) + @info "Is the data the same after a roundtrip?" data == round_trip_data + end + catch err + rethrow(err) + finally + test_filter_cleanup!(buf) + end + @info "Compression Ratio" nbytes_compressed / nbytes_decompressed + return nbytes_compressed, nbytes_decompressed +end + +end \ No newline at end of file diff --git a/src/filters/filters.jl b/src/filters/filters.jl index a911ca79b..f89fc2674 100644 --- a/src/filters/filters.jl +++ b/src/filters/filters.jl @@ -1,6 +1,6 @@ module Filters -export Deflate, Shuffle, Fletcher32, Szip, NBit, ScaleOffset, BloscFilter +export Deflate, Shuffle, Fletcher32, Szip, NBit, ScaleOffset, BloscFilter, Bzip2Filter, Lz4Filter, ZstdFilter import ..HDF5: Properties, h5doc, API @@ -106,8 +106,35 @@ function Base.append!(filters::FilterPipeline, extra) end return filters end +function Base.push!(p::FilterPipeline, f::F) where F <: Filter + ref = Ref(f) + GC.@preserve ref begin + API.h5p_set_filter(p.plist, filterid(F), API.H5Z_FLAG_OPTIONAL, div(sizeof(F), sizeof(Cuint)), pointer_from_objref(ref)) + end + return p +end +function Base.push!(p::FilterPipeline, f::UnknownFilter) + GC.@preserve f begin + API.h5p_set_filter(p.plist, f.filter_id, f.flags, length(f.data), pointer(f.data)) + end +end include("builtin.jl") include("blosc.jl") +include("TestUtils.jl") +include("H5Zbzip2.jl") +include("H5Zlz4.jl") +include("H5Zzstd.jl") + +import .H5Zbzip2: register_bzip2, Bzip2Filter +import .H5Zlz4: register_lz4, Lz4Filter +import .H5Zzstd: register_zstd, ZstdFilter + +function register() + register_blosc() + register_bzip2() + register_lz4() + register_zstd() +end end # module From 8e84992fbe7a4c9f8ecc8cdf4c041edc7f23995f Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Tue, 2 Nov 2021 02:20:03 -0400 Subject: [PATCH 02/28] Create Filters interface def, add tests --- src/HDF5.jl | 2 +- src/filters/H5Zbzip2.jl | 22 ++- src/filters/H5Zlz4.jl | 24 ++- src/filters/H5Zzstd.jl | 22 ++- src/filters/blosc.jl | 33 +++- src/filters/filters.jl | 176 +++++++++++++++++- test/filter.jl | 58 ++++-- .../filters/FilterTestUtils.jl | 40 +++- 8 files changed, 316 insertions(+), 61 deletions(-) rename src/filters/TestUtils.jl => test/filters/FilterTestUtils.jl (70%) diff --git a/src/HDF5.jl b/src/HDF5.jl index 724ea89d1..05de48452 100644 --- a/src/HDF5.jl +++ b/src/HDF5.jl @@ -1593,7 +1593,7 @@ function __init__() ENV["HDF5_USE_FILE_LOCKING"] = "FALSE" end - Filters.register() + Filters.register_filters() # use our own error handling machinery (i.e. turn off automatic error printing) API.h5e_set_auto(API.H5E_DEFAULT, C_NULL, C_NULL) diff --git a/src/filters/H5Zbzip2.jl b/src/filters/H5Zbzip2.jl index a207b8f9f..8275cba6f 100644 --- a/src/filters/H5Zbzip2.jl +++ b/src/filters/H5Zbzip2.jl @@ -14,10 +14,11 @@ module H5Zbzip2 using ..API using CodecBzip2 import CodecBzip2: libbzip2 -import ..TestUtils: test_filter -import ..Filters: FILTERS, Filter, filterid +import ..Filters: FILTERS, Filter, filterid, register_filter, FilterPipeline +import ..Filters: filterid, filtername, encoder_present, decoder_present +import ..Filters: set_local_func, set_local_cfunc, can_apply_func, can_apply_cfunc, filter_func, filter_cfunc -export H5Z_FILTER_BZIP2, H5Z_filter_bzip2, register_bzip2 +export H5Z_FILTER_BZIP2, H5Z_filter_bzip2, Bzip2Filter const H5Z_FILTER_BZIP2 = API.H5Z_filter_t(307) @@ -148,6 +149,7 @@ function register_bzip2() C_NULL, c_bzip2_filter )) + FILTERS[H5Z_FILTER_BZIP2] = Bzip2Filter return nothing end @@ -182,18 +184,20 @@ function BZ2_bzBuffToBuffDecompress(dest, destLen, source, sourceLen, small, ver ) end -function test_bzip2_filter(data = ones(UInt8, 1024)) - cd_values = Cuint[8] - test_filter(H5Z_filter_bzip2; cd_values = cd_values, data = data) -end - # Filters Module struct Bzip2Filter <: Filter blockSize100k::Cuint end +Bzip2Filter() = Bzip2Filter(9) filterid(::Type{Bzip2Filter}) = H5Z_FILTER_BZIP2 -FILTERS[H5Z_FILTER_BZIP2] = Bzip2Filter +filtername(::Type{Bzip2Filter}) = bzip2_name +filter_func(::Type{Bzip2Filter}) = H5Z_filter_bzip2 +filter_cfunc(::Type{Bzip2Filter}) = @cfunction(H5Z_filter_bzip2, Csize_t, + (Cuint, Csize_t, Ptr{Cuint}, Csize_t, + Ptr{Csize_t}, Ptr{Ptr{Cvoid}})) +register_filter(::Type{Bzip2Filter}) = register_bzip2() + end # module H5Zbzip2 \ No newline at end of file diff --git a/src/filters/H5Zlz4.jl b/src/filters/H5Zlz4.jl index af449218a..e13d9ce3b 100644 --- a/src/filters/H5Zlz4.jl +++ b/src/filters/H5Zlz4.jl @@ -15,10 +15,13 @@ module H5Zlz4 using ..API using CodecLz4 -import ..TestUtils: test_filter -import ..Filters: FILTERS, Filter, filterid +import ..Filters: FILTERS, Filter, filterid, register_filter, FilterPipeline +import ..Filters: filterid, filtername, encoder_present, decoder_present +import ..Filters: set_local_func, set_local_cfunc, can_apply_func, can_apply_cfunc, filter_func, filter_cfunc -export H5Z_FILTER_LZ4, H5Z_filter_lz4, register_lz4 + + +export H5Z_FILTER_LZ4, H5Z_filter_lz4, Lz4Filter const H5Z_FILTER_LZ4 = API.H5Z_filter_t(32004) @@ -185,11 +188,6 @@ function H5Z_filter_lz4(flags::Cuint, cd_nelmts::Csize_t, return Csize_t(ret_value) end -function test_lz4_filter() - cd_values = Cuint[1024] - test_filter(H5Z_filter_lz4; cd_values = cd_values) -end - function register_lz4(; aggression = 1) LZ4_AGGRESSION[] = aggression c_lz4_filter = @cfunction(H5Z_filter_lz4, Csize_t, @@ -205,7 +203,7 @@ function register_lz4(; aggression = 1) C_NULL, c_lz4_filter )) - + FILTERS[H5Z_FILTER_LZ4] = Lz4Filter return nothing end @@ -214,8 +212,14 @@ end struct Lz4Filter <: Filter blockSize::Cuint end +Lz4Filter() = Lz4Filter(DEFAULT_BLOCK_SIZE) filterid(::Type{Lz4Filter}) = H5Z_FILTER_LZ4 -FILTERS[H5Z_FILTER_LZ4] = Lz4Filter +filtername(::Type{Lz4Filter}) = lz4_name +filter_func(::Type{Lz4Filter}) = H5Z_filter_lz4 +filter_cfunc(::Type{Lz4Filter}) = @cfunction(H5Z_filter_lz4, Csize_t, + (Cuint, Csize_t, Ptr{Cuint}, Csize_t, + Ptr{Csize_t}, Ptr{Ptr{Cvoid}})) +register_filter(::Type{Lz4Filter}) = register_lz4() end \ No newline at end of file diff --git a/src/filters/H5Zzstd.jl b/src/filters/H5Zzstd.jl index bbdcba49b..b08ad6a00 100644 --- a/src/filters/H5Zzstd.jl +++ b/src/filters/H5Zzstd.jl @@ -12,13 +12,15 @@ module H5Zzstd using ..API using CodecZstd import CodecZstd.LibZstd -import ..TestUtils: test_filter -import ..Filters: FILTERS, Filter, filterid +import ..Filters: FILTERS, Filter, filterid, register_filter, FilterPipeline +import ..Filters: filterid, filtername, encoder_present, decoder_present +import ..Filters: set_local_func, set_local_cfunc, can_apply_func, can_apply_cfunc, filter_func, filter_cfunc + const H5Z_FILTER_ZSTD = API.H5Z_filter_t(32015) const zstd_name = "Zstandard compression: http://www.zstd.net" -export H5Z_filter_zstd, H5Z_FILTER_ZSTD, register_zstd +export H5Z_filter_zstd, H5Z_FILTER_ZSTD, ZstdFilter function H5Z_filter_zstd(flags::Cuint, cd_nelmts::Csize_t, @@ -86,11 +88,6 @@ function H5Z_filter_zstd(flags::Cuint, cd_nelmts::Csize_t, end -function test_zstd_filter() - cd_values = Cuint[3] # aggression - test_filter(H5Z_filter_zstd; cd_values = cd_values) -end - function register_zstd() c_zstd_filter = @cfunction(H5Z_filter_zstd, Csize_t, (Cuint, Csize_t, Ptr{Cuint}, Csize_t, @@ -105,6 +102,7 @@ function register_zstd() C_NULL, c_zstd_filter )) + FILTERS[H5Z_FILTER_ZSTD] = ZstdFilter return nothing end @@ -113,8 +111,14 @@ end struct ZstdFilter <: Filter clevel::Cuint end +ZstdFilter() = ZstdFilter(CodecZstd.LibZstd.ZSTD_CLEVEL_DEFAULT) filterid(::Type{ZstdFilter}) = H5Z_FILTER_ZSTD -FILTERS[H5Z_FILTER_ZSTD] = ZstdFilter +filtername(::Type{ZstdFilter}) = zstd_name +filter_func(::Type{ZstdFilter}) = H5Z_filter_zstd +filter_cfunc(::Type{ZstdFilter}) = @cfunction(H5Z_filter_zstd, Csize_t, + (Cuint, Csize_t, Ptr{Cuint}, Csize_t, + Ptr{Csize_t}, Ptr{Ptr{Cvoid}})) +register_filter(::Type{ZstdFilter}) = register_zstd() end # module H5Zzstd \ No newline at end of file diff --git a/src/filters/blosc.jl b/src/filters/blosc.jl index 47f6b7e3a..ad68f5059 100644 --- a/src/filters/blosc.jl +++ b/src/filters/blosc.jl @@ -1,8 +1,15 @@ +module H5Zblosc # port of https://github.com/Blosc/c-blosc/blob/3a668dcc9f61ad22b5c0a0ab45fe8dad387277fd/hdf5/blosc_filter.c (copyright 2010 Francesc Alted, license: MIT/expat) +using ..API import Blosc +import ..Filters: FILTERS, Filter, filterid, register_filter, FilterPipeline +import ..Filters: filterid, filtername, encoder_present, decoder_present +import ..Filters: set_local_func, set_local_cfunc, can_apply_func, can_apply_cfunc, filter_func, filter_cfunc -const FILTER_BLOSC = API.H5Z_filter_t(32001) # Filter ID registered with the HDF Group for Blosc +export H5Z_FILTER_BLOSC, blosc_filter, BloscFilter + +const H5Z_FILTER_BLOSC = API.H5Z_filter_t(32001) # Filter ID registered with the HDF Group for Blosc const FILTER_BLOSC_VERSION = 2 const blosc_name = "blosc" @@ -12,7 +19,7 @@ function blosc_set_local(dcpl::API.hid_t, htype::API.hid_t, space::API.hid_t) blosc_nelements = Ref{Csize_t}(length(blosc_values)) blosc_chunkdims = Vector{API.hsize_t}(undef,32) - API.h5p_get_filter_by_id(dcpl, FILTER_BLOSC, blosc_flags, blosc_nelements, blosc_values, 0, C_NULL, C_NULL) + API.h5p_get_filter_by_id(dcpl, H5Z_FILTER_BLOSC, blosc_flags, blosc_nelements, blosc_values, 0, C_NULL, C_NULL) flags = blosc_flags[] nelements = max(blosc_nelements[], 4) # First 4 slots reserved @@ -45,7 +52,7 @@ function blosc_set_local(dcpl::API.hid_t, htype::API.hid_t, space::API.hid_t) blosc_values[3] = basetypesize blosc_values[4] = chunksize * htypesize # size of the chunk - API.h5p_modify_filter(dcpl, FILTER_BLOSC, flags, nelements, blosc_values) + API.h5p_modify_filter(dcpl, H5Z_FILTER_BLOSC, flags, nelements, blosc_values) return API.herr_t(1) end @@ -109,8 +116,8 @@ function register_blosc() c_blosc_filter = @cfunction(blosc_filter, Csize_t, (Cuint, Csize_t, Ptr{Cuint}, Csize_t, Ptr{Csize_t}, Ptr{Ptr{Cvoid}})) - API.h5z_register(API.H5Z_class_t(API.H5Z_CLASS_T_VERS, FILTER_BLOSC, 1, 1, pointer(blosc_name), C_NULL, c_blosc_set_local, c_blosc_filter)) - + API.h5z_register(API.H5Z_class_t(API.H5Z_CLASS_T_VERS, H5Z_FILTER_BLOSC, 1, 1, pointer(blosc_name), C_NULL, c_blosc_set_local, c_blosc_filter)) + FILTERS[H5Z_FILTER_BLOSC] = BloscFilter return nothing end @@ -142,6 +149,16 @@ function BloscFilter(;level=5, shuffle=true, compressor="blosclz") BloscFilter(0,0,0,0,level,shuffle,compcode) end +filterid(::Type{BloscFilter}) = H5Z_FILTER_BLOSC +filtername(::Type{BloscFilter}) = blosc_name +set_local_func(::Type{BloscFilter}) = blosc_set_local +set_local_cfunc(::Type{BloscFilter}) = @cfunction(blosc_set_local, API.herr_t, (API.hid_t,API.hid_t,API.hid_t)) +filter_func(::Type{BloscFilter}) = blosc_filter +filter_cfunc(::Type{BloscFilter}) = @cfunction(blosc_filter, Csize_t, + (Cuint, Csize_t, Ptr{Cuint}, Csize_t, + Ptr{Csize_t}, Ptr{Ptr{Cvoid}})) +register_filter(::Type{BloscFilter}) = register_blosc() + function Base.show(io::IO, blosc::BloscFilter) print(io, BloscFilter, "(level=", Int(blosc.level), @@ -150,9 +167,6 @@ function Base.show(io::IO, blosc::BloscFilter) ")") end -filterid(::Type{BloscFilter}) = FILTER_BLOSC -FILTERS[FILTER_BLOSC] = BloscFilter - function Base.push!(f::FilterPipeline, blosc::BloscFilter) 0 <= blosc.level <= 9 || throw(ArgumentError("blosc compression $(blosc.level) not in [0,9]")) ref = Ref(blosc) @@ -161,3 +175,6 @@ function Base.push!(f::FilterPipeline, blosc::BloscFilter) end return f end + + +end # module H5Zblosc \ No newline at end of file diff --git a/src/filters/filters.jl b/src/filters/filters.jl index f89fc2674..62806605e 100644 --- a/src/filters/filters.jl +++ b/src/filters/filters.jl @@ -4,6 +4,40 @@ export Deflate, Shuffle, Fletcher32, Szip, NBit, ScaleOffset, BloscFilter, Bzip2 import ..HDF5: Properties, h5doc, API +""" + Filter + +Abstract type to describe HDF5 Filters. +See the Extended Help for information on implementing a new filter. + +# Extended Help + +## Filter interface + +The Filter interface can be implemented upon either the Filter subtype or an instance. +The instance methods default to calling the same method on the type. + +See API.h5z_register for details. + +### Required Methods to Implement +* `filterid` - registered filter ID +* `filter_func` - implement the actual filter + +### Optional Methods to Implement +* `filtername` - defaults to "Unnamed Filter" +* `encoder_present` - defaults to true +* `decoder_present` - defaults to true +* `can_apply_func` - defaults to nothing +* `set_local_func` - defaults to nothing + +### Advanced Methods to Implement +* `can_apply_cfunc` - Defaults to wrapping @cfunction around the result of `can_apply_func` +* `set_local_cfunc` - Defaults to wrapping @cfunction around the result of `set_local_func` +* `filter_cfunc` - Defaults to wrapping @cfunction around the result of `filter_func` +* `register_filter` - Defaults to using the above functions to register the filter + +Implement the Advanced Methods to avoid @cfunction from generating a runtime closure which may not work on all systems. +""" abstract type Filter end """ @@ -14,13 +48,102 @@ Maps filter id to filter type. const FILTERS = Dict{API.H5Z_filter_t, Any}() """ - filterid(::F) - filterid(F) + filterid(::F) where {F <: Filter} + filterid(F) where {F <: Filter} + The internal filter id of a filter of type `F`. """ filterid(::F) where {F<:Filter} = filterid(F) +""" + encoder_present(::F) where {F<:Filter} + +Can the filter have an encode or compress the data? +Defaults to true. +Returns a Bool. See `API.h5z_register`. +""" +encoder_present(::F) where {F<:Filter} = encoder_present(F) +encoder_present(::Type{F}) where {F<:Filter} = true + +""" + decoder_present(::F) where {F<:Filter} + +Can the filter decode or decompress the data? +Defaults to true. +Returns a Bool. +See `API.h5z_register` +""" +decoder_present(::F) where {F<:Filter} = decoder_present(F) +decoder_present(::Type{F}) where {F<:Filter} = true + +""" + filtername(::F) where {F<:Filter} + +What is the name of a filter? +Defaults to "Unnamed Filter" +Returns a String describing the filter. See `API.h5z_register` +""" +filtername(::F) where {F<:Filter} = filtername(F) +filtername(::Type{F}) where {F<:Filter} = "Unnamed Filter" + +""" + can_apply_func(::F) where {F<:Filter} + +Return a function indicating whether the filter can be applied or `nothing` is no function exists. +The function signature is `func(dcpl_id::API.hid_t, type_id::API.hid_t, space_id::API.hid_t)`. +See `API.h5z_register` +""" +can_apply_func(::F) where {F<:Filter} = can_apply_func(F) +can_apply_func(::Type{F}) where {F<:Filter} = nothing +function can_apply_cfunc(f::F) where {F<:Filter} + func = can_apply_func(f) + if func === nothing + return C_NULL + else + return @cfunction($func, API.herr_t, (API.hid_t,API.hid_t,API.hid_t)) + end +end + +""" + set_local_func(::F) where {F<:Filter} + +Return a function that sets dataset specific parameters or `nothing` if no function exists. +The function signature is `func(dcpl_id::API.hid_t, type_id::API.hid_t, space_id::API.hid_t)`. +See `API.h5z_register` +""" +set_local_func(::F) where {F<:Filter} = set_local_func(F) +set_local_func(::Type{F}) where {F<:Filter} = nothing +function set_local_cfunc(f::F) where {F<:Filter} + func = set_local_func(f) + if func === nothing + return C_NULL + else + return @cfunction($func, API.herr_t, (API.hid_t,API.hid_t,API.hid_t)) + end +end + + +""" + filter_func(::F) where {F<:Filter} + +Returns a function that performs the actual filtering. + +See `API.h5z_register` +""" +filter_func(::F) where {F<:Filter} = filter_func(F) +filter_func(::Type{F}) where {F<:Filter} = nothing +function filter_cfunc(f::F) where {F<:Filter} + func = filter_func(f) + if func === nothing + error("Filter function for $f must be defined via `filter_func`.") + end + c_filter_func = @cfunction($func, Csize_t, + (Cuint, Csize_t, Ptr{Cuint}, Csize_t, + Ptr{Csize_t}, Ptr{Ptr{Cvoid}})) + return c_filter_func +end + struct UnknownFilter <: Filter filter_id::API.H5Z_filter_t flags::Cuint @@ -28,6 +151,11 @@ struct UnknownFilter <: Filter name::String config::Cuint end +filterid(filter::UnknownFilter) = filter.filter_id +filtername(filter::UnknownFilter) = filter.name +filtername(::Type{UnknownFilter}) = "Unknown Filter" +encoder_present(::Type{UnknownFilter}) = false +decoder_present(::Type{UnknownFilter}) = false """ FilterPipeline(plist::DatasetCreateProperties) @@ -119,22 +247,54 @@ function Base.push!(p::FilterPipeline, f::UnknownFilter) end end +# Generic implementation of register_filter +""" + register_filter(filter::F) where F <: Filter + +Register the filter with the HDF5 library via API.h5z_register. +Also add F to the FILTERS dictionary. +""" +function register_filter(filter::F) where F <: Filter + id = filterid(filter) + encoder = encoder_present(filter) + decoder = decoder_present(filter) + name = filtername(filter) + can_apply = can_apply_cfunc(filter) + set_local = set_local_cfunc(filter) + func = filter_cfunc(filter) + GC.@preserve name begin + API.h5z_register(API.H5Z_class_t( + API.H5Z_CLASS_T_VERS, + id, + encoder, + decoder, + pointer(name), + can_apply, + set_local, + func + )) + end + # Should this be the filter instance rather than the type? + FILTERS[id] = F + return nothing +end +register_filter(::Type{F}) where {F<:Filter} = register_filter(F()) + include("builtin.jl") include("blosc.jl") -include("TestUtils.jl") include("H5Zbzip2.jl") include("H5Zlz4.jl") include("H5Zzstd.jl") +import .H5Zblosc: register_blosc, BloscFilter import .H5Zbzip2: register_bzip2, Bzip2Filter import .H5Zlz4: register_lz4, Lz4Filter import .H5Zzstd: register_zstd, ZstdFilter -function register() - register_blosc() - register_bzip2() - register_lz4() - register_zstd() +const FILTERS_TO_REGISTER = [BloscFilter, Bzip2Filter, Lz4Filter, ZstdFilter] + +function register_filters() + register_filter.(FILTERS_TO_REGISTER) end end # module diff --git a/test/filter.jl b/test/filter.jl index 48d7816cc..3320cda06 100644 --- a/test/filter.jl +++ b/test/filter.jl @@ -1,11 +1,10 @@ using HDF5 +using HDF5.Filters using Test +import HDF5.API: H5Z_FILTER_DEFLATE, H5Z_FILTER_SHUFFLE @testset "filter" begin -H5Z_FILTER_DEFLATE = 1 -H5Z_FILTER_SHUFFLE = 2 - # Create a new file fn = tempname() @@ -28,28 +27,59 @@ dsfiltdef = create_dataset(f, "filtdef", datatype(data), dataspace(data), dsfiltshufdef = create_dataset(f, "filtshufdef", datatype(data), dataspace(data), chunk=(100, 100), filters=[Filters.Shuffle(), Filters.Deflate(3)]) + # Write data write(dsdeflate, data) write(dsshufdef, data) write(dsfiltdef, data) write(dsfiltshufdef, data) +# Test compression filters + +compressionFilters = Dict( + "blosc" => Filters.BloscFilter, + "bzip2" => Filters.Bzip2Filter, + "lz4" => Filters.Lz4Filter, + "zstd" => Filters.ZstdFilter +) + +for (name, filter) in compressionFilters + + ds = create_dataset( + f, name, datatype(data), dataspace(data), + chunk=(100,100), filters=filter() + ) + write(ds, data) + + ds = create_dataset( + f, "shuffle+"*name, datatype(data), dataspace(data), + chunk=(100,100), filters=[Filters.Shuffle(), filter()] + ) + write(ds, data) + +end + + # Close and re-open file for reading close(f) f = h5open(fn) -# Read dataseta -datadeflate = f["deflate"][] -datashufdef = f["shufdef"][] -datafiltdef = f["filtdef"][] -datafiltshufdef = f["filtshufdef"][] +# Read datasets and test for equality +for name in keys(f) + ds = f[name] + @testset "$name" begin + @debug "Filter Dataset" HDF5.name(ds) + @test ds[] == data + filters = HDF5.get_create_properties(ds).filters + if startswith(name, "shuffle+") + @test filters[1] isa Shuffle + @test filters[2] isa compressionFilters[name[9:end]] + elseif haskey(compressionFilters, name) + @test filters[1] isa compressionFilters[name] + end + end +end close(f) -# Test for equality -@test datadeflate == data -@test datashufdef == data -@test datafiltdef == data -@test datafiltshufdef == data - end # @testset "filter" diff --git a/src/filters/TestUtils.jl b/test/filters/FilterTestUtils.jl similarity index 70% rename from src/filters/TestUtils.jl rename to test/filters/FilterTestUtils.jl index b5409066c..cd1adb504 100644 --- a/src/filters/TestUtils.jl +++ b/test/filters/FilterTestUtils.jl @@ -1,6 +1,15 @@ -module TestUtils +""" + module FilterTestUtils -import ..API +This module contains utilities for evaluating and debugging HDF5 Filters. +""" +module FilterTestUtils + +import HDF5.API +import HDF5.Filters.H5Zlz4: H5Z_filter_lz4 +import HDF5.Filters.H5Zzstd: H5Z_filter_zstd +import HDF5.Filters.H5Zbzip2: H5Z_filter_bzip2 +using Test export test_filter @@ -77,4 +86,31 @@ function test_filter(filter_func; cd_values::Vector{Cuint} = Cuint[], data = one return nbytes_compressed, nbytes_decompressed end +function test_bzip2_filter(data = ones(UInt8, 1024)) + cd_values = Cuint[8] + test_filter(H5Z_filter_bzip2; cd_values = cd_values, data = data) +end + +function test_lz4_filter(data = ones(UInt8, 1024)) + cd_values = Cuint[1024] + test_filter(H5Z_filter_lz4; cd_values = cd_values, data = data) +end + +function test_zstd_filter(data = ones(UInt8, 1024)) + cd_values = Cuint[3] # aggression + test_filter(H5Z_filter_zstd; cd_values = cd_values, data = data) +end + +function __init__() + @testset "Compression Filter Unit Tests" begin + @test argmin(test_bzip2_filter()) == 1 + @test argmin(test_lz4_filter()) == 1 + @test argmin(test_zstd_filter()) == 1 + str = codeunits(repeat("foobar", 1000)) + @test argmin(test_bzip2_filter(str)) == 1 + @test argmin(test_lz4_filter(str)) == 1 + @test argmin(test_zstd_filter(str)) == 1 + end +end + end \ No newline at end of file From 671368949473f0e30fb424793aa71459931bf526 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Sun, 14 Nov 2021 19:29:38 -0500 Subject: [PATCH 03/28] Filters return 0 rather than rethrowing error --- src/filters/H5Zbzip2.jl | 8 +++++--- src/filters/H5Zlz4.jl | 12 ++++++------ src/filters/H5Zzstd.jl | 23 +++++++++++++---------- 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/src/filters/H5Zbzip2.jl b/src/filters/H5Zbzip2.jl index 8275cba6f..5ccd4019f 100644 --- a/src/filters/H5Zbzip2.jl +++ b/src/filters/H5Zbzip2.jl @@ -26,7 +26,7 @@ const bzip2_name = "HDF5 bzip2 filter; see http://www.hdfgroup.org/services/cont function H5Z_filter_bzip2(flags::Cuint, cd_nelmts::Csize_t, cd_values::Ptr{Cuint}, nbytes::Csize_t, - buf_size::Ptr{Csize_t}, buf::Ptr{Ptr{Cvoid}}) + buf_size::Ptr{Csize_t}, buf::Ptr{Ptr{Cvoid}})::Csize_t outbuf = C_NULL outdatalen = Cuint(0) @@ -125,11 +125,13 @@ function H5Z_filter_bzip2(flags::Cuint, cd_nelmts::Csize_t, unsafe_store!(buf_size, outbuflen) catch err + # "In the case of failure, the return value is 0 (zero) and all pointer arguments are left unchanged." + outdatalen = Csize_t(0) if outbuf != C_NULL Libc.free(outbuf) end - rethrow(err) - outdatalen = Csize_t(0) + @error "H5Zbzip2.jl Non-Fatal ERROR: " err + display(stacktrace(catch_backtrace())) end # try - catch return Csize_t(outdatalen) diff --git a/src/filters/H5Zlz4.jl b/src/filters/H5Zlz4.jl index e13d9ce3b..10bcdcb16 100644 --- a/src/filters/H5Zlz4.jl +++ b/src/filters/H5Zlz4.jl @@ -33,14 +33,14 @@ const LZ4_AGGRESSION = Ref(1) # flags H5Z_FLAG_REVERSE or H5Z_FLAG_OPTIONAL -# cd_nelmts number of elements in cd_values -# cd_values +# cd_nelmts number of elements in cd_values (0 or 1) +# cd_values the first optional element must be the blockSize # nbytes - number of valid bytes of data # buf_size - total size of buffer # buf - pointer to pointer of data function H5Z_filter_lz4(flags::Cuint, cd_nelmts::Csize_t, cd_values::Ptr{Cuint}, nbytes::Csize_t, - buf_size::Ptr{Csize_t}, buf::Ptr{Ptr{Cvoid}}) + buf_size::Ptr{Csize_t}, buf::Ptr{Ptr{Cvoid}})::Csize_t outBuf = C_NULL ret_value = Csize_t(0) @@ -176,10 +176,10 @@ function H5Z_filter_lz4(flags::Cuint, cd_nelmts::Csize_t, end # (flags & API.H5Z_FLAG_REVERSE) != 0 catch err - println("ERROR: ", err) - rethrow() - #println(stacktrace(catch_backtrace())) + # "In the case of failure, the return value is 0 (zero) and all pointer arguments are left unchanged." ret_value = Csize_t(0) + @error "H5Zlz4.jl Non-Fatal ERROR: " err + display(stacktrace(catch_backtrace())) finally if outBuf != C_NULL Libc.free(outBuf) diff --git a/src/filters/H5Zzstd.jl b/src/filters/H5Zzstd.jl index b08ad6a00..d95f2310f 100644 --- a/src/filters/H5Zzstd.jl +++ b/src/filters/H5Zzstd.jl @@ -22,10 +22,11 @@ const zstd_name = "Zstandard compression: http://www.zstd.net" export H5Z_filter_zstd, H5Z_FILTER_ZSTD, ZstdFilter - +# cd_values First optional value is the compressor aggression +# Default is CodecZstd.LibZstd.ZSTD_CLEVEL_DEFAULT function H5Z_filter_zstd(flags::Cuint, cd_nelmts::Csize_t, cd_values::Ptr{Cuint}, nbytes::Csize_t, - buf_size::Ptr{Csize_t}, buf::Ptr{Ptr{Cvoid}}) + buf_size::Ptr{Csize_t}, buf::Ptr{Ptr{Cvoid}})::Csize_t inbuf = unsafe_load(buf) outbuf = C_NULL origSize = nbytes @@ -75,17 +76,19 @@ function H5Z_filter_zstd(flags::Cuint, cd_nelmts::Csize_t, outbuf = C_NULL ret_value = compSize end - catch err - rethrow(err) + catch e + # "In the case of failure, the return value is 0 (zero) and all pointer arguments are left unchanged." + ret_value = Csize_t(0) + @error "H5Zzstd Non-Fatal ERROR: " err + display(stacktrace(catch_backtrace())) finally - if outbuf != C_NULL - free(outbuf) - end - - end - return ret_value + if outbuf != C_NULL + free(outbuf) + end + end # try catch finally + return Csize_t(ret_value) end function register_zstd() From 297e51a04d1fd802940e5742f02c4d4158d4d134 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Mon, 15 Nov 2021 13:35:06 -0500 Subject: [PATCH 04/28] Move blosc.jl to H5Zblosc.jl --- src/filters/{blosc.jl => H5Zblosc.jl} | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) rename src/filters/{blosc.jl => H5Zblosc.jl} (97%) diff --git a/src/filters/blosc.jl b/src/filters/H5Zblosc.jl similarity index 97% rename from src/filters/blosc.jl rename to src/filters/H5Zblosc.jl index ad68f5059..886330611 100644 --- a/src/filters/blosc.jl +++ b/src/filters/H5Zblosc.jl @@ -5,7 +5,7 @@ using ..API import Blosc import ..Filters: FILTERS, Filter, filterid, register_filter, FilterPipeline import ..Filters: filterid, filtername, encoder_present, decoder_present -import ..Filters: set_local_func, set_local_cfunc, can_apply_func, can_apply_cfunc, filter_func, filter_cfunc +import ..Filters: set_local_func, set_local_cfunc, can_apply_func, can_apply_cfunc, filter_func, filter_cfunc, register_filter export H5Z_FILTER_BLOSC, blosc_filter, BloscFilter @@ -158,6 +158,7 @@ filter_cfunc(::Type{BloscFilter}) = @cfunction(blosc_filter, Csize_t, (Cuint, Csize_t, Ptr{Cuint}, Csize_t, Ptr{Csize_t}, Ptr{Ptr{Cvoid}})) register_filter(::Type{BloscFilter}) = register_blosc() +register_filter(::BloscFilter) = register_blosc() function Base.show(io::IO, blosc::BloscFilter) print(io, BloscFilter, @@ -176,5 +177,7 @@ function Base.push!(f::FilterPipeline, blosc::BloscFilter) return f end +precompile(register_filter, (BloscFilter,)) +precompile(register_filter, (Type{BloscFilter},)) end # module H5Zblosc \ No newline at end of file From 4d39a95ab12a117d1f8d9a773b17ad85a4834bf5 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Mon, 15 Nov 2021 13:35:51 -0500 Subject: [PATCH 05/28] Optimize method inference, init, precompile --- src/filters/H5Zbzip2.jl | 3 +++ src/filters/H5Zlz4.jl | 4 ++++ src/filters/H5Zzstd.jl | 4 ++++ src/filters/filters.jl | 14 +++++++++++--- 4 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/filters/H5Zbzip2.jl b/src/filters/H5Zbzip2.jl index 5ccd4019f..c80e6d0e8 100644 --- a/src/filters/H5Zbzip2.jl +++ b/src/filters/H5Zbzip2.jl @@ -200,6 +200,9 @@ filter_cfunc(::Type{Bzip2Filter}) = @cfunction(H5Z_filter_bzip2, Csize_t, (Cuint, Csize_t, Ptr{Cuint}, Csize_t, Ptr{Csize_t}, Ptr{Ptr{Cvoid}})) register_filter(::Type{Bzip2Filter}) = register_bzip2() +register_filter(::Bzip2Filter) = register_bzip2() +precompile(register_filter, (Bzip2Filter,)) +precompile(register_filter, (Type{Bzip2Filter},)) end # module H5Zbzip2 \ No newline at end of file diff --git a/src/filters/H5Zlz4.jl b/src/filters/H5Zlz4.jl index 10bcdcb16..c5553a21c 100644 --- a/src/filters/H5Zlz4.jl +++ b/src/filters/H5Zlz4.jl @@ -221,5 +221,9 @@ filter_cfunc(::Type{Lz4Filter}) = @cfunction(H5Z_filter_lz4, Csize_t, (Cuint, Csize_t, Ptr{Cuint}, Csize_t, Ptr{Csize_t}, Ptr{Ptr{Cvoid}})) register_filter(::Type{Lz4Filter}) = register_lz4() +register_filter(::Lz4Filter) = register_lz4() + +precompile(register_filter, (Lz4Filter,)) +precompile(register_filter, (Type{Lz4Filter},)) end \ No newline at end of file diff --git a/src/filters/H5Zzstd.jl b/src/filters/H5Zzstd.jl index d95f2310f..4f3593c6f 100644 --- a/src/filters/H5Zzstd.jl +++ b/src/filters/H5Zzstd.jl @@ -123,5 +123,9 @@ filter_cfunc(::Type{ZstdFilter}) = @cfunction(H5Z_filter_zstd, Csize_t, (Cuint, Csize_t, Ptr{Cuint}, Csize_t, Ptr{Csize_t}, Ptr{Ptr{Cvoid}})) register_filter(::Type{ZstdFilter}) = register_zstd() +register_filter(::ZstdFilter) = register_zstd() + +precompile(register_filter, (ZstdFilter,)) +precompile(register_filter, (Type{ZstdFilter},)) end # module H5Zzstd \ No newline at end of file diff --git a/src/filters/filters.jl b/src/filters/filters.jl index 62806605e..58bc1f920 100644 --- a/src/filters/filters.jl +++ b/src/filters/filters.jl @@ -281,7 +281,7 @@ end register_filter(::Type{F}) where {F<:Filter} = register_filter(F()) include("builtin.jl") -include("blosc.jl") +include("H5Zblosc.jl") include("H5Zbzip2.jl") include("H5Zlz4.jl") include("H5Zzstd.jl") @@ -291,10 +291,18 @@ import .H5Zbzip2: register_bzip2, Bzip2Filter import .H5Zlz4: register_lz4, Lz4Filter import .H5Zzstd: register_zstd, ZstdFilter -const FILTERS_TO_REGISTER = [BloscFilter, Bzip2Filter, Lz4Filter, ZstdFilter] +# const FILTERS_TO_REGISTER = [BloscFilter, Bzip2Filter, Lz4Filter, ZstdFilter] function register_filters() - register_filter.(FILTERS_TO_REGISTER) + # Simplified to optimize package loading + register_blosc() + register_bzip2() + register_lz4() + register_zstd() + # There are method inference issues with the below + # register_filter.(FILTERS_TO_REGISTER) end +precompile(register_filters, ()) + end # module From 48b53343115c1d5497acc5bc8f353f972028f493 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Fri, 19 Nov 2021 19:03:08 -0500 Subject: [PATCH 06/28] Register filters as when dep packages are loaded --- src/HDF5.jl | 2 -- src/filters/filters.jl | 53 ++++++++++++++++++++++++++---------------- 2 files changed, 33 insertions(+), 22 deletions(-) diff --git a/src/HDF5.jl b/src/HDF5.jl index 05de48452..5bdd73849 100644 --- a/src/HDF5.jl +++ b/src/HDF5.jl @@ -1593,8 +1593,6 @@ function __init__() ENV["HDF5_USE_FILE_LOCKING"] = "FALSE" end - Filters.register_filters() - # use our own error handling machinery (i.e. turn off automatic error printing) API.h5e_set_auto(API.H5E_DEFAULT, C_NULL, C_NULL) diff --git a/src/filters/filters.jl b/src/filters/filters.jl index 58bc1f920..9e9c27568 100644 --- a/src/filters/filters.jl +++ b/src/filters/filters.jl @@ -3,6 +3,7 @@ module Filters export Deflate, Shuffle, Fletcher32, Szip, NBit, ScaleOffset, BloscFilter, Bzip2Filter, Lz4Filter, ZstdFilter import ..HDF5: Properties, h5doc, API +using Requires: @require """ Filter @@ -45,7 +46,7 @@ abstract type Filter end Maps filter id to filter type. """ -const FILTERS = Dict{API.H5Z_filter_t, Any}() +const FILTERS = Dict{API.H5Z_filter_t, Type{<: Filter}}() """ filterid(::F) where {F <: Filter} @@ -280,27 +281,39 @@ function register_filter(filter::F) where F <: Filter end register_filter(::Type{F}) where {F<:Filter} = register_filter(F()) -include("builtin.jl") -include("H5Zblosc.jl") -include("H5Zbzip2.jl") -include("H5Zlz4.jl") -include("H5Zzstd.jl") - -import .H5Zblosc: register_blosc, BloscFilter -import .H5Zbzip2: register_bzip2, Bzip2Filter -import .H5Zlz4: register_lz4, Lz4Filter -import .H5Zzstd: register_zstd, ZstdFilter +function register_filters() + # Load filter codec packages which should trigger Requires.jl + @eval begin + using Blosc + using CodecBzip2 + using CodecLz4 + using CodecZstd + end +end -# const FILTERS_TO_REGISTER = [BloscFilter, Bzip2Filter, Lz4Filter, ZstdFilter] +include("builtin.jl") -function register_filters() - # Simplified to optimize package loading - register_blosc() - register_bzip2() - register_lz4() - register_zstd() - # There are method inference issues with the below - # register_filter.(FILTERS_TO_REGISTER) +function __init__() + @require Blosc="a74b3585-a348-5f62-a45c-50e91977d574" @eval begin + include("H5Zblosc.jl") + import .H5Zblosc: register_blosc, BloscFilter + register_blosc() + end + @require CodecBzip2="523fee87-0ab8-5b00-afb7-3ecf72e48cfd" @eval begin + include("H5Zbzip2.jl") + import .H5Zbzip2: register_bzip2, Bzip2Filter + register_bzip2() + end + @require CodecLz4="5ba52731-8f18-5e0d-9241-30f10d1ec561" @eval begin + include("H5Zlz4.jl") + import .H5Zlz4: register_lz4, Lz4Filter + register_lz4() + end + @require CodecZstd="6b39b394-51ab-5f42-8807-6242bab2b4c2" @eval begin + include("H5Zzstd.jl") + import .H5Zzstd: register_zstd, ZstdFilter + register_zstd() + end end precompile(register_filters, ()) From 5deb63513b4bcc7bcf9d834ae4993b53566d7a64 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Fri, 19 Nov 2021 19:11:10 -0500 Subject: [PATCH 07/28] Fix tests for lazy loaded filters --- test/filter.jl | 1 + test/filters/FilterTestUtils.jl | 1 + test/plain.jl | 1 + 3 files changed, 3 insertions(+) diff --git a/test/filter.jl b/test/filter.jl index 3320cda06..f1c1ed8b2 100644 --- a/test/filter.jl +++ b/test/filter.jl @@ -2,6 +2,7 @@ using HDF5 using HDF5.Filters using Test import HDF5.API: H5Z_FILTER_DEFLATE, H5Z_FILTER_SHUFFLE +import Blosc, CodecLz4, CodecBzip2, CodecZstd @testset "filter" begin diff --git a/test/filters/FilterTestUtils.jl b/test/filters/FilterTestUtils.jl index cd1adb504..8ef919d1f 100644 --- a/test/filters/FilterTestUtils.jl +++ b/test/filters/FilterTestUtils.jl @@ -6,6 +6,7 @@ This module contains utilities for evaluating and debugging HDF5 Filters. module FilterTestUtils import HDF5.API +import Blosc, CodecLz4, CodecZstd, CodecBzip2 import HDF5.Filters.H5Zlz4: H5Z_filter_lz4 import HDF5.Filters.H5Zzstd: H5Z_filter_zstd import HDF5.Filters.H5Zbzip2: H5Z_filter_bzip2 diff --git a/test/plain.jl b/test/plain.jl index 2432bf397..274134335 100644 --- a/test/plain.jl +++ b/test/plain.jl @@ -1,5 +1,6 @@ using HDF5 using CRC32c +using Blosc using Test gatherf(dst_buf, dst_buf_bytes_used, op_data) = HDF5.API.herr_t(0) From 9b9430bc5644f8e200011459e0b9265dffd66cc8 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Sat, 20 Nov 2021 13:17:16 -0500 Subject: [PATCH 08/28] Move plugin licenses to licenses folder --- .../H5Zbzip2_LICENSE.txt | 0 .../H5Zlz4_LICENSE.txt | 0 {src/filters => licenses}/H5Zzstd_LICENSE.txt | 0 licenses/README.md | 14 ++++++++++++++ 4 files changed, 14 insertions(+) rename src/filters/H5bzip2_LICENSE.txt => licenses/H5Zbzip2_LICENSE.txt (100%) rename src/filters/H5Zlz_LICENSE.txt => licenses/H5Zlz4_LICENSE.txt (100%) rename {src/filters => licenses}/H5Zzstd_LICENSE.txt (100%) create mode 100644 licenses/README.md diff --git a/src/filters/H5bzip2_LICENSE.txt b/licenses/H5Zbzip2_LICENSE.txt similarity index 100% rename from src/filters/H5bzip2_LICENSE.txt rename to licenses/H5Zbzip2_LICENSE.txt diff --git a/src/filters/H5Zlz_LICENSE.txt b/licenses/H5Zlz4_LICENSE.txt similarity index 100% rename from src/filters/H5Zlz_LICENSE.txt rename to licenses/H5Zlz4_LICENSE.txt diff --git a/src/filters/H5Zzstd_LICENSE.txt b/licenses/H5Zzstd_LICENSE.txt similarity index 100% rename from src/filters/H5Zzstd_LICENSE.txt rename to licenses/H5Zzstd_LICENSE.txt diff --git a/licenses/README.md b/licenses/README.md new file mode 100644 index 000000000..16677566d --- /dev/null +++ b/licenses/README.md @@ -0,0 +1,14 @@ +# Third Party Licenses + +HDF5.jl contains several derivative works of open source software. + +In particular, the following submodules are licensed from third-parties. +Original and derivative code in HDF5.jl is licensed according to [LICENSE.txt](../LICENSE.txt) +as permitted by licenses for the original software from which they may be derived. +See the files indicated below for the copyright notices and the licenses of the original +software from which individual submodules are derived. + +## Filter Plugins +* [HDF5.Filters.H5Zbzip2](../src/filters/H5Zbzip2.jl): See [H5Zbzip2_LICENSE.txt](H5Zbzip2_LICENSE.txt) +* [HDF5.Filters.H5Zlz4](../src/filters/H5Zlz4.jl): See [H5Zlz4_LICENSE.txt](H5Zlz4_LICENSE.txt) +* [HDF5.Filters.H5Zzstd](../src/filters/H5Zzstd.jl): See [H5Zzstd_LICENSE.txt](H5Zzstd_LICENSE.txt) \ No newline at end of file From b349f57c9642577e493747be92331ec0b02b97e5 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Sat, 20 Nov 2021 13:23:07 -0500 Subject: [PATCH 09/28] Add HHMI copyright to LICENSE.txt --- LICENSE.txt | 1 + src/filters/H5Zbzip2.jl | 4 ++-- src/filters/H5Zlz4.jl | 4 ++-- src/filters/H5Zzstd.jl | 4 ++-- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/LICENSE.txt b/LICENSE.txt index ee06f1fc4..86059529d 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,5 +1,6 @@ The MIT License (MIT) Copyright (c) 2012-2021: Timothy E. Holy, Konrad Hinsen, Tom Short, and Simon Kornblith. +Copyright (c) 2021: Mark Kittisopikul and Howard Hughes Medical Institute. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without diff --git a/src/filters/H5Zbzip2.jl b/src/filters/H5Zbzip2.jl index c80e6d0e8..d86de3f22 100644 --- a/src/filters/H5Zbzip2.jl +++ b/src/filters/H5Zbzip2.jl @@ -7,8 +7,8 @@ # See H5Zbzip2_LICENSE.txt for the license. # The following copyright and license applies to the Julia port itself. -# Copyright © 2021 Mark Kittisopikul, Howard Hughes Medical Institute -# Licensed under MIT License, see LICENSE +# Copyright © 2021 Mark Kittisopikul and Howard Hughes Medical Institute +# Licensed under MIT License, see LICENSE.txt. module H5Zbzip2 using ..API diff --git a/src/filters/H5Zlz4.jl b/src/filters/H5Zlz4.jl index c5553a21c..978a11a5f 100644 --- a/src/filters/H5Zlz4.jl +++ b/src/filters/H5Zlz4.jl @@ -8,8 +8,8 @@ H5Zlz4 is originally a copyright of HDF Group. See "LZ4 filter plugin license" in H5Zlz4_LICENSE.txt The following copyright and license applies to the Julia port itself. -Copyright © 2021 Mark Kittisopikul, Howard Hughes Medical Institute -Licensed under MIT License, see LICENSE +Copyright © 2021 Mark Kittisopikul and Howard Hughes Medical Institute +Licensed under MIT License, see LICENSE.txt =# module H5Zlz4 diff --git a/src/filters/H5Zzstd.jl b/src/filters/H5Zzstd.jl index 4f3593c6f..a7d08ab77 100644 --- a/src/filters/H5Zzstd.jl +++ b/src/filters/H5Zzstd.jl @@ -4,8 +4,8 @@ Originally licensed under Apache License Version 2.0 See H5Zzstd_LICENSE.txt The following copyright and license applies to the Julia port itself. -Copyright © 2021 Mark Kittisopikul, Howard Hughes Medical Institute -Licensed under MIT License, see LICENSE +Copyright © 2021 Mark Kittisopikul and Howard Hughes Medical Institute +Licensed under MIT License, see LICENSE.txt =# module H5Zzstd From cc92c6b269afbd3a09eeed89dab9cd7b8eac118b Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Tue, 23 Nov 2021 02:53:00 -0500 Subject: [PATCH 10/28] Revert change to LICENSE.txt --- LICENSE.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/LICENSE.txt b/LICENSE.txt index 86059529d..ee06f1fc4 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,6 +1,5 @@ The MIT License (MIT) Copyright (c) 2012-2021: Timothy E. Holy, Konrad Hinsen, Tom Short, and Simon Kornblith. -Copyright (c) 2021: Mark Kittisopikul and Howard Hughes Medical Institute. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without From 7303742a8c3c18135c28f1def3eebd6c8d5ca6c6 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Thu, 2 Dec 2021 00:19:08 -0500 Subject: [PATCH 11/28] Remove generic register_filter method --- src/filters/filters.jl | 27 ++------------------------- 1 file changed, 2 insertions(+), 25 deletions(-) diff --git a/src/filters/filters.jl b/src/filters/filters.jl index 9e9c27568..547302e2f 100644 --- a/src/filters/filters.jl +++ b/src/filters/filters.jl @@ -250,36 +250,13 @@ end # Generic implementation of register_filter """ + register_filter(::Type{F}) where F <: Filter register_filter(filter::F) where F <: Filter Register the filter with the HDF5 library via API.h5z_register. Also add F to the FILTERS dictionary. """ -function register_filter(filter::F) where F <: Filter - id = filterid(filter) - encoder = encoder_present(filter) - decoder = decoder_present(filter) - name = filtername(filter) - can_apply = can_apply_cfunc(filter) - set_local = set_local_cfunc(filter) - func = filter_cfunc(filter) - GC.@preserve name begin - API.h5z_register(API.H5Z_class_t( - API.H5Z_CLASS_T_VERS, - id, - encoder, - decoder, - pointer(name), - can_apply, - set_local, - func - )) - end - # Should this be the filter instance rather than the type? - FILTERS[id] = F - return nothing -end -register_filter(::Type{F}) where {F<:Filter} = register_filter(F()) +function register_filter() end function register_filters() # Load filter codec packages which should trigger Requires.jl From b3b4215dd8aebc5334dc1f161d9cf5f563d3a681 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Thu, 2 Dec 2021 00:37:34 -0500 Subject: [PATCH 12/28] Apply suggestions from code review for license notices Co-authored-by: Mustafa M --- src/filters/H5Zbzip2.jl | 14 ++++++-------- src/filters/H5Zlz4.jl | 8 +++----- src/filters/H5Zzstd.jl | 8 +++----- 3 files changed, 12 insertions(+), 18 deletions(-) diff --git a/src/filters/H5Zbzip2.jl b/src/filters/H5Zbzip2.jl index d86de3f22..cce3a3996 100644 --- a/src/filters/H5Zbzip2.jl +++ b/src/filters/H5Zbzip2.jl @@ -1,14 +1,12 @@ -# The code below has been ported to Julia from the original C: +# The code below has been ported to Julia from the original C source: # https://github.com/nexusformat/HDF5-External-Filter-Plugins/blob/master/BZIP2/src/H5Zbzip2.c -# The filter function H5Z_filter_bzip2 was adopted from +# The filter function H5Z_filter_bzip2 was adopted from: # PyTables http://www.pytables.org. -# The plugin can be used with the HDF5 library vesrion 1.8.11+ to read -# HDF5 datasets compressed with bzip2 created by PyTables. -# See H5Zbzip2_LICENSE.txt for the license. +# The plugin can be used with the HDF5 library version 1.8.11+ to read HDF5 datasets compressed with bzip2 created by PyTables. +# License: licenses/H5Zbzip2_LICENSE.txt -# The following copyright and license applies to the Julia port itself. -# Copyright © 2021 Mark Kittisopikul and Howard Hughes Medical Institute -# Licensed under MIT License, see LICENSE.txt. +The following license applies to the Julia port. +Copyright (c) 2021 Mark Kittisopikul and Howard Hughes Medical Institute. License MIT, see LICENSE.txt module H5Zbzip2 using ..API diff --git a/src/filters/H5Zlz4.jl b/src/filters/H5Zlz4.jl index 978a11a5f..4c528c647 100644 --- a/src/filters/H5Zlz4.jl +++ b/src/filters/H5Zlz4.jl @@ -4,12 +4,10 @@ https://github.com/HDFGroup/hdf5_plugins/blob/master/LZ4/src/H5Zlz4.c https://github.com/nexusformat/HDF5-External-Filter-Plugins/blob/master/LZ4/src/H5Zlz4.c https://github.com/silx-kit/hdf5plugin/blob/main/src/LZ4/H5Zlz4.c -H5Zlz4 is originally a copyright of HDF Group. -See "LZ4 filter plugin license" in H5Zlz4_LICENSE.txt +H5Zlz4 is originally a copyright of HDF Group. License: licenses/H5Zlz4_LICENSE.txt -The following copyright and license applies to the Julia port itself. -Copyright © 2021 Mark Kittisopikul and Howard Hughes Medical Institute -Licensed under MIT License, see LICENSE.txt +The following license applies to the Julia port. +Copyright (c) 2021 Mark Kittisopikul and Howard Hughes Medical Institute. License MIT, see LICENSE.txt =# module H5Zlz4 diff --git a/src/filters/H5Zzstd.jl b/src/filters/H5Zzstd.jl index a7d08ab77..8850bba9a 100644 --- a/src/filters/H5Zzstd.jl +++ b/src/filters/H5Zzstd.jl @@ -1,11 +1,9 @@ #= Derived from https://github.com/aparamon/HDF5Plugin-Zstandard, zstd_h5plugin.c -Originally licensed under Apache License Version 2.0 -See H5Zzstd_LICENSE.txt +Licensed under Apache License Version 2.0, see licenses/H5Zzstd_LICENSE.txt -The following copyright and license applies to the Julia port itself. -Copyright © 2021 Mark Kittisopikul and Howard Hughes Medical Institute -Licensed under MIT License, see LICENSE.txt +The following license applies to the Julia port. +Copyright (c) 2021 Mark Kittisopikul and Howard Hughes Medical Institute. License MIT, see LICENSE.txt =# module H5Zzstd From 84c89fc0479dd442c97faf2d87152edca57b1714 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Thu, 2 Dec 2021 00:41:12 -0500 Subject: [PATCH 13/28] Use block comment for H5Zbzip2.jl --- src/filters/H5Zbzip2.jl | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/filters/H5Zbzip2.jl b/src/filters/H5Zbzip2.jl index cce3a3996..9a048f89f 100644 --- a/src/filters/H5Zbzip2.jl +++ b/src/filters/H5Zbzip2.jl @@ -1,12 +1,14 @@ -# The code below has been ported to Julia from the original C source: -# https://github.com/nexusformat/HDF5-External-Filter-Plugins/blob/master/BZIP2/src/H5Zbzip2.c -# The filter function H5Z_filter_bzip2 was adopted from: -# PyTables http://www.pytables.org. -# The plugin can be used with the HDF5 library version 1.8.11+ to read HDF5 datasets compressed with bzip2 created by PyTables. -# License: licenses/H5Zbzip2_LICENSE.txt +#= +The code below has been ported to Julia from the original C source: +https://github.com/nexusformat/HDF5-External-Filter-Plugins/blob/master/BZIP2/src/H5Zbzip2.c +The filter function H5Z_filter_bzip2 was adopted from: +PyTables http://www.pytables.org. +The plugin can be used with the HDF5 library version 1.8.11+ to read HDF5 datasets compressed with bzip2 created by PyTables. +License: licenses/H5Zbzip2_LICENSE.txt The following license applies to the Julia port. Copyright (c) 2021 Mark Kittisopikul and Howard Hughes Medical Institute. License MIT, see LICENSE.txt +=# module H5Zbzip2 using ..API From 61fbb963b82e1d314c71c7f021bf1ba2e9d5840c Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Thu, 2 Dec 2021 11:39:18 -0500 Subject: [PATCH 14/28] Update test/filter.jl Co-authored-by: Mustafa M --- test/filter.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/test/filter.jl b/test/filter.jl index f1c1ed8b2..5729d1230 100644 --- a/test/filter.jl +++ b/test/filter.jl @@ -1,7 +1,6 @@ using HDF5 using HDF5.Filters using Test -import HDF5.API: H5Z_FILTER_DEFLATE, H5Z_FILTER_SHUFFLE import Blosc, CodecLz4, CodecBzip2, CodecZstd @testset "filter" begin From c4f68c3956ab1ee64194df8704f43bf66f53af18 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Thu, 2 Dec 2021 17:29:55 -0500 Subject: [PATCH 15/28] Implement generic register_filter on filter type --- src/filters/H5Zblosc.jl | 22 +++------------- src/filters/H5Zbzip2.jl | 41 ++++++++--------------------- src/filters/H5Zlz4.jl | 27 ++----------------- src/filters/H5Zzstd.jl | 26 ++----------------- src/filters/filters.jl | 57 +++++++++++++++++++++++++++++------------ 5 files changed, 58 insertions(+), 115 deletions(-) diff --git a/src/filters/H5Zblosc.jl b/src/filters/H5Zblosc.jl index 886330611..73961e940 100644 --- a/src/filters/H5Zblosc.jl +++ b/src/filters/H5Zblosc.jl @@ -1,11 +1,10 @@ module H5Zblosc # port of https://github.com/Blosc/c-blosc/blob/3a668dcc9f61ad22b5c0a0ab45fe8dad387277fd/hdf5/blosc_filter.c (copyright 2010 Francesc Alted, license: MIT/expat) -using ..API import Blosc -import ..Filters: FILTERS, Filter, filterid, register_filter, FilterPipeline -import ..Filters: filterid, filtername, encoder_present, decoder_present -import ..Filters: set_local_func, set_local_cfunc, can_apply_func, can_apply_cfunc, filter_func, filter_cfunc, register_filter +using HDF5.API +import HDF5.Filters: Filter, FilterPipeline, filterid, register_filter, filterid, filtername +import HDF5.Filters: set_local_func, set_local_cfunc, filter_func, filter_cfunc, register_filter export H5Z_FILTER_BLOSC, blosc_filter, BloscFilter @@ -109,19 +108,6 @@ function blosc_filter(flags::Cuint, cd_nelmts::Csize_t, return Csize_t(0) end - -# register the Blosc filter function with HDF5 -function register_blosc() - c_blosc_set_local = @cfunction(blosc_set_local, API.herr_t, (API.hid_t,API.hid_t,API.hid_t)) - c_blosc_filter = @cfunction(blosc_filter, Csize_t, - (Cuint, Csize_t, Ptr{Cuint}, Csize_t, - Ptr{Csize_t}, Ptr{Ptr{Cvoid}})) - API.h5z_register(API.H5Z_class_t(API.H5Z_CLASS_T_VERS, H5Z_FILTER_BLOSC, 1, 1, pointer(blosc_name), C_NULL, c_blosc_set_local, c_blosc_filter)) - FILTERS[H5Z_FILTER_BLOSC] = BloscFilter - return nothing -end - - """ BloscFilter(;level=5, shuffle=true, compressor="blosclz") @@ -157,8 +143,6 @@ filter_func(::Type{BloscFilter}) = blosc_filter filter_cfunc(::Type{BloscFilter}) = @cfunction(blosc_filter, Csize_t, (Cuint, Csize_t, Ptr{Cuint}, Csize_t, Ptr{Csize_t}, Ptr{Ptr{Cvoid}})) -register_filter(::Type{BloscFilter}) = register_blosc() -register_filter(::BloscFilter) = register_blosc() function Base.show(io::IO, blosc::BloscFilter) print(io, BloscFilter, diff --git a/src/filters/H5Zbzip2.jl b/src/filters/H5Zbzip2.jl index cce3a3996..d0b28e574 100644 --- a/src/filters/H5Zbzip2.jl +++ b/src/filters/H5Zbzip2.jl @@ -1,20 +1,20 @@ -# The code below has been ported to Julia from the original C source: -# https://github.com/nexusformat/HDF5-External-Filter-Plugins/blob/master/BZIP2/src/H5Zbzip2.c -# The filter function H5Z_filter_bzip2 was adopted from: -# PyTables http://www.pytables.org. -# The plugin can be used with the HDF5 library version 1.8.11+ to read HDF5 datasets compressed with bzip2 created by PyTables. -# License: licenses/H5Zbzip2_LICENSE.txt +#= +The code below has been ported to Julia from the original C source: +https://github.com/nexusformat/HDF5-External-Filter-Plugins/blob/master/BZIP2/src/H5Zbzip2.c +The filter function H5Z_filter_bzip2 was adopted from: +PyTables http://www.pytables.org. +The plugin can be used with the HDF5 library version 1.8.11+ to read HDF5 datasets compressed with bzip2 created by PyTables. +License: licenses/H5Zbzip2_LICENSE.txt The following license applies to the Julia port. Copyright (c) 2021 Mark Kittisopikul and Howard Hughes Medical Institute. License MIT, see LICENSE.txt +=# module H5Zbzip2 -using ..API using CodecBzip2 import CodecBzip2: libbzip2 -import ..Filters: FILTERS, Filter, filterid, register_filter, FilterPipeline -import ..Filters: filterid, filtername, encoder_present, decoder_present -import ..Filters: set_local_func, set_local_cfunc, can_apply_func, can_apply_cfunc, filter_func, filter_cfunc +using HDF5.API +import HDF5.Filters: Filter, filterid, register_filter, filterid, filtername, filter_func, filter_cfunc export H5Z_FILTER_BZIP2, H5Z_filter_bzip2, Bzip2Filter @@ -135,25 +135,6 @@ function H5Z_filter_bzip2(flags::Cuint, cd_nelmts::Csize_t, return Csize_t(outdatalen) end # function H5Z_filter_bzip2 -function register_bzip2() - c_bzip2_filter = @cfunction(H5Z_filter_bzip2, Csize_t, - (Cuint, Csize_t, Ptr{Cuint}, Csize_t, - Ptr{Csize_t}, Ptr{Ptr{Cvoid}})) - API.h5z_register(API.H5Z_class_t( - API.H5Z_CLASS_T_VERS, - H5Z_FILTER_BZIP2, - 1, - 1, - pointer(bzip2_name), - C_NULL, - C_NULL, - c_bzip2_filter - )) - FILTERS[H5Z_FILTER_BZIP2] = Bzip2Filter - return nothing -end - - # Need stdcall for 32-bit Windows? function BZ2_bzBuffToBuffCompress(dest, destLen, source, sourceLen, blockSize100k, verbosity, workFactor) return ccall( @@ -197,8 +178,6 @@ filter_func(::Type{Bzip2Filter}) = H5Z_filter_bzip2 filter_cfunc(::Type{Bzip2Filter}) = @cfunction(H5Z_filter_bzip2, Csize_t, (Cuint, Csize_t, Ptr{Cuint}, Csize_t, Ptr{Csize_t}, Ptr{Ptr{Cvoid}})) -register_filter(::Type{Bzip2Filter}) = register_bzip2() -register_filter(::Bzip2Filter) = register_bzip2() precompile(register_filter, (Bzip2Filter,)) precompile(register_filter, (Type{Bzip2Filter},)) diff --git a/src/filters/H5Zlz4.jl b/src/filters/H5Zlz4.jl index 4c528c647..aee5e7e33 100644 --- a/src/filters/H5Zlz4.jl +++ b/src/filters/H5Zlz4.jl @@ -11,11 +11,9 @@ Copyright (c) 2021 Mark Kittisopikul and Howard Hughes Medical Institute. Licens =# module H5Zlz4 -using ..API using CodecLz4 -import ..Filters: FILTERS, Filter, filterid, register_filter, FilterPipeline -import ..Filters: filterid, filtername, encoder_present, decoder_present -import ..Filters: set_local_func, set_local_cfunc, can_apply_func, can_apply_cfunc, filter_func, filter_cfunc +using HDF5.API +import HDF5.Filters: Filter, filterid, register_filter, filterid, filtername, filter_func, filter_cfunc @@ -186,25 +184,6 @@ function H5Z_filter_lz4(flags::Cuint, cd_nelmts::Csize_t, return Csize_t(ret_value) end -function register_lz4(; aggression = 1) - LZ4_AGGRESSION[] = aggression - c_lz4_filter = @cfunction(H5Z_filter_lz4, Csize_t, - (Cuint, Csize_t, Ptr{Cuint}, Csize_t, - Ptr{Csize_t}, Ptr{Ptr{Cvoid}})) - API.h5z_register(API.H5Z_class_t( - API.H5Z_CLASS_T_VERS, - H5Z_FILTER_LZ4, - 1, - 1, - pointer(lz4_name), - C_NULL, - C_NULL, - c_lz4_filter - )) - FILTERS[H5Z_FILTER_LZ4] = Lz4Filter - return nothing -end - # Filters Module struct Lz4Filter <: Filter @@ -218,8 +197,6 @@ filter_func(::Type{Lz4Filter}) = H5Z_filter_lz4 filter_cfunc(::Type{Lz4Filter}) = @cfunction(H5Z_filter_lz4, Csize_t, (Cuint, Csize_t, Ptr{Cuint}, Csize_t, Ptr{Csize_t}, Ptr{Ptr{Cvoid}})) -register_filter(::Type{Lz4Filter}) = register_lz4() -register_filter(::Lz4Filter) = register_lz4() precompile(register_filter, (Lz4Filter,)) precompile(register_filter, (Type{Lz4Filter},)) diff --git a/src/filters/H5Zzstd.jl b/src/filters/H5Zzstd.jl index 8850bba9a..1f6d6a305 100644 --- a/src/filters/H5Zzstd.jl +++ b/src/filters/H5Zzstd.jl @@ -7,12 +7,10 @@ Copyright (c) 2021 Mark Kittisopikul and Howard Hughes Medical Institute. Licens =# module H5Zzstd -using ..API using CodecZstd import CodecZstd.LibZstd -import ..Filters: FILTERS, Filter, filterid, register_filter, FilterPipeline -import ..Filters: filterid, filtername, encoder_present, decoder_present -import ..Filters: set_local_func, set_local_cfunc, can_apply_func, can_apply_cfunc, filter_func, filter_cfunc +using HDF5.API +import HDF5.Filters: Filter, filterid, register_filter, filterid, filtername, filter_func, filter_cfunc const H5Z_FILTER_ZSTD = API.H5Z_filter_t(32015) @@ -89,24 +87,6 @@ function H5Z_filter_zstd(flags::Cuint, cd_nelmts::Csize_t, return Csize_t(ret_value) end -function register_zstd() - c_zstd_filter = @cfunction(H5Z_filter_zstd, Csize_t, - (Cuint, Csize_t, Ptr{Cuint}, Csize_t, - Ptr{Csize_t}, Ptr{Ptr{Cvoid}})) - API.h5z_register(API.H5Z_class_t( - API.H5Z_CLASS_T_VERS, - H5Z_FILTER_ZSTD, - 1, - 1, - pointer(zstd_name), - C_NULL, - C_NULL, - c_zstd_filter - )) - FILTERS[H5Z_FILTER_ZSTD] = ZstdFilter - return nothing -end - # Filters Module struct ZstdFilter <: Filter @@ -120,8 +100,6 @@ filter_func(::Type{ZstdFilter}) = H5Z_filter_zstd filter_cfunc(::Type{ZstdFilter}) = @cfunction(H5Z_filter_zstd, Csize_t, (Cuint, Csize_t, Ptr{Cuint}, Csize_t, Ptr{Csize_t}, Ptr{Ptr{Cvoid}})) -register_filter(::Type{ZstdFilter}) = register_zstd() -register_filter(::ZstdFilter) = register_zstd() precompile(register_filter, (ZstdFilter,)) precompile(register_filter, (Type{ZstdFilter},)) diff --git a/src/filters/filters.jl b/src/filters/filters.jl index 547302e2f..c63f25baa 100644 --- a/src/filters/filters.jl +++ b/src/filters/filters.jl @@ -16,6 +16,7 @@ See the Extended Help for information on implementing a new filter. ## Filter interface The Filter interface can be implemented upon either the Filter subtype or an instance. +Implementing the interface on a Filter subtype is recommended. The instance methods default to calling the same method on the type. See API.h5z_register for details. @@ -91,14 +92,14 @@ filtername(::Type{F}) where {F<:Filter} = "Unnamed Filter" """ can_apply_func(::F) where {F<:Filter} -Return a function indicating whether the filter can be applied or `nothing` is no function exists. +Return a function indicating whether the filter can be applied or `nothing` if no function exists. The function signature is `func(dcpl_id::API.hid_t, type_id::API.hid_t, space_id::API.hid_t)`. See `API.h5z_register` """ can_apply_func(::F) where {F<:Filter} = can_apply_func(F) can_apply_func(::Type{F}) where {F<:Filter} = nothing -function can_apply_cfunc(f::F) where {F<:Filter} - func = can_apply_func(f) +function can_apply_cfunc(::Type{F}) where {F<:Filter} + func = can_apply_func(F) if func === nothing return C_NULL else @@ -115,8 +116,8 @@ See `API.h5z_register` """ set_local_func(::F) where {F<:Filter} = set_local_func(F) set_local_func(::Type{F}) where {F<:Filter} = nothing -function set_local_cfunc(f::F) where {F<:Filter} - func = set_local_func(f) +function set_local_cfunc(::Type{F}) where {F<:Filter} + func = set_local_func(F) if func === nothing return C_NULL else @@ -134,8 +135,8 @@ See `API.h5z_register` """ filter_func(::F) where {F<:Filter} = filter_func(F) filter_func(::Type{F}) where {F<:Filter} = nothing -function filter_cfunc(f::F) where {F<:Filter} - func = filter_func(f) +function filter_cfunc(::Type{F}) where {F<:Filter} + func = filter_func(F) if func === nothing error("Filter function for $f must be defined via `filter_func`.") end @@ -256,7 +257,31 @@ end Register the filter with the HDF5 library via API.h5z_register. Also add F to the FILTERS dictionary. """ -function register_filter() end +function register_filter(::Type{F}) where F <: Filter + id = filterid(F) + encoder = encoder_present(F) + decoder = decoder_present(F) + name = filtername(F) + can_apply = can_apply_cfunc(F) + set_local = set_local_cfunc(F) + func = filter_cfunc(F) + GC.@preserve name begin + API.h5z_register(API.H5Z_class_t( + API.H5Z_CLASS_T_VERS, + id, + encoder, + decoder, + pointer(name), + can_apply, + set_local, + func + )) + end + FILTERS[id] = F + return nothing +end +register_filter(::F) where {F<:Filter} = register_filter(F) + function register_filters() # Load filter codec packages which should trigger Requires.jl @@ -273,23 +298,23 @@ include("builtin.jl") function __init__() @require Blosc="a74b3585-a348-5f62-a45c-50e91977d574" @eval begin include("H5Zblosc.jl") - import .H5Zblosc: register_blosc, BloscFilter - register_blosc() + import .H5Zblosc: BloscFilter + register_filter(BloscFilter) end @require CodecBzip2="523fee87-0ab8-5b00-afb7-3ecf72e48cfd" @eval begin include("H5Zbzip2.jl") - import .H5Zbzip2: register_bzip2, Bzip2Filter - register_bzip2() + import .H5Zbzip2: Bzip2Filter + register_filter(Bzip2Filter) end @require CodecLz4="5ba52731-8f18-5e0d-9241-30f10d1ec561" @eval begin include("H5Zlz4.jl") - import .H5Zlz4: register_lz4, Lz4Filter - register_lz4() + import .H5Zlz4: Lz4Filter + register_filter(Lz4Filter) end @require CodecZstd="6b39b394-51ab-5f42-8807-6242bab2b4c2" @eval begin include("H5Zzstd.jl") - import .H5Zzstd: register_zstd, ZstdFilter - register_zstd() + import .H5Zzstd: ZstdFilter + register_filter(ZstdFilter) end end From c450eba124f424db41f06c310bdfcc3e585ff03a Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Wed, 8 Dec 2021 02:08:00 -0500 Subject: [PATCH 16/28] Define filters API on the type alone --- src/filters/filters.jl | 135 ++++++++++++++++++++++------------------- 1 file changed, 73 insertions(+), 62 deletions(-) diff --git a/src/filters/filters.jl b/src/filters/filters.jl index c63f25baa..581abbc5a 100644 --- a/src/filters/filters.jl +++ b/src/filters/filters.jl @@ -15,9 +15,7 @@ See the Extended Help for information on implementing a new filter. ## Filter interface -The Filter interface can be implemented upon either the Filter subtype or an instance. -Implementing the interface on a Filter subtype is recommended. -The instance methods default to calling the same method on the type. +The Filter interface is implemented upon the Filter subtype. See API.h5z_register for details. @@ -50,54 +48,60 @@ Maps filter id to filter type. const FILTERS = Dict{API.H5Z_filter_t, Type{<: Filter}}() """ - filterid(::F) where {F <: Filter} filterid(F) where {F <: Filter} The internal filter id of a filter of type `F`. """ -filterid(::F) where {F<:Filter} = filterid(F) +filterid """ - encoder_present(::F) where {F<:Filter} + encoder_present(::Type{F}) where {F<:Filter} Can the filter have an encode or compress the data? Defaults to true. Returns a Bool. See `API.h5z_register`. """ -encoder_present(::F) where {F<:Filter} = encoder_present(F) encoder_present(::Type{F}) where {F<:Filter} = true """ - decoder_present(::F) where {F<:Filter} + decoder_present(::Type{F}) where {F<:Filter} Can the filter decode or decompress the data? Defaults to true. Returns a Bool. See `API.h5z_register` """ -decoder_present(::F) where {F<:Filter} = decoder_present(F) decoder_present(::Type{F}) where {F<:Filter} = true """ - filtername(::F) where {F<:Filter} + filtername(::Type{F}) where {F<:Filter} What is the name of a filter? Defaults to "Unnamed Filter" Returns a String describing the filter. See `API.h5z_register` """ -filtername(::F) where {F<:Filter} = filtername(F) filtername(::Type{F}) where {F<:Filter} = "Unnamed Filter" """ - can_apply_func(::F) where {F<:Filter} + can_apply_func(::Type{F}) where {F<:Filter} Return a function indicating whether the filter can be applied or `nothing` if no function exists. The function signature is `func(dcpl_id::API.hid_t, type_id::API.hid_t, space_id::API.hid_t)`. See `API.h5z_register` """ -can_apply_func(::F) where {F<:Filter} = can_apply_func(F) can_apply_func(::Type{F}) where {F<:Filter} = nothing + +""" + can_apply_cfunc(::Type{F}) where {F<:Filter} + +Return a C function pointer for the can apply function. +By default, this will return the result of using `@cfunction` on the function +specified by `can_apply_func(F)` or `C_NULL` if `nothing`. + +Overriding this will allow `@cfunction` to return a `Ptr{Nothing}` rather +than a `CFunction`` closure which may not work on all systems. +""" function can_apply_cfunc(::Type{F}) where {F<:Filter} func = can_apply_func(F) if func === nothing @@ -108,14 +112,24 @@ function can_apply_cfunc(::Type{F}) where {F<:Filter} end """ - set_local_func(::F) where {F<:Filter} + set_local_func(::Type{F}) where {F<:Filter} Return a function that sets dataset specific parameters or `nothing` if no function exists. The function signature is `func(dcpl_id::API.hid_t, type_id::API.hid_t, space_id::API.hid_t)`. See `API.h5z_register` """ -set_local_func(::F) where {F<:Filter} = set_local_func(F) set_local_func(::Type{F}) where {F<:Filter} = nothing + +""" + set_local_cfunc(::Type{F}) where {F<:Filter} + +Return a C function pointer for the set local function. +By default, this will return the result of using `@cfunction` on the function +specified by `set_local_func(F)` or `C_NULL` if `nothing`. + +Overriding this will allow `@cfunction` to return a `Ptr{Nothing}` rather +than a `CFunction`` closure which may not work on all systems. +""" function set_local_cfunc(::Type{F}) where {F<:Filter} func = set_local_func(F) if func === nothing @@ -127,14 +141,24 @@ end """ - filter_func(::F) where {F<:Filter} + filter_func(::Type{F}) where {F<:Filter} Returns a function that performs the actual filtering. See `API.h5z_register` """ -filter_func(::F) where {F<:Filter} = filter_func(F) filter_func(::Type{F}) where {F<:Filter} = nothing + +""" + filter_cfunc(::Type{F}) where {F<:Filter} + +Return a C function pointer for the filter function. +By default, this will return the result of using `@cfunction` on the function +specified by `filter_func(F)` or will throw an error if `nothing`. + +Overriding this will allow `@cfunction` to return a `Ptr{Nothing}` rather +than a `CFunction`` closure which may not work on all systems. +""" function filter_cfunc(::Type{F}) where {F<:Filter} func = filter_func(F) if func === nothing @@ -146,6 +170,38 @@ function filter_cfunc(::Type{F}) where {F<:Filter} return c_filter_func end +# Generic implementation of register_filter +""" + register_filter(::Type{F}) where F <: Filter + +Register the filter with the HDF5 library via API.h5z_register. +Also add F to the FILTERS dictionary. +""" +function register_filter(::Type{F}) where F <: Filter + id = filterid(F) + encoder = encoder_present(F) + decoder = decoder_present(F) + name = filtername(F) + can_apply = can_apply_cfunc(F) + set_local = set_local_cfunc(F) + func = filter_cfunc(F) + GC.@preserve name begin + API.h5z_register(API.H5Z_class_t( + API.H5Z_CLASS_T_VERS, + id, + encoder, + decoder, + pointer(name), + can_apply, + set_local, + func + )) + end + FILTERS[id] = F + return nothing +end + + struct UnknownFilter <: Filter filter_id::API.H5Z_filter_t flags::Cuint @@ -249,49 +305,6 @@ function Base.push!(p::FilterPipeline, f::UnknownFilter) end end -# Generic implementation of register_filter -""" - register_filter(::Type{F}) where F <: Filter - register_filter(filter::F) where F <: Filter - -Register the filter with the HDF5 library via API.h5z_register. -Also add F to the FILTERS dictionary. -""" -function register_filter(::Type{F}) where F <: Filter - id = filterid(F) - encoder = encoder_present(F) - decoder = decoder_present(F) - name = filtername(F) - can_apply = can_apply_cfunc(F) - set_local = set_local_cfunc(F) - func = filter_cfunc(F) - GC.@preserve name begin - API.h5z_register(API.H5Z_class_t( - API.H5Z_CLASS_T_VERS, - id, - encoder, - decoder, - pointer(name), - can_apply, - set_local, - func - )) - end - FILTERS[id] = F - return nothing -end -register_filter(::F) where {F<:Filter} = register_filter(F) - - -function register_filters() - # Load filter codec packages which should trigger Requires.jl - @eval begin - using Blosc - using CodecBzip2 - using CodecLz4 - using CodecZstd - end -end include("builtin.jl") @@ -318,6 +331,4 @@ function __init__() end end -precompile(register_filters, ()) - end # module From 32560358fc46fd6ca1bcdf283ece4b4592d9ac86 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Wed, 8 Dec 2021 03:25:56 -0500 Subject: [PATCH 17/28] H5Zblosc, H5Zbzip2, H5Zlz4, H5Zzstd as subdir pkgs --- Project.toml | 16 ++++++------ src/HDF5.jl | 4 +++ src/filters/H5Zblosc/Project.toml | 12 +++++++++ src/filters/{ => H5Zblosc/src}/H5Zblosc.jl | 5 ++-- src/filters/H5Zbzip2/Project.toml | 12 +++++++++ src/filters/{ => H5Zbzip2/src}/H5Zbzip2.jl | 5 ++-- src/filters/H5Zlz4/Project.toml | 12 +++++++++ src/filters/{ => H5Zlz4/src}/H5Zlz4.jl | 5 ++-- src/filters/H5Zzstd/Project.toml | 12 +++++++++ src/filters/{ => H5Zzstd/src}/H5Zzstd.jl | 5 ++-- src/filters/filters.jl | 29 +++++++--------------- src/properties.jl | 2 -- test/filter.jl | 10 ++++---- test/filters/FilterTestUtils.jl | 7 +++--- test/plain.jl | 2 +- test/runtests.jl | 3 +++ 16 files changed, 93 insertions(+), 48 deletions(-) create mode 100644 src/filters/H5Zblosc/Project.toml rename src/filters/{ => H5Zblosc/src}/H5Zblosc.jl (98%) create mode 100644 src/filters/H5Zbzip2/Project.toml rename src/filters/{ => H5Zbzip2/src}/H5Zbzip2.jl (98%) create mode 100644 src/filters/H5Zlz4/Project.toml rename src/filters/{ => H5Zlz4/src}/H5Zlz4.jl (98%) create mode 100644 src/filters/H5Zzstd/Project.toml rename src/filters/{ => H5Zzstd/src}/H5Zzstd.jl (97%) diff --git a/Project.toml b/Project.toml index 9a9e17c71..fbb8eb321 100644 --- a/Project.toml +++ b/Project.toml @@ -3,10 +3,6 @@ uuid = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f" version = "0.16.0" [deps] -Blosc = "a74b3585-a348-5f62-a45c-50e91977d574" -CodecBzip2 = "523fee87-0ab8-5b00-afb7-3ecf72e48cfd" -CodecLz4 = "5ba52731-8f18-5e0d-9241-30f10d1ec561" -CodecZstd = "6b39b394-51ab-5f42-8807-6242bab2b4c2" Compat = "34da2185-b29b-5c13-b0c7-acf172513d20" HDF5_jll = "0234f1f7-429e-5d53-9886-15a909be8d59" Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" @@ -15,19 +11,23 @@ Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Requires = "ae029012-a4dd-5104-9daa-d747884805df" [compat] -CodecBzip2 = "0.7" -CodecLz4 = "0.4" -CodecZstd = "0.7" -Blosc = "0.7.1" Compat = "3.1.0" HDF5_jll = "~1.10.5, ~1.12.0" Requires = "1.0" julia = "1.3" [extras] +Blosc = "a74b3585-a348-5f62-a45c-50e91977d574" CRC32c = "8bf52ea8-c179-5cab-976a-9e18b702a9bc" +CodecBzip2 = "523fee87-0ab8-5b00-afb7-3ecf72e48cfd" +CodecLz4 = "5ba52731-8f18-5e0d-9241-30f10d1ec561" +CodecZstd = "6b39b394-51ab-5f42-8807-6242bab2b4c2" Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" +H5Zblosc = "c8ec2601-a99c-407f-b158-e79c03c2f5f7" +H5Zbzip2 = "094576f2-1e46-4c84-8e32-c46c042eaaa2" +H5Zlz4 = "eb20ec05-5464-47b5-ba41-098e3c1068a3" +H5Zzstd = "f6f2d980-1ec6-471c-a70d-0270e22f1103" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" diff --git a/src/HDF5.jl b/src/HDF5.jl index 5bdd73849..56a797793 100644 --- a/src/HDF5.jl +++ b/src/HDF5.jl @@ -1605,6 +1605,10 @@ function __init__() UTF8_ATTRIBUTE_PROPERTIES.char_encoding = :utf8 @require FileIO="5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" @eval include("fileio.jl") + @require H5Zblosc="c8ec2601-a99c-407f-b158-e79c03c2f5f7" @eval begin + set_blosc!(p::Properties, val::Bool) = val && push!(Filters.FilterPipeline(p), H5Zblosc.BloscFilter()) + set_blosc!(p::Properties, level::Integer) = push!(Filters.FilterPipeline(p), H5Zblosc.BloscFilter(level=level)) + end return nothing end diff --git a/src/filters/H5Zblosc/Project.toml b/src/filters/H5Zblosc/Project.toml new file mode 100644 index 000000000..78149ca8f --- /dev/null +++ b/src/filters/H5Zblosc/Project.toml @@ -0,0 +1,12 @@ +name = "H5Zblosc" +uuid = "c8ec2601-a99c-407f-b158-e79c03c2f5f7" +version = "0.16.0" + +[deps] +Blosc = "a74b3585-a348-5f62-a45c-50e91977d574" +HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f" + +[compat] +HDF5 = "0.16" +Blosc = "0.7.1" +julia = "1.3" diff --git a/src/filters/H5Zblosc.jl b/src/filters/H5Zblosc/src/H5Zblosc.jl similarity index 98% rename from src/filters/H5Zblosc.jl rename to src/filters/H5Zblosc/src/H5Zblosc.jl index 73961e940..2f61be32c 100644 --- a/src/filters/H5Zblosc.jl +++ b/src/filters/H5Zblosc/src/H5Zblosc.jl @@ -161,7 +161,8 @@ function Base.push!(f::FilterPipeline, blosc::BloscFilter) return f end -precompile(register_filter, (BloscFilter,)) -precompile(register_filter, (Type{BloscFilter},)) +function __init__() + register_filter(BloscFilter) +end end # module H5Zblosc \ No newline at end of file diff --git a/src/filters/H5Zbzip2/Project.toml b/src/filters/H5Zbzip2/Project.toml new file mode 100644 index 000000000..659dd37bd --- /dev/null +++ b/src/filters/H5Zbzip2/Project.toml @@ -0,0 +1,12 @@ +name = "H5Zbzip2" +uuid = "094576f2-1e46-4c84-8e32-c46c042eaaa2" +version = "0.16.0" + +[deps] +CodecBzip2 = "523fee87-0ab8-5b00-afb7-3ecf72e48cfd" +HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f" + +[compat] +HDF5 = "0.16" +CodecBzip2 = "0.7" +julia = "1.3" diff --git a/src/filters/H5Zbzip2.jl b/src/filters/H5Zbzip2/src/H5Zbzip2.jl similarity index 98% rename from src/filters/H5Zbzip2.jl rename to src/filters/H5Zbzip2/src/H5Zbzip2.jl index d0b28e574..16c2e3d70 100644 --- a/src/filters/H5Zbzip2.jl +++ b/src/filters/H5Zbzip2/src/H5Zbzip2.jl @@ -179,7 +179,8 @@ filter_cfunc(::Type{Bzip2Filter}) = @cfunction(H5Z_filter_bzip2, Csize_t, (Cuint, Csize_t, Ptr{Cuint}, Csize_t, Ptr{Csize_t}, Ptr{Ptr{Cvoid}})) -precompile(register_filter, (Bzip2Filter,)) -precompile(register_filter, (Type{Bzip2Filter},)) +function __init__() + register_filter(Bzip2Filter) +end end # module H5Zbzip2 \ No newline at end of file diff --git a/src/filters/H5Zlz4/Project.toml b/src/filters/H5Zlz4/Project.toml new file mode 100644 index 000000000..891ced313 --- /dev/null +++ b/src/filters/H5Zlz4/Project.toml @@ -0,0 +1,12 @@ +name = "H5Zlz4" +uuid = "eb20ec05-5464-47b5-ba41-098e3c1068a3" +version = "0.16.0" + +[deps] +CodecLz4 = "5ba52731-8f18-5e0d-9241-30f10d1ec561" +HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f" + +[compat] +HDF5 = "0.16" +CodecLz4 = "0.4" +julia = "1.3" diff --git a/src/filters/H5Zlz4.jl b/src/filters/H5Zlz4/src/H5Zlz4.jl similarity index 98% rename from src/filters/H5Zlz4.jl rename to src/filters/H5Zlz4/src/H5Zlz4.jl index aee5e7e33..bcb224513 100644 --- a/src/filters/H5Zlz4.jl +++ b/src/filters/H5Zlz4/src/H5Zlz4.jl @@ -198,7 +198,8 @@ filter_cfunc(::Type{Lz4Filter}) = @cfunction(H5Z_filter_lz4, Csize_t, (Cuint, Csize_t, Ptr{Cuint}, Csize_t, Ptr{Csize_t}, Ptr{Ptr{Cvoid}})) -precompile(register_filter, (Lz4Filter,)) -precompile(register_filter, (Type{Lz4Filter},)) +function __init__() + register_filter(Lz4Filter) +end end \ No newline at end of file diff --git a/src/filters/H5Zzstd/Project.toml b/src/filters/H5Zzstd/Project.toml new file mode 100644 index 000000000..54e7dab9c --- /dev/null +++ b/src/filters/H5Zzstd/Project.toml @@ -0,0 +1,12 @@ +name = "H5Zzstd" +uuid = "f6f2d980-1ec6-471c-a70d-0270e22f1103" +version = "0.16.0" + +[deps] +CodecZstd = "6b39b394-51ab-5f42-8807-6242bab2b4c2" +HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f" + +[compat] +HDF5 = "0.16" +CodecZstd = "0.7" +julia = "1.3" diff --git a/src/filters/H5Zzstd.jl b/src/filters/H5Zzstd/src/H5Zzstd.jl similarity index 97% rename from src/filters/H5Zzstd.jl rename to src/filters/H5Zzstd/src/H5Zzstd.jl index 1f6d6a305..1387c2f3a 100644 --- a/src/filters/H5Zzstd.jl +++ b/src/filters/H5Zzstd/src/H5Zzstd.jl @@ -101,7 +101,8 @@ filter_cfunc(::Type{ZstdFilter}) = @cfunction(H5Z_filter_zstd, Csize_t, (Cuint, Csize_t, Ptr{Cuint}, Csize_t, Ptr{Csize_t}, Ptr{Ptr{Cvoid}})) -precompile(register_filter, (ZstdFilter,)) -precompile(register_filter, (Type{ZstdFilter},)) +function __init__() + register_filter(ZstdFilter) +end end # module H5Zzstd \ No newline at end of file diff --git a/src/filters/filters.jl b/src/filters/filters.jl index 581abbc5a..0f4b9b23e 100644 --- a/src/filters/filters.jl +++ b/src/filters/filters.jl @@ -308,26 +308,15 @@ end include("builtin.jl") -function __init__() - @require Blosc="a74b3585-a348-5f62-a45c-50e91977d574" @eval begin - include("H5Zblosc.jl") - import .H5Zblosc: BloscFilter - register_filter(BloscFilter) - end - @require CodecBzip2="523fee87-0ab8-5b00-afb7-3ecf72e48cfd" @eval begin - include("H5Zbzip2.jl") - import .H5Zbzip2: Bzip2Filter - register_filter(Bzip2Filter) - end - @require CodecLz4="5ba52731-8f18-5e0d-9241-30f10d1ec561" @eval begin - include("H5Zlz4.jl") - import .H5Zlz4: Lz4Filter - register_filter(Lz4Filter) - end - @require CodecZstd="6b39b394-51ab-5f42-8807-6242bab2b4c2" @eval begin - include("H5Zzstd.jl") - import .H5Zzstd: ZstdFilter - register_filter(ZstdFilter) +macro dev_embedded_filters() + quote + @eval begin + using Pkg + Pkg.develop(path=joinpath(dirname(pathof(HDF5)), "filters", "H5Zblosc")) + Pkg.develop(path=joinpath(dirname(pathof(HDF5)), "filters", "H5Zbzip2")) + Pkg.develop(path=joinpath(dirname(pathof(HDF5)), "filters", "H5Zlz4")) + Pkg.develop(path=joinpath(dirname(pathof(HDF5)), "filters", "H5Zzstd")) + end end end diff --git a/src/properties.jl b/src/properties.jl index aa07029bb..b18950570 100644 --- a/src/properties.jl +++ b/src/properties.jl @@ -407,8 +407,6 @@ set_deflate!(p::Properties, val::Bool) = val && push!(Filters.FilterPipeline(p), set_deflate!(p::Properties, level::Integer) = push!(Filters.FilterPipeline(p), Filters.Deflate(level=level)) set_shuffle!(p::Properties, val::Bool) = val && push!(Filters.FilterPipeline(p), Filters.Shuffle()) set_fletcher32!(p::Properties, val::Bool) = val && push!(Filters.FilterPipeline(p), Filters.Fletcher32()) -set_blosc!(p::Properties, val::Bool) = val && push!(Filters.FilterPipeline(p), Filters.BloscFilter()) -set_blosc!(p::Properties, level::Integer) = push!(Filters.FilterPipeline(p), Filters.BloscFilter(level=level)) class_propertynames(::Type{DatasetCreateProperties}) = ( diff --git a/test/filter.jl b/test/filter.jl index 5729d1230..80afe9dac 100644 --- a/test/filter.jl +++ b/test/filter.jl @@ -1,7 +1,7 @@ using HDF5 using HDF5.Filters using Test -import Blosc, CodecLz4, CodecBzip2, CodecZstd +using H5Zblosc, H5Zlz4, H5Zbzip2, H5Zzstd @testset "filter" begin @@ -37,10 +37,10 @@ write(dsfiltshufdef, data) # Test compression filters compressionFilters = Dict( - "blosc" => Filters.BloscFilter, - "bzip2" => Filters.Bzip2Filter, - "lz4" => Filters.Lz4Filter, - "zstd" => Filters.ZstdFilter + "blosc" => BloscFilter, + "bzip2" => Bzip2Filter, + "lz4" => Lz4Filter, + "zstd" => ZstdFilter ) for (name, filter) in compressionFilters diff --git a/test/filters/FilterTestUtils.jl b/test/filters/FilterTestUtils.jl index 8ef919d1f..b3cd0ac12 100644 --- a/test/filters/FilterTestUtils.jl +++ b/test/filters/FilterTestUtils.jl @@ -6,10 +6,9 @@ This module contains utilities for evaluating and debugging HDF5 Filters. module FilterTestUtils import HDF5.API -import Blosc, CodecLz4, CodecZstd, CodecBzip2 -import HDF5.Filters.H5Zlz4: H5Z_filter_lz4 -import HDF5.Filters.H5Zzstd: H5Z_filter_zstd -import HDF5.Filters.H5Zbzip2: H5Z_filter_bzip2 +import H5Zlz4: H5Z_filter_lz4 +import H5Zzstd: H5Z_filter_zstd +import H5Zbzip2: H5Z_filter_bzip2 using Test export test_filter diff --git a/test/plain.jl b/test/plain.jl index 274134335..4ab36fd69 100644 --- a/test/plain.jl +++ b/test/plain.jl @@ -1,6 +1,6 @@ using HDF5 using CRC32c -using Blosc +using H5Zblosc using Test gatherf(dst_buf, dst_buf_bytes_used, op_data) = HDF5.API.herr_t(0) diff --git a/test/runtests.jl b/test/runtests.jl index 39f3e4cf8..6261d8da8 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2,6 +2,8 @@ using HDF5 using Test using Pkg +HDF5.Filters.@dev_embedded_filters + @info "libhdf5 v$(HDF5.API.h5_get_libversion())" @testset "HDF5.jl" begin @@ -23,6 +25,7 @@ include("table.jl") include("filter.jl") include("chunkstorage.jl") include("fileio.jl") +include("filters/FilterTestUtils.jl") using MPI if HDF5.has_parallel() From b50ac8bd5e5032ea60005ae2c86a4ddd751d80dc Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Wed, 8 Dec 2021 03:44:37 -0500 Subject: [PATCH 18/28] Fix typos in H5Zlz4.jl --- src/filters/H5Zlz4/src/H5Zlz4.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/filters/H5Zlz4/src/H5Zlz4.jl b/src/filters/H5Zlz4/src/H5Zlz4.jl index bcb224513..2f0494a5a 100644 --- a/src/filters/H5Zlz4/src/H5Zlz4.jl +++ b/src/filters/H5Zlz4/src/H5Zlz4.jl @@ -54,7 +54,7 @@ function H5Z_filter_lz4(flags::Cuint, cd_nelmts::Csize_t, origSize = ntoh(unsafe_load(Ptr{UInt64}(rpos))) rpos += 8 # advance the pointer - # Next read the next four bytes from teh buffer as a big endian UInt32 + # Next read the next four bytes from the buffer as a big endian UInt32 # This is the blocksize #i32Buf[] = rpos blockSize = ntoh(unsafe_load(Ptr{UInt32}(rpos))) @@ -73,7 +73,7 @@ function H5Z_filter_lz4(flags::Cuint, cd_nelmts::Csize_t, # Start with the first blockSize while decompSize < origSize # compressedBlockSize = UInt32(0) - if origSize - decompSize < blockSize # the lsat block can be smaller than block size + if origSize - decompSize < blockSize # the last block can be smaller than block size blockSize = origSize - decompSize end From 46c6b8e3f3c6d76788f692db8e2a700b27d53a57 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Wed, 8 Dec 2021 03:45:51 -0500 Subject: [PATCH 19/28] Use PackageSpec to make dev_embedded_filters Julia 1.3 compatible --- src/filters/filters.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/filters/filters.jl b/src/filters/filters.jl index 0f4b9b23e..362d96ae5 100644 --- a/src/filters/filters.jl +++ b/src/filters/filters.jl @@ -312,10 +312,10 @@ macro dev_embedded_filters() quote @eval begin using Pkg - Pkg.develop(path=joinpath(dirname(pathof(HDF5)), "filters", "H5Zblosc")) - Pkg.develop(path=joinpath(dirname(pathof(HDF5)), "filters", "H5Zbzip2")) - Pkg.develop(path=joinpath(dirname(pathof(HDF5)), "filters", "H5Zlz4")) - Pkg.develop(path=joinpath(dirname(pathof(HDF5)), "filters", "H5Zzstd")) + Pkg.develop(PackageSpec(; path=joinpath(dirname(pathof(HDF5)), "filters", "H5Zblosc"))) + Pkg.develop(PackageSpec(; path=joinpath(dirname(pathof(HDF5)), "filters", "H5Zbzip2"))) + Pkg.develop(PackageSpec(; path=joinpath(dirname(pathof(HDF5)), "filters", "H5Zlz4"))) + Pkg.develop(PackageSpec(; path=joinpath(dirname(pathof(HDF5)), "filters", "H5Zzstd"))) end end end From 9fc79f371c445ef05472bdfcd8f770a43c156a05 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Fri, 10 Dec 2021 02:18:32 -0500 Subject: [PATCH 20/28] Add Licenses to subpackages --- licenses/README.md | 6 +- src/filters/H5Zblosc/LICENSE.txt | 17 +++ src/filters/H5Zbzip2/LICENSE.txt | 17 +++ src/filters/H5Zbzip2/THIRDPARTY.txt | 31 +++++ src/filters/H5Zlz4/LICENSE.txt | 17 +++ src/filters/H5Zlz4/THIRDPARTY.txt | 45 ++++++ src/filters/H5Zzstd/LICENSE.txt | 17 +++ src/filters/H5Zzstd/THIRDPARTY.txt | 205 ++++++++++++++++++++++++++++ 8 files changed, 352 insertions(+), 3 deletions(-) create mode 100644 src/filters/H5Zblosc/LICENSE.txt create mode 100644 src/filters/H5Zbzip2/LICENSE.txt create mode 100644 src/filters/H5Zbzip2/THIRDPARTY.txt create mode 100644 src/filters/H5Zlz4/LICENSE.txt create mode 100644 src/filters/H5Zlz4/THIRDPARTY.txt create mode 100644 src/filters/H5Zzstd/LICENSE.txt create mode 100644 src/filters/H5Zzstd/THIRDPARTY.txt diff --git a/licenses/README.md b/licenses/README.md index 16677566d..17b9e7ed2 100644 --- a/licenses/README.md +++ b/licenses/README.md @@ -9,6 +9,6 @@ See the files indicated below for the copyright notices and the licenses of the software from which individual submodules are derived. ## Filter Plugins -* [HDF5.Filters.H5Zbzip2](../src/filters/H5Zbzip2.jl): See [H5Zbzip2_LICENSE.txt](H5Zbzip2_LICENSE.txt) -* [HDF5.Filters.H5Zlz4](../src/filters/H5Zlz4.jl): See [H5Zlz4_LICENSE.txt](H5Zlz4_LICENSE.txt) -* [HDF5.Filters.H5Zzstd](../src/filters/H5Zzstd.jl): See [H5Zzstd_LICENSE.txt](H5Zzstd_LICENSE.txt) \ No newline at end of file +* [H5Zbzip2](../src/filters/H5Zbzip2/src/H5Zbzip2.jl): See [H5Zbzip2_LICENSE.txt](H5Zbzip2_LICENSE.txt) +* [H5Zlz4](../src/filters/H5Zlz4/src/H5Zlz4.jl): See [H5Zlz4_LICENSE.txt](H5Zlz4_LICENSE.txt) +* [H5Zzstd](../src/filters/H5Zzstd/src/H5Zzstd.jl): See [H5Zzstd_LICENSE.txt](H5Zzstd_LICENSE.txt) \ No newline at end of file diff --git a/src/filters/H5Zblosc/LICENSE.txt b/src/filters/H5Zblosc/LICENSE.txt new file mode 100644 index 000000000..b5b30a868 --- /dev/null +++ b/src/filters/H5Zblosc/LICENSE.txt @@ -0,0 +1,17 @@ +The MIT License (MIT) +Copyright (c) 2012-2021: Timothy E. Holy, Simon Kornblith, and contributors: https://github.com/JuliaIO/HDF5.jl/contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software +and associated documentation files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or +substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/src/filters/H5Zbzip2/LICENSE.txt b/src/filters/H5Zbzip2/LICENSE.txt new file mode 100644 index 000000000..b5b30a868 --- /dev/null +++ b/src/filters/H5Zbzip2/LICENSE.txt @@ -0,0 +1,17 @@ +The MIT License (MIT) +Copyright (c) 2012-2021: Timothy E. Holy, Simon Kornblith, and contributors: https://github.com/JuliaIO/HDF5.jl/contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software +and associated documentation files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or +substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/src/filters/H5Zbzip2/THIRDPARTY.txt b/src/filters/H5Zbzip2/THIRDPARTY.txt new file mode 100644 index 000000000..812ff0293 --- /dev/null +++ b/src/filters/H5Zbzip2/THIRDPARTY.txt @@ -0,0 +1,31 @@ +H5Z_filter_bzip2 in H5Zbzip2.jl was derived from H5Zbzip2.c from PyTables: + +Copyright Notice and Statement for PyTables Software Library and Utilities: +Copyright (c) 2002-2004 by Francesc Alted +Copyright (c) 2005-2007 by Carabos Coop. V. +Copyright (c) 2008-2010 by Francesc Alted +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +a. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +b. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the + distribution. +c. Neither the name of Francesc Alted nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/src/filters/H5Zlz4/LICENSE.txt b/src/filters/H5Zlz4/LICENSE.txt new file mode 100644 index 000000000..b5b30a868 --- /dev/null +++ b/src/filters/H5Zlz4/LICENSE.txt @@ -0,0 +1,17 @@ +The MIT License (MIT) +Copyright (c) 2012-2021: Timothy E. Holy, Simon Kornblith, and contributors: https://github.com/JuliaIO/HDF5.jl/contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software +and associated documentation files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or +substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/src/filters/H5Zlz4/THIRDPARTY.txt b/src/filters/H5Zlz4/THIRDPARTY.txt new file mode 100644 index 000000000..745176c31 --- /dev/null +++ b/src/filters/H5Zlz4/THIRDPARTY.txt @@ -0,0 +1,45 @@ +H5Zlz4.jl is derived from H5Zlz4.c from HDF5 Group + +================================================================================ +LZ4 filter plugin license +================================================================================ + +Copyright Notice and License Terms for +HDF5 LZ4 compression filter plugin +----------------------------------------------------------------------------- + +HDF5 LZ4 compression filter plugin +Copyright 2013-2015 by The HDF Group. + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted for any purpose (including commercial purposes) +provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or materials provided with the distribution. + +3. In addition, redistributions of modified forms of the source or binary + code must carry prominent notices stating that the original code was + changed and the date of the change. + +4. All publications or advertising materials mentioning features or use of + this software are asked, but not required, to acknowledge that it was + developed by The HDF Group and credit the contributors. + +5. Neither the name of The HDF Group, nor the name of any Contributor may + be used to endorse or promote products derived from this software + without specific prior written permission from The HDF Group or the + Contributor, respectively. + +DISCLAIMER: +THIS SOFTWARE IS PROVIDED BY THE HDF GROUP AND THE CONTRIBUTORS +"AS IS" WITH NO WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED. In no +event shall The HDF Group or the Contributors be liable for any damages +suffered by the users arising out of the use of this software, even if +advised of the possibility of such damage. \ No newline at end of file diff --git a/src/filters/H5Zzstd/LICENSE.txt b/src/filters/H5Zzstd/LICENSE.txt new file mode 100644 index 000000000..b5b30a868 --- /dev/null +++ b/src/filters/H5Zzstd/LICENSE.txt @@ -0,0 +1,17 @@ +The MIT License (MIT) +Copyright (c) 2012-2021: Timothy E. Holy, Simon Kornblith, and contributors: https://github.com/JuliaIO/HDF5.jl/contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software +and associated documentation files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or +substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/src/filters/H5Zzstd/THIRDPARTY.txt b/src/filters/H5Zzstd/THIRDPARTY.txt new file mode 100644 index 000000000..4c5921650 --- /dev/null +++ b/src/filters/H5Zzstd/THIRDPARTY.txt @@ -0,0 +1,205 @@ +H5Zzstd.jl is derived from +https://github.com/aparamon/HDF5Plugin-Zstandard, zstd_h5plugin.c +under Apache License Version 2.0: + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file From 7a2efd892e873f9e5843499fb7a0076063b7005e Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Fri, 10 Dec 2021 02:35:12 -0500 Subject: [PATCH 21/28] H5Zbzip2: Add 32-bit Windows support --- src/filters/H5Zbzip2/src/H5Zbzip2.jl | 77 +++++++++++++++++++--------- 1 file changed, 54 insertions(+), 23 deletions(-) diff --git a/src/filters/H5Zbzip2/src/H5Zbzip2.jl b/src/filters/H5Zbzip2/src/H5Zbzip2.jl index 16c2e3d70..3c2c4148d 100644 --- a/src/filters/H5Zbzip2/src/H5Zbzip2.jl +++ b/src/filters/H5Zbzip2/src/H5Zbzip2.jl @@ -137,32 +137,63 @@ end # function H5Z_filter_bzip2 # Need stdcall for 32-bit Windows? function BZ2_bzBuffToBuffCompress(dest, destLen, source, sourceLen, blockSize100k, verbosity, workFactor) - return ccall( - (:BZ2_bzBuffToBuffCompress, libbzip2), - Cint, - (Ptr{Cchar}, Ptr{Cuint}, Ptr{Cchar}, Cuint, Cint, Cint, Cint), - dest, - destLen, - source, - sourceLen, - blockSize100k, - verbosity, - workFactor - ) + @static if CodecBzip2.WIN32 + return ccall( + ("BZ2_bzBuffToBuffCompress@28", libbzip2), + stdcall, + Cint, + (Ptr{Cchar}, Ptr{Cuint}, Ptr{Cchar}, Cuint, Cint, Cint, Cint), + dest, + destLen, + source, + sourceLen, + blockSize100k, + verbosity, + workFactor + ) + else + return ccall( + (:BZ2_bzBuffToBuffCompress, libbzip2), + Cint, + (Ptr{Cchar}, Ptr{Cuint}, Ptr{Cchar}, Cuint, Cint, Cint, Cint), + dest, + destLen, + source, + sourceLen, + blockSize100k, + verbosity, + workFactor + ) + end end function BZ2_bzBuffToBuffDecompress(dest, destLen, source, sourceLen, small, verbosity) - return ccall( - (:BZ2_bzBuffToBuffDecompress, libbzip2), - Cint, - (Ptr{Cchar}, Ptr{Cuint}, Ptr{Cchar}, Cuint, Cint, Cint), - dest, - destLen, - source, - sourceLen, - small, - verbosity - ) + @static if CodecBzip2.WIN32 + return ccall( + ("BZ2_bzBuffToBuffDecompress@24", libbzip2), + stdcall, + Cint, + (Ptr{Cchar}, Ptr{Cuint}, Ptr{Cchar}, Cuint, Cint, Cint), + dest, + destLen, + source, + sourceLen, + small, + verbosity + ) + else + return ccall( + (:BZ2_bzBuffToBuffDecompress, libbzip2), + Cint, + (Ptr{Cchar}, Ptr{Cuint}, Ptr{Cchar}, Cuint, Cint, Cint), + dest, + destLen, + source, + sourceLen, + small, + verbosity + ) + end end # Filters Module From 2ab4410268a083fb710a7740ea565bc9760e27b2 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Fri, 10 Dec 2021 21:55:37 -0500 Subject: [PATCH 22/28] Reorganized licenses, create root THIRDPARTY.md --- THIRDPARTY.md | 14 +++ licenses/H5Zbzip2_LICENSE.txt | 29 ----- licenses/H5Zlz4_LICENSE.txt | 43 -------- licenses/H5Zzstd_LICENSE.txt | 201 ---------------------------------- licenses/README.md | 14 --- 5 files changed, 14 insertions(+), 287 deletions(-) create mode 100644 THIRDPARTY.md delete mode 100644 licenses/H5Zbzip2_LICENSE.txt delete mode 100644 licenses/H5Zlz4_LICENSE.txt delete mode 100644 licenses/H5Zzstd_LICENSE.txt delete mode 100644 licenses/README.md diff --git a/THIRDPARTY.md b/THIRDPARTY.md new file mode 100644 index 000000000..719f1734d --- /dev/null +++ b/THIRDPARTY.md @@ -0,0 +1,14 @@ +# Third Party Licenses + +HDF5.jl contains several derivative works of open source software. + +In particular, the following submodules are licensed as derivative works from third-parties. +Original and derivative code in HDF5.jl is licensed according to [LICENSE.txt](LICENSE.txt) +as permitted by licenses for the original software from which they may be derived. +See the files indicated below for the copyright notices and the licenses of the original +software from which individual submodules are derived. + +## Filter Plugins +* [H5Zbzip2](src/filters/H5Zbzip2/src/H5Zbzip2.jl): See [src/filters/H5Zbzip2/THIRDPARTY.txt](src/filters/H5Zbzip2/THIRDPARTY.txt) +* [H5Zlz4](src/filters/H5Zlz4/src/H5Zlz4.jl): See [src/filters/H5Zlz4/THIRDPARTY.txt](src/filters/H5Zlz4/THIRDPARTY.txt) +* [H5Zzstd](src/filters/H5Zzstd/src/H5Zzstd.jl): See [src/filters/H5Zzstd/THIRDPARTY.txt](src/filters/H5Zzstd/THIRDPARTY.txt) \ No newline at end of file diff --git a/licenses/H5Zbzip2_LICENSE.txt b/licenses/H5Zbzip2_LICENSE.txt deleted file mode 100644 index 46c548303..000000000 --- a/licenses/H5Zbzip2_LICENSE.txt +++ /dev/null @@ -1,29 +0,0 @@ -Copyright Notice and Statement for PyTables Software Library and Utilities: -Copyright (c) 2002-2004 by Francesc Alted -Copyright (c) 2005-2007 by Carabos Coop. V. -Copyright (c) 2008-2010 by Francesc Alted -All rights reserved. -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: -a. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. -b. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the - distribution. -c. Neither the name of Francesc Alted nor the names of its - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - diff --git a/licenses/H5Zlz4_LICENSE.txt b/licenses/H5Zlz4_LICENSE.txt deleted file mode 100644 index 3257bb370..000000000 --- a/licenses/H5Zlz4_LICENSE.txt +++ /dev/null @@ -1,43 +0,0 @@ -================================================================================ -LZ4 filter plugin license -================================================================================ - -Copyright Notice and License Terms for -HDF5 LZ4 compression filter plugin ------------------------------------------------------------------------------ - -HDF5 LZ4 compression filter plugin -Copyright 2013-2015 by The HDF Group. - -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted for any purpose (including commercial purposes) -provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, - this list of conditions, and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions, and the following disclaimer in the documentation - and/or materials provided with the distribution. - -3. In addition, redistributions of modified forms of the source or binary - code must carry prominent notices stating that the original code was - changed and the date of the change. - -4. All publications or advertising materials mentioning features or use of - this software are asked, but not required, to acknowledge that it was - developed by The HDF Group and credit the contributors. - -5. Neither the name of The HDF Group, nor the name of any Contributor may - be used to endorse or promote products derived from this software - without specific prior written permission from The HDF Group or the - Contributor, respectively. - -DISCLAIMER: -THIS SOFTWARE IS PROVIDED BY THE HDF GROUP AND THE CONTRIBUTORS -"AS IS" WITH NO WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED. In no -event shall The HDF Group or the Contributors be liable for any damages -suffered by the users arising out of the use of this software, even if -advised of the possibility of such damage. \ No newline at end of file diff --git a/licenses/H5Zzstd_LICENSE.txt b/licenses/H5Zzstd_LICENSE.txt deleted file mode 100644 index 9c8f3ea08..000000000 --- a/licenses/H5Zzstd_LICENSE.txt +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. \ No newline at end of file diff --git a/licenses/README.md b/licenses/README.md deleted file mode 100644 index 17b9e7ed2..000000000 --- a/licenses/README.md +++ /dev/null @@ -1,14 +0,0 @@ -# Third Party Licenses - -HDF5.jl contains several derivative works of open source software. - -In particular, the following submodules are licensed from third-parties. -Original and derivative code in HDF5.jl is licensed according to [LICENSE.txt](../LICENSE.txt) -as permitted by licenses for the original software from which they may be derived. -See the files indicated below for the copyright notices and the licenses of the original -software from which individual submodules are derived. - -## Filter Plugins -* [H5Zbzip2](../src/filters/H5Zbzip2/src/H5Zbzip2.jl): See [H5Zbzip2_LICENSE.txt](H5Zbzip2_LICENSE.txt) -* [H5Zlz4](../src/filters/H5Zlz4/src/H5Zlz4.jl): See [H5Zlz4_LICENSE.txt](H5Zlz4_LICENSE.txt) -* [H5Zzstd](../src/filters/H5Zzstd/src/H5Zzstd.jl): See [H5Zzstd_LICENSE.txt](H5Zzstd_LICENSE.txt) \ No newline at end of file From 740983ed4bc99ec56ba09aa1f6915d789066cec5 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Sat, 11 Dec 2021 03:31:43 -0500 Subject: [PATCH 23/28] Debug tests for Julia 1.8 --- src/filters/H5Zblosc/src/H5Zblosc.jl | 14 ++++++++++---- test/external.jl | 5 +++++ test/filters/FilterTestUtils.jl | 8 ++++---- test/plain.jl | 2 -- test/runtests.jl | 26 +++++++++++++++++++++++++- test/swmr.jl | 9 ++++++++- 6 files changed, 52 insertions(+), 12 deletions(-) diff --git a/src/filters/H5Zblosc/src/H5Zblosc.jl b/src/filters/H5Zblosc/src/H5Zblosc.jl index 2f61be32c..e82ea5c55 100644 --- a/src/filters/H5Zblosc/src/H5Zblosc.jl +++ b/src/filters/H5Zblosc/src/H5Zblosc.jl @@ -91,10 +91,16 @@ function blosc_filter(flags::Cuint, cd_nelmts::Csize_t, # uncompressed chunk size but it should not be used in a general # cases since other filters in the pipeline can modify the buffer # size. - outbuf_size, cbytes, blocksize = Blosc.cbuffer_sizes(unsafe_load(buf)) - outbuf = Libc.malloc(outbuf_size) - outbuf == C_NULL && return Csize_t(0) - status = Blosc.blosc_decompress(unsafe_load(buf), outbuf, outbuf_size) + in = unsafe_load(buf) + outbuf_size, cbytes, blocksize = Blosc.cbuffer_sizes(in) + # We need to preserve the output of cbuffer_sizes under Julia 1.8 (as of 2021/12/11) + # Otherwise their Refs get freed and the buffer becomes corrupted + # If this fails, consider copying `in` or wrapping that array. + GC.@preserve outbuf_size cbytes blocksize begin + outbuf = Libc.malloc(outbuf_size) + outbuf == C_NULL && return Csize_t(0) + status = Blosc.blosc_decompress(in, outbuf, outbuf_size) + end status <= 0 && (Libc.free(outbuf); return Csize_t(0)) end diff --git a/test/external.jl b/test/external.jl index a22b82347..82fa92658 100644 --- a/test/external.jl +++ b/test/external.jl @@ -20,22 +20,26 @@ close(target_file) # test both an HDF5.File and an HDF5.Group for first argument HDF5.create_external(source_file, "ext_link", target_file.filename, "target_group") HDF5.create_external(agroup, "ext_link", target_file.filename, "target_group") +close(agroup) # write some things via the external link new_group = create_group(source_file["ext_link"], "new_group") new_group["abc"] = "abc" new_group["1"] = 1 new_group["1.1"] = 1.1 +close(new_group) # read things from target_group via exernal link created with HDF5File argument group = source_file["ext_link"] @test read(group["abc"]) == "abc" @test read(group["1"]) == 1 @test read(group["1.1"]) == 1.1 +close(group) # read things from target_group via the external link created with HDF5.Group argument groupalt = source_file["agroup/ext_link"] @test read(groupalt["abc"]) == "abc" @test read(groupalt["1"]) == 1 @test read(groupalt["1.1"]) == 1.1 +close(groupalt) close(source_file) ##### tests that should be included but don't work @@ -50,5 +54,6 @@ close(source_file) rm(fn1) # rm(fn2) +@debug "external tests did not delete" fn2 end # testset external diff --git a/test/filters/FilterTestUtils.jl b/test/filters/FilterTestUtils.jl index b3cd0ac12..86c58cf97 100644 --- a/test/filters/FilterTestUtils.jl +++ b/test/filters/FilterTestUtils.jl @@ -36,7 +36,7 @@ function test_filter_compress!(filter_func, flags::Cuint, cd_values::Vector{Cuin Base.unsafe_convert(Ptr{Csize_t}, buf_size), Base.unsafe_convert(Ptr{Ptr{Cvoid}}, buf) ) - @info "Compression:" ret_code buf_size[] + @debug "Compression:" ret_code buf_size[] if ret_code <= 0 error("Test compression failed: $ret_code.") end @@ -57,7 +57,7 @@ function test_filter_decompress!(filter_func, flags::Cuint, cd_values::Vector{Cu Base.unsafe_convert(Ptr{Csize_t},buf_size), Base.unsafe_convert(Ptr{Ptr{Cvoid}}, buf) ) - @info "Decompression:" ret_code buf_size[] + @debug "Decompression:" ret_code buf_size[] end return ret_code end @@ -75,14 +75,14 @@ function test_filter(filter_func; cd_values::Vector{Cuint} = Cuint[], data = one if nbytes_decompressed > 0 # ret_code is the number of bytes out round_trip_data = unsafe_wrap(Array,Ptr{UInt8}(buf[]), nbytes_decompressed) - @info "Is the data the same after a roundtrip?" data == round_trip_data + @debug "Is the data the same after a roundtrip?" data == round_trip_data end catch err rethrow(err) finally test_filter_cleanup!(buf) end - @info "Compression Ratio" nbytes_compressed / nbytes_decompressed + @debug "Compression Ratio" nbytes_compressed / nbytes_decompressed return nbytes_compressed, nbytes_decompressed end diff --git a/test/plain.jl b/test/plain.jl index 5677798ee..93cba9f0f 100644 --- a/test/plain.jl +++ b/test/plain.jl @@ -12,7 +12,6 @@ function scatterf(src_buf, src_buf_bytes_used, op_data) A = [1,2,3,4] unsafe_store!(src_buf, pointer(A)) unsafe_store!(src_buf_bytes_used, sizeof(A)) - @debug "op_data: " opdata return HDF5.API.herr_t(0) end scatterf_bad(src_buf, src_buf_bytes_used, op_data) = HDF5.API.herr_t(-1) @@ -20,7 +19,6 @@ function scatterf_data(src_buf, src_buf_bytes_used, op_data) A = [1,2,3,4] unsafe_store!(src_buf, pointer(A)) unsafe_store!(src_buf_bytes_used, sizeof(A)) - @debug "op_data: " opdata return HDF5.API.herr_t((op_data == 9)-1) end diff --git a/test/runtests.jl b/test/runtests.jl index 6261d8da8..856bc89a1 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -6,25 +6,46 @@ HDF5.Filters.@dev_embedded_filters @info "libhdf5 v$(HDF5.API.h5_get_libversion())" -@testset "HDF5.jl" begin +# To debug HDF5.jl tests, uncomment the next line +# ENV["JULIA_DEBUG"] = "Main" +@testset verbose=true "HDF5.jl" begin + +@debug "plain" include("plain.jl") +@debug "compound" include("compound.jl") +@debug "custom" include("custom.jl") +@debug "reference" include("reference.jl") +@debug "dataspace" include("dataspace.jl") +@debug "hyperslab" include("hyperslab.jl") +@debug "readremote" include("readremote.jl") +@debug "extend_test" include("extend_test.jl") +@debug "gc" include("gc.jl") +@debug "external" include("external.jl") +@debug "swmr" include("swmr.jl") +@debug "mmap" include("mmap.jl") +@debug "properties" include("properties.jl") +@debug "table" include("table.jl") +@debug "filter" include("filter.jl") +@debug "chunkstorage" include("chunkstorage.jl") +@debug "fileio" include("fileio.jl") +@debug "filter test utils" include("filters/FilterTestUtils.jl") using MPI @@ -33,4 +54,7 @@ if HDF5.has_parallel() include("mpio.jl") end +# Clean up after all resources +HDF5.API.h5_close() + end diff --git a/test/swmr.jl b/test/swmr.jl index ae216b332..0cc6e7e2a 100644 --- a/test/swmr.jl +++ b/test/swmr.jl @@ -5,7 +5,9 @@ using Test using Distributed if nprocs() == 1 - addprocs(1) + procs = addprocs(1) +else + procs = Int64[] end @everywhere using HDF5 @@ -114,4 +116,9 @@ end end rm(fname) # cleanup file created by swmr tests + +if nprocs() > 1 + rmprocs(procs) +end + end # testset swmr From a0de87500bb143c6b53dfab1fc89bf7c9fe9abf7 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Sat, 11 Dec 2021 03:36:57 -0500 Subject: [PATCH 24/28] Remove verbose testset for Julia 1.3 --- test/runtests.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/runtests.jl b/test/runtests.jl index 856bc89a1..02f59c136 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -9,7 +9,7 @@ HDF5.Filters.@dev_embedded_filters # To debug HDF5.jl tests, uncomment the next line # ENV["JULIA_DEBUG"] = "Main" -@testset verbose=true "HDF5.jl" begin +@testset "HDF5.jl" begin @debug "plain" include("plain.jl") From dcc8554094166aef3df5236fc6e1449c2861bcb0 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Sat, 11 Dec 2021 15:01:24 -0500 Subject: [PATCH 25/28] Comment out rmprocs to demo nightly stall --- test/swmr.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/swmr.jl b/test/swmr.jl index 0cc6e7e2a..087ce3cfb 100644 --- a/test/swmr.jl +++ b/test/swmr.jl @@ -118,7 +118,7 @@ end rm(fname) # cleanup file created by swmr tests if nprocs() > 1 - rmprocs(procs) +# rmprocs(procs) end end # testset swmr From a008acb5814fe3c9e7b4046b4b644968f8c8e55e Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Sat, 11 Dec 2021 15:22:30 -0500 Subject: [PATCH 26/28] Enable debugging, be verbose when register filters --- src/filters/filters.jl | 1 + test/runtests.jl | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/filters/filters.jl b/src/filters/filters.jl index 362d96ae5..54066ccd0 100644 --- a/src/filters/filters.jl +++ b/src/filters/filters.jl @@ -198,6 +198,7 @@ function register_filter(::Type{F}) where F <: Filter )) end FILTERS[id] = F + @info "Registering filter" F return nothing end diff --git a/test/runtests.jl b/test/runtests.jl index 02f59c136..e8b23757e 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -7,7 +7,7 @@ HDF5.Filters.@dev_embedded_filters @info "libhdf5 v$(HDF5.API.h5_get_libversion())" # To debug HDF5.jl tests, uncomment the next line -# ENV["JULIA_DEBUG"] = "Main" +ENV["JULIA_DEBUG"] = "Main" @testset "HDF5.jl" begin From 93255ac888a0565a9734db1f66ff5496c9121e9b Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Tue, 14 Dec 2021 11:39:48 -0500 Subject: [PATCH 27/28] Check ability to observe stall condition --- test/runtests.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index e8b23757e..eb2b0e8b9 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -7,9 +7,9 @@ HDF5.Filters.@dev_embedded_filters @info "libhdf5 v$(HDF5.API.h5_get_libversion())" # To debug HDF5.jl tests, uncomment the next line -ENV["JULIA_DEBUG"] = "Main" +# ENV["JULIA_DEBUG"] = "Main" -@testset "HDF5.jl" begin +#@testset "HDF5.jl" begin @debug "plain" include("plain.jl") @@ -57,4 +57,4 @@ end # Clean up after all resources HDF5.API.h5_close() -end +#end From 5d966557a42d6d9302ac1d2fa17f5cf6ef204b29 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Tue, 14 Dec 2021 12:09:12 -0500 Subject: [PATCH 28/28] Turn debugging on --- test/runtests.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/runtests.jl b/test/runtests.jl index eb2b0e8b9..cef6b934e 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -7,7 +7,7 @@ HDF5.Filters.@dev_embedded_filters @info "libhdf5 v$(HDF5.API.h5_get_libversion())" # To debug HDF5.jl tests, uncomment the next line -# ENV["JULIA_DEBUG"] = "Main" +ENV["JULIA_DEBUG"] = "Main" #@testset "HDF5.jl" begin