JuliaIO · mkitti · Nov 1, 2021 · Nov 2, 2021 · Nov 15, 2021 · Nov 15, 2021
diff --git a/Project.toml b/Project.toml
@@ -3,7 +3,6 @@ uuid = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f"
 version = "0.16.0"
 
 [deps]
-Blosc = "a74b3585-a348-5f62-a45c-50e91977d574"
 Compat = "34da2185-b29b-5c13-b0c7-acf172513d20"
 HDF5_jll = "0234f1f7-429e-5d53-9886-15a909be8d59"
 Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
@@ -12,16 +11,23 @@ Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 Requires = "ae029012-a4dd-5104-9daa-d747884805df"
 
 [compat]
-Blosc = "0.7.1"
 Compat = "3.1.0"
 HDF5_jll = "~1.10.5, ~1.12.0"
 Requires = "1.0"
 julia = "1.3"
 
 [extras]
+Blosc = "a74b3585-a348-5f62-a45c-50e91977d574"
 CRC32c = "8bf52ea8-c179-5cab-976a-9e18b702a9bc"
+CodecBzip2 = "523fee87-0ab8-5b00-afb7-3ecf72e48cfd"
+CodecLz4 = "5ba52731-8f18-5e0d-9241-30f10d1ec561"
+CodecZstd = "6b39b394-51ab-5f42-8807-6242bab2b4c2"
 Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
 FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
+H5Zblosc = "c8ec2601-a99c-407f-b158-e79c03c2f5f7"
+H5Zbzip2 = "094576f2-1e46-4c84-8e32-c46c042eaaa2"
+H5Zlz4 = "eb20ec05-5464-47b5-ba41-098e3c1068a3"
+H5Zzstd = "f6f2d980-1ec6-471c-a70d-0270e22f1103"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"

diff --git a/THIRDPARTY.md b/THIRDPARTY.md
@@ -0,0 +1,14 @@
+# Third Party Licenses
+
+HDF5.jl contains several derivative works of open source software.
+
+In particular, the following submodules are licensed as derivative works from third-parties.
+Original and derivative code in HDF5.jl is licensed according to [LICENSE.txt](LICENSE.txt)
+as permitted by licenses for the original software from which they may be derived.
+See the files indicated below for the copyright notices and the licenses of the original
+software from which individual submodules are derived.
+
+## Filter Plugins
+* [H5Zbzip2](src/filters/H5Zbzip2/src/H5Zbzip2.jl): See [src/filters/H5Zbzip2/THIRDPARTY.txt](src/filters/H5Zbzip2/THIRDPARTY.txt)
+* [H5Zlz4](src/filters/H5Zlz4/src/H5Zlz4.jl): See [src/filters/H5Zlz4/THIRDPARTY.txt](src/filters/H5Zlz4/THIRDPARTY.txt)
+* [H5Zzstd](src/filters/H5Zzstd/src/H5Zzstd.jl): See [src/filters/H5Zzstd/THIRDPARTY.txt](src/filters/H5Zzstd/THIRDPARTY.txt)
diff --git a/src/HDF5.jl b/src/HDF5.jl
@@ -1600,8 +1600,6 @@ function __init__()
         ENV["HDF5_USE_FILE_LOCKING"] = "FALSE"
     end
 
-    Filters.register_blosc()
-
     # use our own error handling machinery (i.e. turn off automatic error printing)
     API.h5e_set_auto(API.H5E_DEFAULT, C_NULL, C_NULL)
 
@@ -1614,6 +1612,10 @@ function __init__()
     UTF8_ATTRIBUTE_PROPERTIES.char_encoding = :utf8
 
     @require FileIO="5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" @eval include("fileio.jl")
+    @require H5Zblosc="c8ec2601-a99c-407f-b158-e79c03c2f5f7" @eval begin
+        set_blosc!(p::Properties, val::Bool) = val && push!(Filters.FilterPipeline(p), H5Zblosc.BloscFilter())
+        set_blosc!(p::Properties, level::Integer) = push!(Filters.FilterPipeline(p), H5Zblosc.BloscFilter(level=level))
+    end
 
     return nothing
 end

diff --git a/src/filters/H5Zblosc/LICENSE.txt b/src/filters/H5Zblosc/LICENSE.txt
@@ -0,0 +1,17 @@
+The MIT License (MIT)
+Copyright (c) 2012-2021: Timothy E. Holy, Simon Kornblith, and contributors: https://github.com/JuliaIO/HDF5.jl/contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software
+and associated documentation files (the "Software"), to deal in the Software without
+restriction, including without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or
+substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/src/filters/H5Zblosc/Project.toml b/src/filters/H5Zblosc/Project.toml
@@ -0,0 +1,12 @@
+name = "H5Zblosc"
+uuid = "c8ec2601-a99c-407f-b158-e79c03c2f5f7"
+version = "0.16.0"
+
+[deps]
+Blosc = "a74b3585-a348-5f62-a45c-50e91977d574"
+HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f"
+
+[compat]
+HDF5 = "0.16"
+Blosc = "0.7.1"
+julia = "1.3"
diff --git a/src/filters/blosc.jl → src/filters/H5Zblosc/src/H5Zblosc.jl b/src/filters/blosc.jl → src/filters/H5Zblosc/src/H5Zblosc.jl
@@ -1,8 +1,14 @@
+module H5Zblosc
 # port of https://github.com/Blosc/c-blosc/blob/3a668dcc9f61ad22b5c0a0ab45fe8dad387277fd/hdf5/blosc_filter.c (copyright 2010 Francesc Alted, license: MIT/expat)
 
 import Blosc
+using HDF5.API
+import HDF5.Filters: Filter, FilterPipeline, filterid, register_filter, filterid, filtername
+import HDF5.Filters: set_local_func, set_local_cfunc, filter_func, filter_cfunc, register_filter
 
-const FILTER_BLOSC = API.H5Z_filter_t(32001) # Filter ID registered with the HDF Group for Blosc
+export H5Z_FILTER_BLOSC, blosc_filter, BloscFilter
+
+const H5Z_FILTER_BLOSC = API.H5Z_filter_t(32001) # Filter ID registered with the HDF Group for Blosc
 const FILTER_BLOSC_VERSION = 2
 const blosc_name = "blosc"
 
@@ -12,7 +18,7 @@ function blosc_set_local(dcpl::API.hid_t, htype::API.hid_t, space::API.hid_t)
     blosc_nelements = Ref{Csize_t}(length(blosc_values))
     blosc_chunkdims = Vector{API.hsize_t}(undef,32)
 
-    API.h5p_get_filter_by_id(dcpl, FILTER_BLOSC, blosc_flags, blosc_nelements, blosc_values, 0, C_NULL, C_NULL)
+    API.h5p_get_filter_by_id(dcpl, H5Z_FILTER_BLOSC, blosc_flags, blosc_nelements, blosc_values, 0, C_NULL, C_NULL)
     flags = blosc_flags[]
 
     nelements = max(blosc_nelements[], 4) # First 4 slots reserved
@@ -45,7 +51,7 @@ function blosc_set_local(dcpl::API.hid_t, htype::API.hid_t, space::API.hid_t)
     blosc_values[3] = basetypesize
     blosc_values[4] = chunksize * htypesize # size of the chunk
 
-    API.h5p_modify_filter(dcpl, FILTER_BLOSC, flags, nelements, blosc_values)
+    API.h5p_modify_filter(dcpl, H5Z_FILTER_BLOSC, flags, nelements, blosc_values)
 
     return API.herr_t(1)
 end
@@ -85,10 +91,16 @@ function blosc_filter(flags::Cuint, cd_nelmts::Csize_t,
         # uncompressed chunk size but it should not be used in a general
         # cases since other filters in the pipeline can modify the buffer
         # size.
-        outbuf_size, cbytes, blocksize = Blosc.cbuffer_sizes(unsafe_load(buf))
-        outbuf = Libc.malloc(outbuf_size)
-        outbuf == C_NULL && return Csize_t(0)
-        status = Blosc.blosc_decompress(unsafe_load(buf), outbuf, outbuf_size)
+        in = unsafe_load(buf)
+        outbuf_size, cbytes, blocksize = Blosc.cbuffer_sizes(in)
+        # We need to preserve the output of cbuffer_sizes under Julia 1.8 (as of 2021/12/11)
+        # Otherwise their Refs get freed and the buffer becomes corrupted
+        # If this fails, consider copying `in` or wrapping that array.
+        GC.@preserve outbuf_size cbytes blocksize begin
+            outbuf = Libc.malloc(outbuf_size)
+            outbuf == C_NULL && return Csize_t(0)
+            status = Blosc.blosc_decompress(in, outbuf, outbuf_size)
+        end
         status <= 0 && (Libc.free(outbuf); return Csize_t(0))
     end
 
@@ -102,19 +114,6 @@ function blosc_filter(flags::Cuint, cd_nelmts::Csize_t,
     return Csize_t(0)
 end
 
-
-# register the Blosc filter function with HDF5
-function register_blosc()
-    c_blosc_set_local = @cfunction(blosc_set_local, API.herr_t, (API.hid_t,API.hid_t,API.hid_t))
-    c_blosc_filter = @cfunction(blosc_filter, Csize_t,
-                                (Cuint, Csize_t, Ptr{Cuint}, Csize_t,
-                                 Ptr{Csize_t}, Ptr{Ptr{Cvoid}}))
-    API.h5z_register(API.H5Z_class_t(API.H5Z_CLASS_T_VERS, FILTER_BLOSC, 1, 1, pointer(blosc_name), C_NULL, c_blosc_set_local, c_blosc_filter))
-
-    return nothing
-end
-
-
 """
     BloscFilter(;level=5, shuffle=true, compressor="blosclz")
 
@@ -142,6 +141,15 @@ function BloscFilter(;level=5, shuffle=true, compressor="blosclz")
     BloscFilter(0,0,0,0,level,shuffle,compcode)
 end
 
+filterid(::Type{BloscFilter}) = H5Z_FILTER_BLOSC
+filtername(::Type{BloscFilter}) = blosc_name
+set_local_func(::Type{BloscFilter}) = blosc_set_local
+set_local_cfunc(::Type{BloscFilter}) = @cfunction(blosc_set_local, API.herr_t, (API.hid_t,API.hid_t,API.hid_t))
+filter_func(::Type{BloscFilter}) = blosc_filter
+filter_cfunc(::Type{BloscFilter}) = @cfunction(blosc_filter, Csize_t,
+                                                 (Cuint, Csize_t, Ptr{Cuint}, Csize_t,
+                                                 Ptr{Csize_t}, Ptr{Ptr{Cvoid}}))
+
 function Base.show(io::IO, blosc::BloscFilter)
     print(io, BloscFilter,
           "(level=", Int(blosc.level),
@@ -150,9 +158,6 @@ function Base.show(io::IO, blosc::BloscFilter)
           ")")
 end
 
-filterid(::Type{BloscFilter}) = FILTER_BLOSC
-FILTERS[FILTER_BLOSC] = BloscFilter
-
 function Base.push!(f::FilterPipeline, blosc::BloscFilter)
     0 <= blosc.level <= 9 || throw(ArgumentError("blosc compression $(blosc.level) not in [0,9]"))
     ref = Ref(blosc)
@@ -161,3 +166,9 @@ function Base.push!(f::FilterPipeline, blosc::BloscFilter)
     end
     return f
 end
+
+function __init__()
+    register_filter(BloscFilter)
+end
+
+end # module H5Zblosc
diff --git a/src/filters/H5Zbzip2/LICENSE.txt b/src/filters/H5Zbzip2/LICENSE.txt
@@ -0,0 +1,17 @@
+The MIT License (MIT)
+Copyright (c) 2012-2021: Timothy E. Holy, Simon Kornblith, and contributors: https://github.com/JuliaIO/HDF5.jl/contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software
+and associated documentation files (the "Software"), to deal in the Software without
+restriction, including without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or
+substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/src/filters/H5Zbzip2/Project.toml b/src/filters/H5Zbzip2/Project.toml
@@ -0,0 +1,12 @@
+name = "H5Zbzip2"
+uuid = "094576f2-1e46-4c84-8e32-c46c042eaaa2"
+version = "0.16.0"
+
+[deps]
+CodecBzip2 = "523fee87-0ab8-5b00-afb7-3ecf72e48cfd"
+HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f"
+
+[compat]
+HDF5 = "0.16"
+CodecBzip2 = "0.7"
+julia = "1.3"
diff --git a/src/filters/H5Zbzip2/THIRDPARTY.txt b/src/filters/H5Zbzip2/THIRDPARTY.txt
@@ -0,0 +1,31 @@
+H5Z_filter_bzip2 in H5Zbzip2.jl was derived from H5Zbzip2.c from PyTables:
+
+Copyright Notice and Statement for PyTables Software Library and Utilities:
+Copyright (c) 2002-2004 by Francesc Alted
+Copyright (c) 2005-2007 by Carabos Coop. V.
+Copyright (c) 2008-2010 by Francesc Alted
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+a. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+b. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the
+   distribution.
+c. Neither the name of Francesc Alted nor the names of its
+   contributors may be used to endorse or promote products derived
+   from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+