Add SZ

.github/workflows/ci.yml

@@ -24,12 +24,12 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - name-suffix: "(Minimum Versions)"
-            os: ubuntu-latest
-            python-version: 3.6
-            BUILD_COMMAND: sdist
-            H5PY_OLDER_VERSION: 2.6.0
-            NUMPY_OLDER_VERSION: 1.11.3
+      #- name-suffix: "(Minimum Versions)"
+      #os: ubuntu-latest
+      #      python-version: 3.6
+      #      BUILD_COMMAND: sdist
+      #      H5PY_OLDER_VERSION: 2.6.0
+      #      NUMPY_OLDER_VERSION: 1.11.3
 
           - name-suffix: "bdist_wheel-h5py_2.8.0"
             os: ubuntu-latest

README.rst

@@ -4,7 +4,7 @@ hdf5plugin
 .. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.7257761.svg
    :target: https://doi.org/10.5281/zenodo.7257761
 
-*hdf5plugin* provides `HDF5 compression filters <https://portal.hdfgroup.org/display/support/Registered+Filter+Plugins>`_ (namely: blosc, bitshuffle, bzip2, FCIDECOMP, lz4, ZFP, zstd) and makes them usable from `h5py <https://www.h5py.org>`_.
+*hdf5plugin* provides `HDF5 compression filters <https://portal.hdfgroup.org/display/support/Registered+Filter+Plugins>`_ (namely: blosc, bitshuffle, bzip2, FCIDECOMP, lz4, SZ, ZFP, zstd) and makes them usable from `h5py <https://www.h5py.org>`_.
 
 See `documentation <http://www.silx.org/doc/hdf5plugin/latest/>`_.
 

doc/information.rst

@@ -56,6 +56,7 @@ HDF5 compression filters and compression libraries sources were obtained from:
 * **FCIDECOMP plugin** (v1.0.2) and **CharLS** (1.x branch, commit `25160a4 <https://github.com/team-charls/charls/tree/25160a42fb62e71e4b0ce081f5cb3f8bb73938b5>`_):
   ftp://ftp.eumetsat.int/pub/OPS/out/test-data/Test-data-for-External-Users/MTG_FCI_Test-Data/FCI_Decompression_Software_V1.0.2 and
   https://github.com/team-charls/charls
+* **SZ plugin** (commit `f36afa4ae9 <https://github.com/szcompressor/SZ/commit/f36afa4ae9da0a8c50da3b9e89bfa89c8908b200>`_) and https://github.com/szcompressor/SZ
 * **HDF5-ZFP plugin** (commit `cd5422c <https://github.com/LLNL/H5Z-ZFP/tree/cd5422c146836e17c7a0380bfb05cf52d0c4467c>`_) and **zfp** (v1.0.0): https://github.com/LLNL/H5Z-ZFP and https://github.com/LLNL/zfp
 * **HDF5Plugin-Zstandard** (commit `d5afdb5 <https://github.com/aparamon/HDF5Plugin-Zstandard/tree/d5afdb5f04116d5c2d1a869dc9c7c0c72832b143>`_) and **zstd** (v1.5.2): https://github.com/aparamon/HDF5Plugin-Zstandard and https://github.com/Blosc/c-blosc/tree/9dc93b1de7c1ff6265d0ae554bd79077840849d8/internal-complibs/zstd-1.5.2
 
@@ -74,6 +75,7 @@ Please read the different licenses:
 * bzip2: See `src/PyTables/LICENSE.txt <https://github.com/silx-kit/hdf5plugin/blob/main/src/PyTables/LICENSE.txt>`_ and `src/bzip2/LICENSE <https://github.com/silx-kit/hdf5plugin/blob/main/src/bzip2/LICENSE>`_
 * lz4: See `src/LZ4/COPYING <https://github.com/silx-kit/hdf5plugin/blob/main/src/LZ4/COPYING>`_, `src/LZ4/LICENSE <https://github.com/silx-kit/hdf5plugin/blob/main/src/LZ4/LICENSE>`_ and `src/c-blosc/LICENSES/LZ4.txt <https://github.com/silx-kit/hdf5plugin/blob/main/src/c-blosc/LICENSES/LZ4.txt>`_
 * FCIDECOMP: See `src/fcidecomp/LICENSE <https://github.com/silx-kit/hdf5plugin/blob/main/src/fcidecomp/LICENSE.txt>`_ and `src/charls/src/License.txt  <https://github.com/silx-kit/hdf5plugin/blob/main/src/charls/src/License.txt>`_
+* SZ: See `src/SZ/copyright-and-BSD-license.txt <https://github.com/silx-kit/hdf5plugin/blob/main/src/SZ/copyright-and-BSD-license.txt>`_
 * zfp: See `src/H5Z-ZFP/LICENSE <https://github.com/silx-kit/hdf5plugin/blob/main/src/H5Z-ZFP/LICENSE>`_ and `src/zfp/LICENSE <https://github.com/silx-kit/hdf5plugin/blob/main/src/zfp/LICENSE>`_
 * zstd: See `src/HDF5Plugin-Zstandard/LICENSE <https://github.com/silx-kit/hdf5plugin/blob/main/src/HDF5Plugin-Zstandard/LICENSE>`_
 

setup.py

@@ -420,6 +420,52 @@ def build_libraries(self, libraries):
             build_info['cflags'] = cflags
 
             updated_libraries.append((lib_name, build_info))
+            # FIXME: Is there a cleaner way of running the configure script for SZ?
+            if lib_name == "sz":
+                from pathlib import Path
+                configure_path = Path().cwd() / "src" / "SZ" / "configure"
+                if sys.platform.startswith("win"):
+                    # TODO: decide if providing a config.h or write it here
+                    with open(configure_path.parent / "config.h", 'w') as f:
+                        f.write("/* Define to 1 if you have the <sys/time.h> header file. */\n")
+                        f.write("/* #undef HAVE_SYS_TIME_H */\n")
+                        f.write("\n")
+                        f.write("/* Define to 1 if you have the <unistd.h> header file. */\n")
+                        f.write("/* #undef HAVE_UNISTD_H */\n")
+                        f.write("\n")
+                        f.write("/* Define to 1 if you have the `clock_gettime' function. */\n")
+                        f.write("/* #undef HAVE_CLOCK_GETTIME */\n")
+                        f.write("\n")
+                        f.write("/* Define to 1 if you have the `gettimeofday' function. */\n")
+                        # I think this one should be set ot 1, but cmake did not generate it
+                        f.write("/* #undef HAVE_GETTIMEOFDAY */\n")
+                        f.write("#define HAVE_GETTIMEOFDAY 1\n")
+                elif sys.platform.startswith("darwin"):
+                    # TODO: decide if providing a config.h or write it here
+                    with open(configure_path.parent / "config.h", 'w') as f:
+                        f.write("/* Define to 1 if you have the <sys/time.h> header file. */\n")
+                        f.write("/* #undef HAVE_SYS_TIME_H */\n")
+                        f.write("#define HAVE_SYS_TIME_H 1\n")
+                        f.write("\n")
+                        f.write("/* Define to 1 if you have the <unistd.h> header file. */\n")
+                        f.write("/* #undef HAVE_UNISTD_H */\n")
+                        f.write("#define HAVE_UNISTD_H 1\n")
+                        f.write("\n")
+                        f.write("/* Define to 1 if you have the `clock_gettime' function. */\n")
+                        f.write("/* #undef HAVE_CLOCK_GETTIME */\n")
+                        f.write("#define HAVE_CLOCK_GETTIME 1\n")
+                        f.write("\n")
+                        f.write("/* Define to 1 if you have the `gettimeofday' function. */\n")
+                        # I think this one should be set ot 1, but cmake did not generate it
+                        f.write("/* #undef HAVE_GETTIMEOFDAY */\n")
+                        f.write("#define HAVE_GETTIMEOFDAY 1\n")
+                else:
+                    # TODO: Use cmake or supplied file instead of configure
+                    # Execute configure script
+                    self.spawn([configure_path.as_posix()])
+                    # Move config.h from the current working directory to SZ directory.
+                    config_h_path = Path().cwd() / "config.h"
+                    config_h_path.rename(configure_path.parent / "config.h")
 
         super().build_libraries(updated_libraries)
 
@@ -822,6 +868,58 @@ def prefix(directory, files):
     'cflags': ['-DBIT_STREAM_WORD_TYPE=uint8'],
     })
 
+# SZ library and its hdf5 filter
+sz_dir = os.path.join("src", "SZ", "sz")
+sz_sources = glob(os.path.join(sz_dir, "src", "*.c"))
+sz_include_dirs = [os.path.join(sz_dir, "include"), sz_dir]
+sz_sources += glob('src/SZ/zstd/*/*.c')
+# TODO sz_depends += glob('src/SZ/zstd*/*/*.h')
+
+HDF5PLUGIN_ZLIB_FROM_BLOSC = True
+if HDF5PLUGIN_ZLIB_FROM_BLOSC:
+    sz_sources += glob(blosc_dir + 'internal-complibs/zlib*/*.c')
+    ##depends += glob(blosc_dir + 'internal-complibs/zlib*/*.h')
+else:
+    sz_sources += glob('src/SZ/zlib/*.c')
+
+sz_include_dirs += glob('src/SZ/zstd')
+sz_include_dirs += glob('src/SZ/zstd/common')
+
+if HDF5PLUGIN_ZLIB_FROM_BLOSC:
+    sz_include_dirs += glob(blosc_dir + 'internal-complibs/zlib*')
+else:
+    sz_include_dirs += glob('src/SZ/zlib')
+
+sz_include_dirs += glob('src/SZ/')
+sz_lib = ("sz", {
+    "sources": sz_sources,
+    "include_dirs": sz_include_dirs,
+    #"cflags": ["-lzstd"],
+})
+
+# "cflags": ['-DBUILD_HDF5_FILTER:BOOL=ON'],
+
+
+h5zsz_dir = os.path.join("src", "SZ", "hdf5-filter", "H5Z-SZ")
+sources = glob(h5zsz_dir + "/src/" + "*.c")
+sources += sz_sources
+depends = glob(h5zsz_dir + "/include/" + "*.h")
+include_dirs = [os.path.join(sz_dir, 'include'), os.path.join(h5zsz_dir, 'include')]
+include_dirs += sz_include_dirs
+extra_compile_args = ['-O3', '-ffast-math', '-std=c99', '-fopenmp']
+extra_compile_args += ['/Ox', '/fp:fast', '/openmp']
+extra_link_args = ['-fopenmp', '/openmp', "-lm"]
+
+sz_plugin = HDF5PluginExtension(
+    "hdf5plugin.plugins.libh5sz",
+    sources=sources,
+    depends=depends,
+    include_dirs=include_dirs,
+    extra_compile_args=extra_compile_args,
+    extra_link_args=extra_link_args,
+)
+PLUGIN_LIB_DEPENDENCIES['sz'] = 'sz'
+
 
 def apply_filter_strip(libraries, extensions, dependencies):
     """Strip C libraries and extensions according to HDF5PLUGIN_STRIP env. var."""
@@ -851,7 +949,7 @@ def apply_filter_strip(libraries, extensions, dependencies):
     return libraries, extensions
 
 libraries, extensions = apply_filter_strip(
-    libraries=[snappy_lib, charls_lib, zfp_lib],
+    libraries=[snappy_lib, charls_lib, zfp_lib, sz_lib],
     extensions=[
         bzip2_plugin,
         lz4_plugin,
@@ -860,6 +958,7 @@ def apply_filter_strip(libraries, extensions, dependencies):
         fcidecomp_plugin,
         h5zfp_plugin,
         zstandard_plugin,
+        sz_plugin,
     ],
     dependencies=PLUGIN_LIB_DEPENDENCIES,
 )

src/SZ/.dockerignore

@@ -0,0 +1 @@
+build.*

src/SZ/.e4s/e4s.yaml

@@ -0,0 +1,6 @@
+- e4s_product: SZ
+  version: 1.2.12
+  type: Compression
+  website: https://szcompressor.org
+  docs: [README.md]
+  subrepo_urls: ["https://github.com/szcompressor/SZ", "https://github.com/szcompressor/SZ3", "https://github.com/szcompressor/SZx", "https://github.com/szcompressor/cuSZ", "https://github.com/szcompressor/SZauto", "https://github.com/szcompressor/MMD-SZ"]

src/SZ/.gitignore

@@ -0,0 +1,6 @@
+build
+compile_commands.json
+tags
+CMakeCache.txt
+cmake-build-debug/
+CMakeFiles/

src/SZ/.travis.yml

@@ -0,0 +1,45 @@
+sudo: false
+
+language: c
+
+before_install:
+- cd test/travis-ci && ./getData.sh && cd -
+
+matrix:
+  include:
+    - dist: xenial
+      os: linux
+      addons:
+        apt:
+          sources:
+            - ubuntu-toolchain-r-test     # For gcc 4.9, 5 and 7
+          packages:
+            - gcc-7
+            - gfortran-7
+            - zstd
+            - libzstd1-dev
+            - exuberant-ctags
+            - libcunit1-dev 
+            - libnetcdf-dev 
+    - osx_image: xcode11
+      os: osx
+      env: PATH=/usr/local/bin:$PATH
+install:
+    - mkdir build
+    - cd build
+    - |
+      if [[ "${TRAVIS_OS_NAME}" != "linux" ]]; then
+        brew install ctags
+        brew install cunit
+        brew upgrade pkg-config
+      fi
+    - cmake -DCMAKE_INSTALL_PREFIX=$HOME -DBUILD_TESTS=ON -DBUILD_INTEGRATION_TESTS=ON ..
+    - make 
+    - make install
+    - make test
+
+script:
+- cd ..
+- ./configure && make
+- cd example && ./test.sh && cd -
+- cd test/travis-ci && ./test.sh && cd -