Updated SZ build

doc/contribute.rst

@@ -153,23 +153,23 @@ compression_opts: (**block_size**,)
 sz
 ..
 
-compression_opts: (
-  **error_bound_mode** (int32),
-  **abs_error high** (big endian float64),
-  **abs_error low**,
-  **rel_error high** (big endian float64),
-  **rel_error low**,
-  **pw_rel_error high** (big endian float64),
-  **pw_rel_error low**,
-  **psnr high** (big endian float64),
-  **psnr low**,
-)
+compression_opts:
+
+- **error_bound_mode** (int32)
+- **abs_error high** (big endian float64)
+- **abs_error low**
+- **rel_error high** (big endian float64)
+- **rel_error low**
+- **pw_rel_error high** (big endian float64)
+- **pw_rel_error low**
+- **psnr high** (big endian float64)
+- **psnr low**
 
 The `set_local` function prepends:
 
 - For **dim size** from 2 to 5:
 
-  (**dim size**, **data type**, **r1**, **r2**, **r3** (if **dim size**>=3), **r4** (if **dim size**>=4), **r5** (if **dim size**==5))
+  (**dim size**, **data type**, **r1**, **r2**, **r3** (if **dim size** >= 3), **r4** (if **dim size** >= 4), **r5** (if **dim size** == 5))
 
   **rX** are set up to **dim size** (e.g., For **dim size** == 2 only **r1** and **r2** are used)
 

doc/information.rst

@@ -56,7 +56,7 @@ HDF5 compression filters and compression libraries sources were obtained from:
 * **FCIDECOMP plugin** (v1.0.2) and **CharLS** (1.x branch, commit `25160a4 <https://github.com/team-charls/charls/tree/25160a42fb62e71e4b0ce081f5cb3f8bb73938b5>`_):
   ftp://ftp.eumetsat.int/pub/OPS/out/test-data/Test-data-for-External-Users/MTG_FCI_Test-Data/FCI_Decompression_Software_V1.0.2 and
   https://github.com/team-charls/charls
-* **SZ plugin** (commit `f36afa4ae9 <https://github.com/szcompressor/SZ/commit/f36afa4ae9da0a8c50da3b9e89bfa89c8908b200>`_) and https://github.com/szcompressor/SZ
+* **SZ plugin** (commit `f36afa4ae9 <https://github.com/szcompressor/SZ/commit/f36afa4ae9da0a8c50da3b9e89bfa89c8908b200>`_), `SZ <https://github.com/szcompressor/SZ>`_, `zlib <https://github.com/Blosc/c-blosc/tree/9dc93b1de7c1ff6265d0ae554bd79077840849d8/internal-complibs/zlib-1.2.11>`_ (v1.2.11) and `zstd <https://github.com/Blosc/c-blosc/tree/9dc93b1de7c1ff6265d0ae554bd79077840849d8/internal-complibs/zstd-1.5.2>`_ (v1.5.2)
 * **HDF5-ZFP plugin** (commit `cd5422c <https://github.com/LLNL/H5Z-ZFP/tree/cd5422c146836e17c7a0380bfb05cf52d0c4467c>`_) and **zfp** (v1.0.0): https://github.com/LLNL/H5Z-ZFP and https://github.com/LLNL/zfp
 * **HDF5Plugin-Zstandard** (commit `d5afdb5 <https://github.com/aparamon/HDF5Plugin-Zstandard/tree/d5afdb5f04116d5c2d1a869dc9c7c0c72832b143>`_) and **zstd** (v1.5.2): https://github.com/aparamon/HDF5Plugin-Zstandard and https://github.com/Blosc/c-blosc/tree/9dc93b1de7c1ff6265d0ae554bd79077840849d8/internal-complibs/zstd-1.5.2
 

setup.py

@@ -420,52 +420,6 @@ def build_libraries(self, libraries):
             build_info['cflags'] = cflags
 
             updated_libraries.append((lib_name, build_info))
-            # FIXME: Is there a cleaner way of running the configure script for SZ?
-            if lib_name == "sz":
-                from pathlib import Path
-                configure_path = Path().cwd() / "src" / "SZ" / "configure"
-                if sys.platform.startswith("win"):
-                    # TODO: decide if providing a config.h or write it here
-                    with open(configure_path.parent / "config.h", 'w') as f:
-                        f.write("/* Define to 1 if you have the <sys/time.h> header file. */\n")
-                        f.write("/* #undef HAVE_SYS_TIME_H */\n")
-                        f.write("\n")
-                        f.write("/* Define to 1 if you have the <unistd.h> header file. */\n")
-                        f.write("/* #undef HAVE_UNISTD_H */\n")
-                        f.write("\n")
-                        f.write("/* Define to 1 if you have the `clock_gettime' function. */\n")
-                        f.write("/* #undef HAVE_CLOCK_GETTIME */\n")
-                        f.write("\n")
-                        f.write("/* Define to 1 if you have the `gettimeofday' function. */\n")
-                        # I think this one should be set ot 1, but cmake did not generate it
-                        f.write("/* #undef HAVE_GETTIMEOFDAY */\n")
-                        f.write("#define HAVE_GETTIMEOFDAY 1\n")
-                elif sys.platform.startswith("darwin"):
-                    # TODO: decide if providing a config.h or write it here
-                    with open(configure_path.parent / "config.h", 'w') as f:
-                        f.write("/* Define to 1 if you have the <sys/time.h> header file. */\n")
-                        f.write("/* #undef HAVE_SYS_TIME_H */\n")
-                        f.write("#define HAVE_SYS_TIME_H 1\n")
-                        f.write("\n")
-                        f.write("/* Define to 1 if you have the <unistd.h> header file. */\n")
-                        f.write("/* #undef HAVE_UNISTD_H */\n")
-                        f.write("#define HAVE_UNISTD_H 1\n")
-                        f.write("\n")
-                        f.write("/* Define to 1 if you have the `clock_gettime' function. */\n")
-                        f.write("/* #undef HAVE_CLOCK_GETTIME */\n")
-                        f.write("#define HAVE_CLOCK_GETTIME 1\n")
-                        f.write("\n")
-                        f.write("/* Define to 1 if you have the `gettimeofday' function. */\n")
-                        # I think this one should be set ot 1, but cmake did not generate it
-                        f.write("/* #undef HAVE_GETTIMEOFDAY */\n")
-                        f.write("#define HAVE_GETTIMEOFDAY 1\n")
-                else:
-                    # TODO: Use cmake or supplied file instead of configure
-                    # Execute configure script
-                    self.spawn([configure_path.as_posix()])
-                    # Move config.h from the current working directory to SZ directory.
-                    config_h_path = Path().cwd() / "config.h"
-                    config_h_path.rename(configure_path.parent / "config.h")
 
         super().build_libraries(updated_libraries)
 
@@ -651,9 +605,13 @@ def prefix(directory, files):
     }
 
 #zlib
-sources += glob(blosc_dir + 'internal-complibs/zlib*/*.c')
-depends += glob(blosc_dir + 'internal-complibs/zlib*/*.h')
-include_dirs += glob(blosc_dir + 'internal-complibs/zlib*')
+zlib_sources = glob(blosc_dir + 'internal-complibs/zlib*/*.c')
+zlib_depends = glob(blosc_dir + 'internal-complibs/zlib*/*.h')
+zlib_include_dirs = glob(blosc_dir + 'internal-complibs/zlib*')
+
+sources += zlib_sources
+depends += zlib_depends
+include_dirs += zlib_include_dirs
 define_macros.append(('HAVE_ZLIB', 1))
 
 # zstd
@@ -672,6 +630,7 @@ def prefix(directory, files):
 sources += zstd_sources
 depends += zstd_depends
 include_dirs += zstd_include_dirs
+define_macros += zstd_define_macros
 define_macros.append(('HAVE_ZSTD', 1))
 
 extra_compile_args = ['-std=gnu99']  # Needed to build manylinux1 wheels
@@ -688,7 +647,7 @@ def prefix(directory, files):
     depends=depends + \
         prefix(hdf5_blosc_dir, ['blosc_filter.h', 'blosc_plugin.h']),
     include_dirs=include_dirs + [hdf5_blosc_dir],
-    define_macros=define_macros + zstd_define_macros,
+    define_macros=define_macros,
     extra_compile_args=extra_compile_args,
     extra_link_args=extra_link_args,
     sse2=sse2_kwargs,
@@ -872,39 +831,26 @@ def prefix(directory, files):
 sz_dir = os.path.join("src", "SZ", "sz")
 sz_sources = glob(os.path.join(sz_dir, "src", "*.c"))
 sz_include_dirs = [os.path.join(sz_dir, "include"), sz_dir]
-sz_sources += glob('src/SZ/zstd/*/*.c')
-# TODO sz_depends += glob('src/SZ/zstd*/*/*.h')
-
-HDF5PLUGIN_ZLIB_FROM_BLOSC = True
-if HDF5PLUGIN_ZLIB_FROM_BLOSC:
-    sz_sources += glob(blosc_dir + 'internal-complibs/zlib*/*.c')
-    ##depends += glob(blosc_dir + 'internal-complibs/zlib*/*.h')
-else:
-    sz_sources += glob('src/SZ/zlib/*.c')
-
-sz_include_dirs += glob('src/SZ/zstd')
-sz_include_dirs += glob('src/SZ/zstd/common')
+sz_include_dirs += glob('src/SZ_extra/')
+sz_depends = glob('src/SZ/sz/include/*.h')
+sz_depends += glob('src/SZ/sz/*.h')
 
-if HDF5PLUGIN_ZLIB_FROM_BLOSC:
-    sz_include_dirs += glob(blosc_dir + 'internal-complibs/zlib*')
-else:
-    sz_include_dirs += glob('src/SZ/zlib')
-
-sz_include_dirs += glob('src/SZ/')
-sz_lib = ("sz", {
-    "sources": sz_sources,
-    "include_dirs": sz_include_dirs,
-    #"cflags": ["-lzstd"],
-})
-
-# "cflags": ['-DBUILD_HDF5_FILTER:BOOL=ON'],
+# zlib
+sz_sources += zlib_sources
+sz_include_dirs += zlib_include_dirs
+sz_depends += zlib_depends
 
+# zstd
+sz_sources += zstd_sources
+sz_include_dirs += zstd_include_dirs
+sz_depends += zstd_depends
 
 h5zsz_dir = os.path.join("src", "SZ", "hdf5-filter", "H5Z-SZ")
-sources = glob(h5zsz_dir + "/src/" + "*.c")
+sources = glob(h5zsz_dir + "/src/*.c")
 sources += sz_sources
-depends = glob(h5zsz_dir + "/include/" + "*.h")
-include_dirs = [os.path.join(sz_dir, 'include'), os.path.join(h5zsz_dir, 'include')]
+depends = glob(h5zsz_dir + "/include/*.h")
+depends += sz_depends
+include_dirs = [os.path.join(h5zsz_dir, 'include')]
 include_dirs += sz_include_dirs
 extra_compile_args = ['-O3', '-ffast-math', '-std=c99', '-fopenmp']
 extra_compile_args += ['/Ox', '/fp:fast', '/openmp']
@@ -917,8 +863,9 @@ def prefix(directory, files):
     include_dirs=include_dirs,
     extra_compile_args=extra_compile_args,
     extra_link_args=extra_link_args,
+    define_macros=zstd_define_macros,
+    extra_objects=zstd_extra_objects,
 )
-PLUGIN_LIB_DEPENDENCIES['sz'] = 'sz'
 
 
 def apply_filter_strip(libraries, extensions, dependencies):
@@ -949,7 +896,7 @@ def apply_filter_strip(libraries, extensions, dependencies):
     return libraries, extensions
 
 libraries, extensions = apply_filter_strip(
-    libraries=[snappy_lib, charls_lib, zfp_lib, sz_lib],
+    libraries=[snappy_lib, charls_lib, zfp_lib],
     extensions=[
         bzip2_plugin,
         lz4_plugin,

src/SZ_extra/config.h

@@ -0,0 +1,4 @@
+/* Static config.h file added for hdf5plugin to spare running the configuration */
+#if !defined(_WIN32) || defined(__MINGW32__)
+#define HAVE_UNISTD_H
+#endif