From 4e05f701e58729315fb23f1c196ae38ef088274b Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Fri, 25 Nov 2022 14:48:41 +0100 Subject: [PATCH 1/4] Update SZ build: static config.h, use zlib and zstd from c-blosc --- setup.py | 105 ++++++++++++------------------------------------ src/SZ/config.h | 4 ++ 2 files changed, 30 insertions(+), 79 deletions(-) create mode 100644 src/SZ/config.h diff --git a/setup.py b/setup.py index 72de49df..34bebcd1 100644 --- a/setup.py +++ b/setup.py @@ -420,52 +420,6 @@ def build_libraries(self, libraries): build_info['cflags'] = cflags updated_libraries.append((lib_name, build_info)) - # FIXME: Is there a cleaner way of running the configure script for SZ? - if lib_name == "sz": - from pathlib import Path - configure_path = Path().cwd() / "src" / "SZ" / "configure" - if sys.platform.startswith("win"): - # TODO: decide if providing a config.h or write it here - with open(configure_path.parent / "config.h", 'w') as f: - f.write("/* Define to 1 if you have the header file. */\n") - f.write("/* #undef HAVE_SYS_TIME_H */\n") - f.write("\n") - f.write("/* Define to 1 if you have the header file. */\n") - f.write("/* #undef HAVE_UNISTD_H */\n") - f.write("\n") - f.write("/* Define to 1 if you have the `clock_gettime' function. */\n") - f.write("/* #undef HAVE_CLOCK_GETTIME */\n") - f.write("\n") - f.write("/* Define to 1 if you have the `gettimeofday' function. */\n") - # I think this one should be set ot 1, but cmake did not generate it - f.write("/* #undef HAVE_GETTIMEOFDAY */\n") - f.write("#define HAVE_GETTIMEOFDAY 1\n") - elif sys.platform.startswith("darwin"): - # TODO: decide if providing a config.h or write it here - with open(configure_path.parent / "config.h", 'w') as f: - f.write("/* Define to 1 if you have the header file. */\n") - f.write("/* #undef HAVE_SYS_TIME_H */\n") - f.write("#define HAVE_SYS_TIME_H 1\n") - f.write("\n") - f.write("/* Define to 1 if you have the header file. */\n") - f.write("/* #undef HAVE_UNISTD_H */\n") - f.write("#define HAVE_UNISTD_H 1\n") - f.write("\n") - f.write("/* Define to 1 if you have the `clock_gettime' function. */\n") - f.write("/* #undef HAVE_CLOCK_GETTIME */\n") - f.write("#define HAVE_CLOCK_GETTIME 1\n") - f.write("\n") - f.write("/* Define to 1 if you have the `gettimeofday' function. */\n") - # I think this one should be set ot 1, but cmake did not generate it - f.write("/* #undef HAVE_GETTIMEOFDAY */\n") - f.write("#define HAVE_GETTIMEOFDAY 1\n") - else: - # TODO: Use cmake or supplied file instead of configure - # Execute configure script - self.spawn([configure_path.as_posix()]) - # Move config.h from the current working directory to SZ directory. - config_h_path = Path().cwd() / "config.h" - config_h_path.rename(configure_path.parent / "config.h") super().build_libraries(updated_libraries) @@ -651,9 +605,13 @@ def prefix(directory, files): } #zlib -sources += glob(blosc_dir + 'internal-complibs/zlib*/*.c') -depends += glob(blosc_dir + 'internal-complibs/zlib*/*.h') -include_dirs += glob(blosc_dir + 'internal-complibs/zlib*') +zlib_sources = glob(blosc_dir + 'internal-complibs/zlib*/*.c') +zlib_depends = glob(blosc_dir + 'internal-complibs/zlib*/*.h') +zlib_include_dirs = glob(blosc_dir + 'internal-complibs/zlib*') + +sources += zlib_sources +depends += zlib_depends +include_dirs += zlib_include_dirs define_macros.append(('HAVE_ZLIB', 1)) # zstd @@ -672,6 +630,7 @@ def prefix(directory, files): sources += zstd_sources depends += zstd_depends include_dirs += zstd_include_dirs +define_macros += zstd_define_macros define_macros.append(('HAVE_ZSTD', 1)) extra_compile_args = ['-std=gnu99'] # Needed to build manylinux1 wheels @@ -688,7 +647,7 @@ def prefix(directory, files): depends=depends + \ prefix(hdf5_blosc_dir, ['blosc_filter.h', 'blosc_plugin.h']), include_dirs=include_dirs + [hdf5_blosc_dir], - define_macros=define_macros + zstd_define_macros, + define_macros=define_macros, extra_compile_args=extra_compile_args, extra_link_args=extra_link_args, sse2=sse2_kwargs, @@ -872,39 +831,26 @@ def prefix(directory, files): sz_dir = os.path.join("src", "SZ", "sz") sz_sources = glob(os.path.join(sz_dir, "src", "*.c")) sz_include_dirs = [os.path.join(sz_dir, "include"), sz_dir] -sz_sources += glob('src/SZ/zstd/*/*.c') -# TODO sz_depends += glob('src/SZ/zstd*/*/*.h') - -HDF5PLUGIN_ZLIB_FROM_BLOSC = True -if HDF5PLUGIN_ZLIB_FROM_BLOSC: - sz_sources += glob(blosc_dir + 'internal-complibs/zlib*/*.c') - ##depends += glob(blosc_dir + 'internal-complibs/zlib*/*.h') -else: - sz_sources += glob('src/SZ/zlib/*.c') - -sz_include_dirs += glob('src/SZ/zstd') -sz_include_dirs += glob('src/SZ/zstd/common') - -if HDF5PLUGIN_ZLIB_FROM_BLOSC: - sz_include_dirs += glob(blosc_dir + 'internal-complibs/zlib*') -else: - sz_include_dirs += glob('src/SZ/zlib') - sz_include_dirs += glob('src/SZ/') -sz_lib = ("sz", { - "sources": sz_sources, - "include_dirs": sz_include_dirs, - #"cflags": ["-lzstd"], -}) +sz_depends = glob('src/SZ/sz/include/*.h') +sz_depends += glob('src/SZ/sz/*.h') -# "cflags": ['-DBUILD_HDF5_FILTER:BOOL=ON'], +# zlib +sz_sources += zlib_sources +sz_include_dirs += zlib_include_dirs +sz_depends += zlib_depends +# zstd +sz_sources += zstd_sources +sz_include_dirs += zstd_include_dirs +sz_depends += zstd_depends h5zsz_dir = os.path.join("src", "SZ", "hdf5-filter", "H5Z-SZ") -sources = glob(h5zsz_dir + "/src/" + "*.c") +sources = glob(h5zsz_dir + "/src/*.c") sources += sz_sources -depends = glob(h5zsz_dir + "/include/" + "*.h") -include_dirs = [os.path.join(sz_dir, 'include'), os.path.join(h5zsz_dir, 'include')] +depends = glob(h5zsz_dir + "/include/*.h") +depends += sz_depends +include_dirs = [os.path.join(h5zsz_dir, 'include')] include_dirs += sz_include_dirs extra_compile_args = ['-O3', '-ffast-math', '-std=c99', '-fopenmp'] extra_compile_args += ['/Ox', '/fp:fast', '/openmp'] @@ -917,8 +863,9 @@ def prefix(directory, files): include_dirs=include_dirs, extra_compile_args=extra_compile_args, extra_link_args=extra_link_args, + define_macros=zstd_define_macros, + extra_objects=zstd_extra_objects, ) -PLUGIN_LIB_DEPENDENCIES['sz'] = 'sz' def apply_filter_strip(libraries, extensions, dependencies): @@ -949,7 +896,7 @@ def apply_filter_strip(libraries, extensions, dependencies): return libraries, extensions libraries, extensions = apply_filter_strip( - libraries=[snappy_lib, charls_lib, zfp_lib, sz_lib], + libraries=[snappy_lib, charls_lib, zfp_lib], extensions=[ bzip2_plugin, lz4_plugin, diff --git a/src/SZ/config.h b/src/SZ/config.h new file mode 100644 index 00000000..5385b171 --- /dev/null +++ b/src/SZ/config.h @@ -0,0 +1,4 @@ +/* Static config.h file added for hdf5plugin to spare running the configuration */ +#if !defined(_WIN32) || defined(__MINGW32__) +#define HAVE_UNISTD_H +#endif From dff55fdbb339bf77d8604af2ef26751c166149a5 Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Fri, 25 Nov 2022 15:18:54 +0100 Subject: [PATCH 2/4] Update doc of used versions --- doc/information.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/information.rst b/doc/information.rst index 229fd99c..96bf8242 100644 --- a/doc/information.rst +++ b/doc/information.rst @@ -56,7 +56,7 @@ HDF5 compression filters and compression libraries sources were obtained from: * **FCIDECOMP plugin** (v1.0.2) and **CharLS** (1.x branch, commit `25160a4 `_): ftp://ftp.eumetsat.int/pub/OPS/out/test-data/Test-data-for-External-Users/MTG_FCI_Test-Data/FCI_Decompression_Software_V1.0.2 and https://github.com/team-charls/charls -* **SZ plugin** (commit `f36afa4ae9 `_) and https://github.com/szcompressor/SZ +* **SZ plugin** (commit `f36afa4ae9 `_), `SZ `_, `zlib `_ (v1.2.11) and `zstd `_ (v1.5.2) * **HDF5-ZFP plugin** (commit `cd5422c `_) and **zfp** (v1.0.0): https://github.com/LLNL/H5Z-ZFP and https://github.com/LLNL/zfp * **HDF5Plugin-Zstandard** (commit `d5afdb5 `_) and **zstd** (v1.5.2): https://github.com/aparamon/HDF5Plugin-Zstandard and https://github.com/Blosc/c-blosc/tree/9dc93b1de7c1ff6265d0ae554bd79077840849d8/internal-complibs/zstd-1.5.2 From 8d0746ade58e575ccb11e52438be912352c400da Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Fri, 25 Nov 2022 15:19:05 +0100 Subject: [PATCH 3/4] Fix doc build wrnings --- doc/contribute.rst | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/doc/contribute.rst b/doc/contribute.rst index 9e8eaa7c..2bfc56f3 100644 --- a/doc/contribute.rst +++ b/doc/contribute.rst @@ -153,23 +153,23 @@ compression_opts: (**block_size**,) sz .. -compression_opts: ( - **error_bound_mode** (int32), - **abs_error high** (big endian float64), - **abs_error low**, - **rel_error high** (big endian float64), - **rel_error low**, - **pw_rel_error high** (big endian float64), - **pw_rel_error low**, - **psnr high** (big endian float64), - **psnr low**, -) +compression_opts: + +- **error_bound_mode** (int32) +- **abs_error high** (big endian float64) +- **abs_error low** +- **rel_error high** (big endian float64) +- **rel_error low** +- **pw_rel_error high** (big endian float64) +- **pw_rel_error low** +- **psnr high** (big endian float64) +- **psnr low** The `set_local` function prepends: - For **dim size** from 2 to 5: - (**dim size**, **data type**, **r1**, **r2**, **r3** (if **dim size**>=3), **r4** (if **dim size**>=4), **r5** (if **dim size**==5)) + (**dim size**, **data type**, **r1**, **r2**, **r3** (if **dim size** >= 3), **r4** (if **dim size** >= 4), **r5** (if **dim size** == 5)) **rX** are set up to **dim size** (e.g., For **dim size** == 2 only **r1** and **r2** are used) From 78c00b84cb2b91fbe29f2891530081b66492885c Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Fri, 25 Nov 2022 15:58:34 +0100 Subject: [PATCH 4/4] Move config.h to SZ_extra --- setup.py | 2 +- src/{SZ => SZ_extra}/config.h | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename src/{SZ => SZ_extra}/config.h (100%) diff --git a/setup.py b/setup.py index 34bebcd1..7c6bd9bb 100644 --- a/setup.py +++ b/setup.py @@ -831,7 +831,7 @@ def prefix(directory, files): sz_dir = os.path.join("src", "SZ", "sz") sz_sources = glob(os.path.join(sz_dir, "src", "*.c")) sz_include_dirs = [os.path.join(sz_dir, "include"), sz_dir] -sz_include_dirs += glob('src/SZ/') +sz_include_dirs += glob('src/SZ_extra/') sz_depends = glob('src/SZ/sz/include/*.h') sz_depends += glob('src/SZ/sz/*.h') diff --git a/src/SZ/config.h b/src/SZ_extra/config.h similarity index 100% rename from src/SZ/config.h rename to src/SZ_extra/config.h