diff --git a/doc/contribute.rst b/doc/contribute.rst index 9e8eaa7c..2bfc56f3 100644 --- a/doc/contribute.rst +++ b/doc/contribute.rst @@ -153,23 +153,23 @@ compression_opts: (**block_size**,) sz .. -compression_opts: ( - **error_bound_mode** (int32), - **abs_error high** (big endian float64), - **abs_error low**, - **rel_error high** (big endian float64), - **rel_error low**, - **pw_rel_error high** (big endian float64), - **pw_rel_error low**, - **psnr high** (big endian float64), - **psnr low**, -) +compression_opts: + +- **error_bound_mode** (int32) +- **abs_error high** (big endian float64) +- **abs_error low** +- **rel_error high** (big endian float64) +- **rel_error low** +- **pw_rel_error high** (big endian float64) +- **pw_rel_error low** +- **psnr high** (big endian float64) +- **psnr low** The `set_local` function prepends: - For **dim size** from 2 to 5: - (**dim size**, **data type**, **r1**, **r2**, **r3** (if **dim size**>=3), **r4** (if **dim size**>=4), **r5** (if **dim size**==5)) + (**dim size**, **data type**, **r1**, **r2**, **r3** (if **dim size** >= 3), **r4** (if **dim size** >= 4), **r5** (if **dim size** == 5)) **rX** are set up to **dim size** (e.g., For **dim size** == 2 only **r1** and **r2** are used) diff --git a/doc/information.rst b/doc/information.rst index 229fd99c..96bf8242 100644 --- a/doc/information.rst +++ b/doc/information.rst @@ -56,7 +56,7 @@ HDF5 compression filters and compression libraries sources were obtained from: * **FCIDECOMP plugin** (v1.0.2) and **CharLS** (1.x branch, commit `25160a4 `_): ftp://ftp.eumetsat.int/pub/OPS/out/test-data/Test-data-for-External-Users/MTG_FCI_Test-Data/FCI_Decompression_Software_V1.0.2 and https://github.com/team-charls/charls -* **SZ plugin** (commit `f36afa4ae9 `_) and https://github.com/szcompressor/SZ +* **SZ plugin** (commit `f36afa4ae9 `_), `SZ `_, `zlib `_ (v1.2.11) and `zstd `_ (v1.5.2) * **HDF5-ZFP plugin** (commit `cd5422c `_) and **zfp** (v1.0.0): https://github.com/LLNL/H5Z-ZFP and https://github.com/LLNL/zfp * **HDF5Plugin-Zstandard** (commit `d5afdb5 `_) and **zstd** (v1.5.2): https://github.com/aparamon/HDF5Plugin-Zstandard and https://github.com/Blosc/c-blosc/tree/9dc93b1de7c1ff6265d0ae554bd79077840849d8/internal-complibs/zstd-1.5.2 diff --git a/setup.py b/setup.py index 72de49df..7c6bd9bb 100644 --- a/setup.py +++ b/setup.py @@ -420,52 +420,6 @@ def build_libraries(self, libraries): build_info['cflags'] = cflags updated_libraries.append((lib_name, build_info)) - # FIXME: Is there a cleaner way of running the configure script for SZ? - if lib_name == "sz": - from pathlib import Path - configure_path = Path().cwd() / "src" / "SZ" / "configure" - if sys.platform.startswith("win"): - # TODO: decide if providing a config.h or write it here - with open(configure_path.parent / "config.h", 'w') as f: - f.write("/* Define to 1 if you have the header file. */\n") - f.write("/* #undef HAVE_SYS_TIME_H */\n") - f.write("\n") - f.write("/* Define to 1 if you have the header file. */\n") - f.write("/* #undef HAVE_UNISTD_H */\n") - f.write("\n") - f.write("/* Define to 1 if you have the `clock_gettime' function. */\n") - f.write("/* #undef HAVE_CLOCK_GETTIME */\n") - f.write("\n") - f.write("/* Define to 1 if you have the `gettimeofday' function. */\n") - # I think this one should be set ot 1, but cmake did not generate it - f.write("/* #undef HAVE_GETTIMEOFDAY */\n") - f.write("#define HAVE_GETTIMEOFDAY 1\n") - elif sys.platform.startswith("darwin"): - # TODO: decide if providing a config.h or write it here - with open(configure_path.parent / "config.h", 'w') as f: - f.write("/* Define to 1 if you have the header file. */\n") - f.write("/* #undef HAVE_SYS_TIME_H */\n") - f.write("#define HAVE_SYS_TIME_H 1\n") - f.write("\n") - f.write("/* Define to 1 if you have the header file. */\n") - f.write("/* #undef HAVE_UNISTD_H */\n") - f.write("#define HAVE_UNISTD_H 1\n") - f.write("\n") - f.write("/* Define to 1 if you have the `clock_gettime' function. */\n") - f.write("/* #undef HAVE_CLOCK_GETTIME */\n") - f.write("#define HAVE_CLOCK_GETTIME 1\n") - f.write("\n") - f.write("/* Define to 1 if you have the `gettimeofday' function. */\n") - # I think this one should be set ot 1, but cmake did not generate it - f.write("/* #undef HAVE_GETTIMEOFDAY */\n") - f.write("#define HAVE_GETTIMEOFDAY 1\n") - else: - # TODO: Use cmake or supplied file instead of configure - # Execute configure script - self.spawn([configure_path.as_posix()]) - # Move config.h from the current working directory to SZ directory. - config_h_path = Path().cwd() / "config.h" - config_h_path.rename(configure_path.parent / "config.h") super().build_libraries(updated_libraries) @@ -651,9 +605,13 @@ def prefix(directory, files): } #zlib -sources += glob(blosc_dir + 'internal-complibs/zlib*/*.c') -depends += glob(blosc_dir + 'internal-complibs/zlib*/*.h') -include_dirs += glob(blosc_dir + 'internal-complibs/zlib*') +zlib_sources = glob(blosc_dir + 'internal-complibs/zlib*/*.c') +zlib_depends = glob(blosc_dir + 'internal-complibs/zlib*/*.h') +zlib_include_dirs = glob(blosc_dir + 'internal-complibs/zlib*') + +sources += zlib_sources +depends += zlib_depends +include_dirs += zlib_include_dirs define_macros.append(('HAVE_ZLIB', 1)) # zstd @@ -672,6 +630,7 @@ def prefix(directory, files): sources += zstd_sources depends += zstd_depends include_dirs += zstd_include_dirs +define_macros += zstd_define_macros define_macros.append(('HAVE_ZSTD', 1)) extra_compile_args = ['-std=gnu99'] # Needed to build manylinux1 wheels @@ -688,7 +647,7 @@ def prefix(directory, files): depends=depends + \ prefix(hdf5_blosc_dir, ['blosc_filter.h', 'blosc_plugin.h']), include_dirs=include_dirs + [hdf5_blosc_dir], - define_macros=define_macros + zstd_define_macros, + define_macros=define_macros, extra_compile_args=extra_compile_args, extra_link_args=extra_link_args, sse2=sse2_kwargs, @@ -872,39 +831,26 @@ def prefix(directory, files): sz_dir = os.path.join("src", "SZ", "sz") sz_sources = glob(os.path.join(sz_dir, "src", "*.c")) sz_include_dirs = [os.path.join(sz_dir, "include"), sz_dir] -sz_sources += glob('src/SZ/zstd/*/*.c') -# TODO sz_depends += glob('src/SZ/zstd*/*/*.h') - -HDF5PLUGIN_ZLIB_FROM_BLOSC = True -if HDF5PLUGIN_ZLIB_FROM_BLOSC: - sz_sources += glob(blosc_dir + 'internal-complibs/zlib*/*.c') - ##depends += glob(blosc_dir + 'internal-complibs/zlib*/*.h') -else: - sz_sources += glob('src/SZ/zlib/*.c') - -sz_include_dirs += glob('src/SZ/zstd') -sz_include_dirs += glob('src/SZ/zstd/common') +sz_include_dirs += glob('src/SZ_extra/') +sz_depends = glob('src/SZ/sz/include/*.h') +sz_depends += glob('src/SZ/sz/*.h') -if HDF5PLUGIN_ZLIB_FROM_BLOSC: - sz_include_dirs += glob(blosc_dir + 'internal-complibs/zlib*') -else: - sz_include_dirs += glob('src/SZ/zlib') - -sz_include_dirs += glob('src/SZ/') -sz_lib = ("sz", { - "sources": sz_sources, - "include_dirs": sz_include_dirs, - #"cflags": ["-lzstd"], -}) - -# "cflags": ['-DBUILD_HDF5_FILTER:BOOL=ON'], +# zlib +sz_sources += zlib_sources +sz_include_dirs += zlib_include_dirs +sz_depends += zlib_depends +# zstd +sz_sources += zstd_sources +sz_include_dirs += zstd_include_dirs +sz_depends += zstd_depends h5zsz_dir = os.path.join("src", "SZ", "hdf5-filter", "H5Z-SZ") -sources = glob(h5zsz_dir + "/src/" + "*.c") +sources = glob(h5zsz_dir + "/src/*.c") sources += sz_sources -depends = glob(h5zsz_dir + "/include/" + "*.h") -include_dirs = [os.path.join(sz_dir, 'include'), os.path.join(h5zsz_dir, 'include')] +depends = glob(h5zsz_dir + "/include/*.h") +depends += sz_depends +include_dirs = [os.path.join(h5zsz_dir, 'include')] include_dirs += sz_include_dirs extra_compile_args = ['-O3', '-ffast-math', '-std=c99', '-fopenmp'] extra_compile_args += ['/Ox', '/fp:fast', '/openmp'] @@ -917,8 +863,9 @@ def prefix(directory, files): include_dirs=include_dirs, extra_compile_args=extra_compile_args, extra_link_args=extra_link_args, + define_macros=zstd_define_macros, + extra_objects=zstd_extra_objects, ) -PLUGIN_LIB_DEPENDENCIES['sz'] = 'sz' def apply_filter_strip(libraries, extensions, dependencies): @@ -949,7 +896,7 @@ def apply_filter_strip(libraries, extensions, dependencies): return libraries, extensions libraries, extensions = apply_filter_strip( - libraries=[snappy_lib, charls_lib, zfp_lib, sz_lib], + libraries=[snappy_lib, charls_lib, zfp_lib], extensions=[ bzip2_plugin, lz4_plugin, diff --git a/src/SZ_extra/config.h b/src/SZ_extra/config.h new file mode 100644 index 00000000..5385b171 --- /dev/null +++ b/src/SZ_extra/config.h @@ -0,0 +1,4 @@ +/* Static config.h file added for hdf5plugin to spare running the configuration */ +#if !defined(_WIN32) || defined(__MINGW32__) +#define HAVE_UNISTD_H +#endif