Provides ZFP lossy compression support

.travis.yml

@@ -27,15 +27,14 @@ matrix:
           os: osx
           env:
               BUILD_COMMAND=bdist_wheel
-              H5PY_OLDER_VERSION=2.6.0
+              H5PY_OLDER_VERSION=2.8.0
 
 cache:
     apt: true
 
 install:
     # Mac OS X specific bootstrap
     - source ./ci/travis_osx.sh
-    - travis_osx_install_begin
 
     # Upgrade distribution modules
     - pip install --upgrade setuptools
@@ -53,13 +52,7 @@ install:
     - python setup.py $BUILD_COMMAND
     - ls dist
 
-    # Mac OS X specific cleanup
-    - travis_osx_install_end
-
 script:
-    # Mac OS X specific bootstrap
-    - travis_osx_run_begin
-
     # Upgrade distribution modules
     - pip install --upgrade setuptools
     - python -m pip install --upgrade pip
@@ -84,6 +77,3 @@ script:
     - pip install --upgrade h5py==$H5PY_OLDER_VERSION
     - pip list
     - python test/test.py
-
-    # Mac OS X specific cleanup
-    - travis_osx_run_end

README.rst

@@ -16,6 +16,7 @@ The HDF5 plugin sources were obtained from:
 * bitshuffle plugin (0.3.5): https://github.com/kiyo-masui/bitshuffle
 * hdf5-blosc plugin (v1.0.0) and c-blosc (v1.17.0): https://github.com/Blosc/hdf5-blosc and https://github.com/Blosc/c-blosc
 * FICDECOMP plugin (v1.0.2) and CharLS (branch 1.x-master SHA1 ID:25160a42fb62e71e4b0ce081f5cb3f8bb73938b5): ftp://ftp.eumetsat.int/pub/OPS/out/test-data/Test-data-for-External-Users/MTG_FCI_Test-Data/FCI_Decompression_Software_V1.0.2/ and https://github.com/team-charls/charls.git 
+* HDF5-ZFP plugin (v1.0.1) and zfp (v0.5.5): https://github.com/LLNL/H5Z-ZFP and https://github.com/LLNL/zfp
 
 Installation
 ------------
@@ -63,18 +64,22 @@ Sample code:
 
   - `Bitshuffle(nelems=0, lz4=True)`_
   - `Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE)`_
+  - `FciDecomp()`_
   - `LZ4(nbytes=0)`_
+  - `Zfp()`_
+
 
 * The HDF5 filter ID of embedded plugins:
 
   - ``BLOSC_ID``
   - ``BSHUF_ID``
+  - ``FCIDECOMP_ID``
   - ``LZ4_ID``
+  - ``ZFP_ID``
 
 * ``FILTERS``: A dictionary mapping provided filters to their ID
 * ``PLUGINS_PATH``: The directory where the provided filters library are stored.
 
-
 Bitshuffle(nelems=0, lz4=True)
 ******************************
 
@@ -94,7 +99,6 @@ Sample code:
 	      **hdf5plugin.Bitshuffle(nelems=0, lz4=True))
         f.close()
 
-
 Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE)
 *********************************************
 
@@ -127,6 +131,21 @@ Sample code:
             **hdf5plugin.Blosc(cname='blosclz', clevel=9, shuffle=hdf5plugin.Blosc.SHUFFLE))
         f.close()
 
+FciDecomp()
+***********
+
+This class returns the compression options to feed into ``h5py.Group.create_dataset`` for using the FciDecomp filter:
+
+It can be passed as keyword arguments.
+
+Sample code:
+
+.. code-block:: python
+
+        f = h5py.File('test.h5', 'w')
+        f.create_dataset('fcidecomp', data=numpy.arange(100),
+            **hdf5plugin.FciDecomp())
+        f.close()
 
 LZ4(nbytes=0)
 *************
@@ -147,11 +166,10 @@ Sample code:
             **hdf5plugin.LZ4(nbytes=0))
         f.close()
 
+Zfp()
+*****
 
-FciDecomp()
-***********
-
-This class returns the compression options to feed into ``h5py.Group.create_dataset`` for using the FciDecomp filter:
+This class returns the compression options to feed into ``h5py.Group.create_dataset`` for using the zfp filter:
 
 It can be passed as keyword arguments.
 
@@ -160,17 +178,15 @@ Sample code:
 .. code-block:: python
 
         f = h5py.File('test.h5', 'w')
-        f.create_dataset('fcidecomp', data=numpy.arange(100),
-            **hdf5plugin.FciDecomp())
+        f.create_dataset('zfp', data=numpy.random.random(100),
+            **hdf5plugin.Zfp())
         f.close()
 
-
 Dependencies
 ------------
 
 * `h5py <https://www.h5py.org>`_
 
-
 Testing
 -------
 
@@ -205,5 +221,6 @@ Please read the different licenses:
 * blosc: See `src/hdf5-blosc/LICENSES/ <https://github.com/silx-kit/hdf5plugin/blob/master/src/hdf5-blosc/LICENSES/>`_ and `src/c-blosc/LICENSES/ <https://github.com/silx-kit/hdf5plugin/blob/master/src/c-blosc/LICENSES/>`_
 * lz4: See `src/LZ4/COPYING  <https://github.com/silx-kit/hdf5plugin/blob/master/src/LZ4/COPYING>`_
 * FCIDECOMP: See `src/fcidecomp/LICENSE <https://github.com/silx-kit/hdf5plugin/blob/master/src/fcidecomp/LICENSE.txt>`_ and `src/charls/src/License.txt  <https://github.com/silx-kit/hdf5plugin/blob/master/src/charls/License.txt>`_
+* zfp: See `src/H5Z-ZFP/LICENSE`_ and `src/zfp/LICENSE`_
 
 The HDF5 v1.10.5 headers (and Windows .lib file) used to build the filters are stored for convenience in the repository. The license is available here: `src/hdf5/COPYING <https://github.com/silx-kit/hdf5plugin/blob/master/src/hdf5/COPYING>`_.

ci/travis_osx.sh

@@ -1,85 +1,27 @@
-#!/bin/bash
 # Script for travis-CI Mac OS X specific setup.
-#
-# It provides 4 functions:
-# - travis_osx_install_begin: Install pip and setup a virtualenv for the build
-# - travis_osx_install_end: Deactivate the build virtualenv
-# - travis_osx_run_begin: Setup a virtualenv for the tests
-# - travis_osx_run_end: Deactivate test virtualenv
-#
-# On Linux, those functions do nothing.
+# source this script with PYTHON_VERSION env variable set
 
-# Directory where to create build virtualenv
-VENV_BUILD_DIR=./venv_build
+if [ "$TRAVIS_OS_NAME" == "osx" ]; then
 
-# Directory qhere to create test virtualenv
-VENV_TEST_DIR=./venv_test
 
-
-if [ "$TRAVIS_OS_NAME" != "osx" ]; then
-    function travis_osx_install_begin {
-        echo NoOp
-    }
-    function travis_osx_install_end {
-        echo NoOp
-    }
-    function travis_osx_run_begin {
-        echo NoOp
-    }
-    function travis_osx_run_end {
-        echo NoOp
-    }
+if [ "$PYTHON_VERSION" == "2" ];
+then
+    curl -o miniconda_installer.sh "https://repo.continuum.io/miniconda/Miniconda2-latest-MacOSX-x86_64.sh"
+    bash miniconda_installer.sh -b -p miniconda
+    export PATH="`pwd`/miniconda/bin":$PATH
 
 else
-    echo "Installing miniconda"
-    curl -o miniconda_installer.sh "https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh"
+# Use brew for python3
+#    VENV_DIR=./venv
+#    brew upgrade python@3
+#    PYTHON_EXE=`brew list python@3 | grep "bin/python3$" | head -n 1`
+#    # Create virtual env
+#    $PYTHON_EXE -m venv $VENV_DIR
+#    source $VENV_DIR/bin/activate
+# Alternative python installation using miniconda
+    curl -o miniconda_installer.sh "https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh"
     bash miniconda_installer.sh -b -p miniconda
     export PATH="`pwd`/miniconda/bin":$PATH
-
-    function travis_osx_install_begin {
-        echo Mac OS X install begin: Install pip and setup build venv
-        set -x  # echo on
-        export PATH=${PATH}:/Users/travis/Library/Python/2.7/bin
-        curl -O https://bootstrap.pypa.io/get-pip.py
-        python get-pip.py --user
-
-        pip install virtualenv --user
-        virtualenv --version
-
-        virtualenv --clear $VENV_BUILD_DIR
-
-        set +x  # echo off
-        echo "Activate virtualenv $VENV_BUILD_DIR"
-        source $VENV_BUILD_DIR/bin/activate
-    }
-
-    function travis_osx_install_end {
-        echo Mac OS X install end: Deactivate and delete virtualenv
-        echo deactivate
-        deactivate
-        set -x  # echo on
-        rm -rf $VENV_BUILD_DIR
-        set +x  # echo off
-    }
-
-
-    function travis_osx_run_begin {
-        echo Mac OS X run begin: Setup test venv
-
-        set -x  # echo on
-        virtualenv --clear $VENV_TEST_DIR
-        set +x  # echo off
-        echo "Activate virtualenv $VENV_TEST_DIR"
-        source $VENV_TEST_DIR/bin/activate
-    }
-
-    function travis_osx_run_end {
-        echo Mac OS X run end: Deactivate and delete virtualenv
-        echo deactivate
-        deactivate
-        set -x  # echo on
-        rm -rf $VENV_TEST_DIR
-        set +x  # echo off
-    }
+fi
 
 fi

hdf5plugin/__init__.py

@@ -63,18 +63,22 @@
 BLOSC_ID = 32001
 """Blosc filter ID"""
 
+LZ4_ID = 32004
+"""LZ4_ID filter ID"""
+
 BSHUF_ID = 32008
 """Bitshuffle filter ID"""
 
-LZ4_ID = 32004
-"""LZ4_ID filter ID"""
+ZFP_ID = 32013
+"""ZFP filter ID"""
 
 FCIDECOMP_ID = 32018
 """FCIDECOMP filter ID"""
 
 FILTERS = {'blosc': BLOSC_ID,
            'bshuf': BSHUF_ID,
            'lz4': LZ4_ID,
+           'zfp': ZFP_ID,
            'fcidecomp': FCIDECOMP_ID,
            }
 """Mapping of filter name to HDF5 filter ID for available filters"""
@@ -191,6 +195,11 @@ def __init__(self, nbytes=0):
         self.filter_options = (nbytes,)
 
 
+class Zfp(_FilterRefClass):
+    """h5py.Group.create_dataset's compression and compression_opts arguments for using ZFP filter.
+    """
+    filter_id = ZFP_ID
+
 class FciDecomp(_FilterRefClass):
     """h5py.Group.create_dataset's compression and compression_opts arguments for using FciDecomp filter.
     """

hdf5plugin/test.py

@@ -1,7 +1,7 @@
 # coding: utf-8
 # /*##########################################################################
 #
-# Copyright (c) 2019 European Synchrotron Radiation Facility
+# Copyright (c) 2019-2020 European Synchrotron Radiation Facility
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -58,6 +58,7 @@ def _test(self, filter_name, dtype=numpy.int32, **options):
                 "bshuf": hdf5plugin.Bitshuffle,
                 "lz4": hdf5plugin.LZ4,
                 "fcidecomp": hdf5plugin.FciDecomp,
+                "zfp": hdf5plugin.Zfp,
                 }[filter_name](**options)
 
         # Write
@@ -71,8 +72,11 @@ def _test(self, filter_name, dtype=numpy.int32, **options):
         plist = f['data'].id.get_create_plist()
         filters = [plist.get_filter(i) for i in range(plist.get_nfilters())]
         f.close()
-
-        self.assertTrue(numpy.array_equal(saved, data))
+
+        if filter_name in ['zfp']:
+            self.assertTrue(numpy.allclose(saved, data))
+        else:
+            self.assertTrue(numpy.array_equal(saved, data))
         self.assertEqual(saved.dtype, data.dtype)
 
         self.assertEqual(len(filters), 1)
@@ -125,6 +129,12 @@ def testFciDecomp(self):
         """Write/read test with fcidecomp filter plugin"""
         self._test('fcidecomp')
 
+    @unittest.skipUnless(h5py.h5z.filter_avail(hdf5plugin.ZFP_ID),
+                         "ZFP filter not available")
+    def testZfp(self):
+        """Write/read test with zfp filter plugin"""
+        self._test('zfp', dtype=numpy.float32)
+        self._test('zfp', dtype=numpy.float64)
 
 def suite():
     test_suite = unittest.TestSuite()

setup.py

@@ -24,7 +24,7 @@
 # ###########################################################################*/
 __authors__ = ["V.A. Sole", "T. Vincent"]
 __license__ = "MIT"
-__date__ = "21/04/2020"
+__date__ = "12/05/2020"
 
 
 from glob import glob
@@ -496,6 +496,33 @@ def prefix(directory, files):
     'extra_link_args': ['-lstdc++'],
     }
 
+# H5Z-ZFP
+h5zfp_dir = 'src/H5Z-ZFP/src'
+extra_compile_args = ['-O3', '-ffast-math', '-std=c99', '-fopenmp']
+extra_compile_args += ['/Ox', '/fp:fast', '/openmp']
+extra_link_args = ['-fopenmp', '/openmp']
+
+sources = glob(h5zfp_dir + "/" + "*.c")
+depends = glob(h5zfp_dir + "/" + "*.h")
+include_dirs = [h5zfp_dir + "/src", "src/zfp/include"]
+h5zfp_plugin = HDF5PluginExtension(
+    "hdf5plugin.plugins.libh5zfp",
+    sources=sources,
+    depends=depends,
+    include_dirs=include_dirs,
+    extra_compile_args=extra_compile_args,
+    extra_link_args=extra_link_args,
+    )
+
+# zfp
+zfp_dir = os.path.join("src", "zfp")
+zfp_sources = glob(os.path.join(zfp_dir, 'src', '*.c'))
+zfp_include_dirs = [os.path.join(zfp_dir, 'include')]
+zfp_lib = ('zfp', {
+    'sources': zfp_sources,
+    'include_dirs': zfp_include_dirs,
+    'cflags': ['-DBIT_STREAM_WORD_TYPE=uint8'],
+    })
 
 libraries = [snappy_lib, charls_lib]
 
@@ -505,6 +532,13 @@ def prefix(directory, files):
               fcidecomp_plugin,
               ]
 
+if sys.platform.startswith("win32") and (sys.version_info < (3,)):
+    logger.warn(
+            "ZFP not supported in this platform: Windows and Python 2")
+else:
+    libraries += [zfp_lib]
+    extensions += [h5zfp_plugin]
+
 # setup
 
 # ########## #