implement fletcher32

numcodecs/__init__.py

@@ -111,3 +111,6 @@
     register_codec(VLenUTF8)
     register_codec(VLenBytes)
     register_codec(VLenArray)
+
+from numcodecs.fletcher32 import Fletcher32
+register_codec(Fletcher32)

numcodecs/fletcher32.pyx

@@ -0,0 +1,54 @@
+# cython: boundscheck=False
+# cython: wraparound=False
+# cython: overflowcheck=False
+# cython: cdivision=True
+
+import struct
+import numpy as np
+
+from numcodecs.abc import Codec
+from numcodecs.compat import ensure_contiguous_ndarray
+
+from libc.stdint cimport uint8_t, uint16_t, uint32_t
+
+cpdef uint32_t fletcher32(const uint16_t[::1] data):
+    cdef:
+        uint32_t sum1 = 0
+        uint32_t sum2 = 0
+        int index
+        int size = data.shape[0]
+
+    for index in range(0, size):
+        sum1 = (sum1 + data[index]) % 0xffff
+        sum2 = (sum2 + sum1) % 0xffff
+
+    return (sum2 << 16) | sum1
+
+
+class Fletcher32(Codec):
+    codec_id = "fletcher32"
+
+    def encode(self, buf):
+        buf = ensure_contiguous_ndarray(buf).ravel()
+        if len(buf) % 2:
+            # rare, odd size of bytes data only
+            arr = np.frombuffer(buf.tobytes() + b"\x00", dtype="uint16")
+            val = fletcher32(arr)
+        else:
+            val = fletcher32(buf.view('uint16'))
+        return buf.tobytes() + struct.pack("<I", val)
+
+    def decode(self, buf, out=None):
+        b = ensure_contiguous_ndarray(buf).view('uint8')
+        if len(buf) % 2:
+            # rare, odd size of bytes data only
+            arr = np.frombuffer(b.tobytes() + b"\x00", dtype="uint16")
+            val = fletcher32(arr)
+        else:
+            val = fletcher32(b[:-4].view('uint16'))
+        found = b[-4:].view('uint32')[0]
+        assert val == found
+        if out:
+            out.view("uint8")[:] = b[:-4]
+            return out
+        return memoryview(b[:-4])

numcodecs/tests/test_fletcher32.py

@@ -0,0 +1,24 @@
+import numpy as np
+import pytest
+
+from numcodecs.fletcher32 import Fletcher32, fletcher32
+
+
+@pytest.mark.parametrize("inval,outval", [
+    [b"abcdef", 1448095018],
+    [b"abcdefgh", 3957429649]
+])
+def test_vectors(inval, outval):
+    arr = np.array(list(inval), dtype="uint8").view('uint16')
+    assert fletcher32(arr) == outval
+
+
+@pytest.mark.parametrize(
+    "dtype",
+    ["uint8", "int32", "float32"]
+)
+def test_with_data(dtype):
+    data = np.empty(100, dtype=dtype)
+    f = Fletcher32()
+    arr = np.frombuffer(f.decode(f.encode(data)), dtype=dtype)
+    assert (arr == data).all()

setup.py

@@ -198,6 +198,31 @@ def vlen_extension():
     return extensions
 
 
+def fletcher_extension():
+    info('setting up fletcher32 extension')
+
+    extra_compile_args = base_compile_args.copy()
+    define_macros = []
+
+    # setup sources
+    include_dirs = ['numcodecs']
+    # define_macros += [('CYTHON_TRACE', '1')]
+
+    sources = ['numcodecs/fletcher32.pyx']
+
+    # define extension module
+    extensions = [
+        Extension('numcodecs.fletcher32',
+                  sources=sources,
+                  include_dirs=include_dirs,
+                  define_macros=define_macros,
+                  extra_compile_args=extra_compile_args,
+                  ),
+    ]
+
+    return extensions
+
+
 def compat_extension():
     info('setting up compat extension')
 
@@ -265,7 +290,8 @@ def run_setup(with_extensions):
 
     if with_extensions:
         ext_modules = (blosc_extension() + zstd_extension() + lz4_extension() +
-                       compat_extension() + shuffle_extension() + vlen_extension())
+                       compat_extension() + shuffle_extension() + vlen_extension() +
+                       fletcher_extension())
 
         cmdclass = dict(build_ext=ve_build_ext)
     else: