global: reflect upgrade of zstd to 1.5.4

An error message change resulted in a changed test. Pretty minor.

Rust complained about new fields on `ZSTD_frameHeader`. So we added
them.

Static APIs are now prefixed by new variants of `ZSTDLIB_API`. So
we strip those as well to make cffi parsing happy on non-Windows.

Closes #190.

Cargo.toml

@@ -18,11 +18,11 @@ num_cpus = "1.15.0"
 rayon = "1.6.1"
 
 [dependencies.zstd-safe]
-version = "=6.0.3+zstd.1.5.2"
+version = "6.0.4+zstd.1.5.4"
 features = ["experimental", "legacy", "zstdmt"]
 
 [dependencies.zstd-sys]
-version = "=2.0.6+zstd.1.5.2"
+version = "2.0.7+zstd.1.5.4"
 features = ["experimental", "legacy", "zstdmt"]
 
 [dependencies.pyo3]

c-ext/backend_c.c

@@ -152,7 +152,7 @@ void zstd_module_init(PyObject *m) {
     PyObject *features = NULL;
     PyObject *feature = NULL;
     unsigned zstd_ver_no = ZSTD_versionNumber();
-    unsigned our_hardcoded_version = 10502;
+    unsigned our_hardcoded_version = 10504;
     if (ZSTD_VERSION_NUMBER != our_hardcoded_version ||
         zstd_ver_no != our_hardcoded_version) {
         PyErr_Format(

c-ext/python-zstandard.h

@@ -18,7 +18,7 @@
 #define ZDICT_STATIC_LINKING_ONLY
 
 #ifdef ZSTD_SINGLE_FILE
-#include <zstdlib.c>
+#include <zstd.c>
 
 /* We use private APIs from pool.h. We can't rely on availability
    of this header or symbols when linking against the system libzstd.

docs/installing.rst

@@ -122,7 +122,7 @@ be specified via the ``ZSTD_EXTRA_COMPILER_ARGS`` environment variable. e.g.
 ``python-zstandard`` can be sensitive about what version of ``libzstd`` it links
 against. For best results, point this package at the exact same version of
 ``libzstd`` that it bundles. See the bundled ``zstd/zstd.h`` or
-``zstd/zstdlib.c`` for which version that is.
+``zstd/zstd.c`` for which version that is.
 
 When linking against an external ``libzstd``, not all package features may be
 available. Notably, the ``multi_compress_to_buffer()`` and

docs/news.rst

@@ -82,6 +82,11 @@ Other Actions Not Blocking Release
 0.20.0 (not yet released)
 =========================
 
+Changes
+-------
+
+* Bundled zstd library upgraded from 1.5.2 to 1.5.4.
+
 0.19.0 (released 2022-10-29)
 ============================
 

make_cffi.py

@@ -18,7 +18,7 @@
 HERE = os.path.abspath(os.path.dirname(__file__))
 
 SOURCES = [
-    "zstd/zstdlib.c",
+    "zstd/zstd.c",
 ]
 
 # Headers whose preprocessed output will be fed into cdef().
@@ -101,9 +101,15 @@ def preprocess(path):
                 l = b"#define INT_MAX 2147483647\n"
 
             # ZSTDLIB_API may not be defined if we dropped zstd.h. It isn't
-            # important so just filter it out.
-            if l.startswith(b"ZSTDLIB_API"):
-                l = l[len(b"ZSTDLIB_API ") :]
+            # important so just filter it out. Ditto for ZSTDLIB_STATIC_API and
+            # ZDICTLIB_STATIC_API.
+            for prefix in (
+                b"ZSTDLIB_API",
+                b"ZSTDLIB_STATIC_API",
+                b"ZDICTLIB_STATIC_API",
+            ):
+                if l.startswith(prefix):
+                    l = l[len(prefix) :]
 
             lines.append(l)
 

rust-ext/src/decompressor.rs

@@ -264,6 +264,8 @@ impl ZstdDecompressor {
             headerSize: 0,
             dictID: 0,
             checksumFlag: 0,
+            _reserved1: 0,
+            _reserved2: 0,
         };
         let zresult = unsafe {
             zstd_sys::ZSTD_getFrameHeader(

rust-ext/src/frame_parameters.rs

@@ -80,6 +80,8 @@ fn get_frame_parameters(py: Python, buffer: PyBuffer<u8>) -> PyResult<Py<FramePa
         headerSize: 0,
         dictID: 0,
         checksumFlag: 0,
+        _reserved1: 0,
+        _reserved2: 0,
     };
     let zresult = unsafe {
         zstd_sys::ZSTD_getFrameHeader(&mut header, raw_data.as_ptr() as *const _, raw_data.len())

setup_zstd.py

@@ -80,9 +80,9 @@ def get_c_extension(
         extra_args.append("-DZSTD_MULTITHREAD")
     else:
         extra_args.append("-DZSTD_SINGLE_FILE")
-        extra_args.append("-DZSTDLIB_VISIBILITY=")
-        extra_args.append("-DZDICTLIB_VISIBILITY=")
-        extra_args.append("-DZSTDERRORLIB_VISIBILITY=")
+        extra_args.append("-DZSTDLIB_VISIBLE=")
+        extra_args.append("-DZDICTLIB_VISIBLE=")
+        extra_args.append("-DZSTDERRORLIB_VISIBLE=")
 
         if compiler_type == "unix":
             extra_args.append("-fvisibility=hidden")

tests/test_decompressor_multi_decompress_to_buffer.py

@@ -216,16 +216,12 @@ def test_item_failure(self):
 
         with self.assertRaisesRegex(
             zstd.ZstdError,
-            "error decompressing item 1: ("
-            "Corrupted block|"
-            "Destination buffer is too small)",
+            "error decompressing item 1: (Data corruption detected|Destination buffer is too small)",
         ):
             dctx.multi_decompress_to_buffer(frames)
 
         with self.assertRaisesRegex(
             zstd.ZstdError,
-            "error decompressing item 1: ("
-            "Corrupted block|"
-            "Destination buffer is too small)",
+            "error decompressing item 1: (Data corruption detected|Destination buffer is too small)",
         ):
             dctx.multi_decompress_to_buffer(frames, threads=2)

tests/test_module_attributes.py

@@ -5,7 +5,7 @@
 
 class TestModuleAttributes(unittest.TestCase):
     def test_version(self):
-        self.assertEqual(zstd.ZSTD_VERSION, (1, 5, 2))
+        self.assertEqual(zstd.ZSTD_VERSION, (1, 5, 4))
 
         self.assertEqual(zstd.__version__, "0.20.0.dev0")