assert, test, and CI misc

.github/workflows/debug.yaml

@@ -9,7 +9,7 @@ jobs:
       matrix:
         rust: [
           { version: "1.60" }, # MSRV
-          { version: "nightly-2023-06-30" },
+          { version: "nightly-2023-08-30" },
         ]
         python: [
           { version: '3.11', abi: 'cp311-cp311' },
@@ -43,6 +43,9 @@ jobs:
     - run: python -m pip install --user -r test/requirements.txt -r integration/requirements.txt
 
     - run: pytest -s -rxX -v test
+      env:
+        PYTHONMALLOC: "debug"
+
     - run: ./integration/run thread
     - run: ./integration/run http
     - run: ./integration/run init

.github/workflows/linux.yaml

@@ -31,6 +31,9 @@ jobs:
     - run: python3 -m pip install --user -r test/requirements.txt -r integration/requirements.txt mypy
 
     - run: pytest -s -rxX -v test
+      env:
+        PYTHONMALLOC: "debug"
+
     - run: ./integration/run thread
     - run: ./integration/run http
     - run: ./integration/run init
@@ -69,8 +72,8 @@ jobs:
       options: --user 0
     steps:
     - run: yum install -y clang lld
-    - run: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2023-06-30 --profile minimal -y
-    - run: rustup component add rust-src --toolchain nightly-2023-06-30-x86_64-unknown-linux-gnu
+    - run: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2023-08-30 --profile minimal -y
+    - run: rustup component add rust-src --toolchain nightly-2023-08-30-x86_64-unknown-linux-gnu
     - uses: actions/checkout@v3
 
     - name: build-std
@@ -91,6 +94,9 @@ jobs:
     - run: python3 -m pip install --user -r test/requirements.txt -r integration/requirements.txt
 
     - run: pytest -s -rxX -v test
+      env:
+        PYTHONMALLOC: "debug"
+
     - run: ./integration/run thread
     - run: ./integration/run http
     - run: ./integration/run init
@@ -141,7 +147,7 @@ jobs:
         RUSTFLAGS: "-C target-feature=-crt-static"
         CARGO_UNSTABLE_SPARSE_REGISTRY: "true"
       with:
-        rust-toolchain: nightly-2023-06-30
+        rust-toolchain: nightly-2023-08-30
         rustup-components: rust-src
         target: ${{ matrix.platform.target }}
         manylinux: musllinux_1_1
@@ -230,7 +236,7 @@ jobs:
         CARGO_UNSTABLE_SPARSE_REGISTRY: "true"
       with:
         target: ${{ matrix.target.target }}
-        rust-toolchain: nightly-2023-06-30
+        rust-toolchain: nightly-2023-08-30
         rustup-components: rust-src
         manylinux: auto
         args: --release --strip --out=dist --features=no-panic,yyjson -i python${{ matrix.python.version }}

README.md

@@ -1195,7 +1195,7 @@ It benefits from also having a C build environment to compile a faster
 deserialization backend. See this project's `manylinux_2_28` builds for an
 example using clang and LTO.
 
-The project's own CI tests against `nightly-2023-06-30` and stable 1.60. It
+The project's own CI tests against `nightly-2023-08-30` and stable 1.60. It
 is prudent to pin the nightly version because that channel can introduce
 breaking changes.
 

ci/azure-macos.yml

@@ -37,6 +37,8 @@ steps:
   displayName: install universal2
 
 - bash: pytest -s -rxX -v test
+  env:
+    PYTHONMALLOC: "debug"
   displayName: pytest
 - bash: ./integration/run thread
   displayName: thread

ci/azure-pipelines.yml

@@ -1,5 +1,5 @@
 variables:
-  toolchain: nightly-2023-06-30
+  toolchain: nightly-2023-08-30
 
 jobs:
 

ci/azure-win.yml

@@ -24,6 +24,8 @@ steps:
 - script: python.exe -m pip install orjson --no-index --find-links=D:\a\1\s\target\wheels
   displayName: install
 - script: python.exe -m pytest -s -rxX -v test
+  env:
+    PYTHONMALLOC: "debug"
   displayName: pytest
 - script: python.exe integration\thread
   displayName: thread

integration/client

@@ -9,7 +9,7 @@ import httpx
 port = sys.argv[1]
 url = f"http://127.0.0.1:{port}"
 
-timeout = httpx.Timeout(5.0, connect_timeout=5.0)
+timeout = httpx.Timeout(5.0)
 client = httpx.AsyncClient(timeout=timeout)
 
 stop_time = time.time() + 5
@@ -30,3 +30,4 @@ async def main():
 loop = asyncio.new_event_loop()
 asyncio.set_event_loop(loop)
 asyncio.run(main())
+loop.close()

integration/requirements.txt

@@ -1,3 +1,3 @@
 flask;sys_platform!="win"
 gunicorn;sys_platform!="win"
-httpx==0.14.3;sys_platform!="win"
+httpx==0.24.1;sys_platform!="win"

integration/run

@@ -6,6 +6,8 @@ _dir="$(dirname "${BASH_SOURCE[0]}")"
 
 to_run="${@:-thread http init}"
 
+export PYTHONMALLOC="debug"
+
 if [[ $to_run == *"thread"* ]]; then
 	"${_dir}"/thread
 fi

script/develop

@@ -7,6 +7,6 @@ export CFLAGS="-O2 -fno-plt -flto=thin"
 export LDFLAGS="${CFLAGS} -fuse-ld=lld -Wl,--as-needed"
 export RUSTFLAGS="-C linker=clang -C link-arg=-fuse-ld=lld"
 
-maturin build --release "$@"
+maturin build "$@"
 
 pip install --force target/wheels/*.whl

script/pytest

@@ -1,3 +1,3 @@
 #!/bin/sh -e
 
-pytest -s -rxX --random-order test
+PYTHONMALLOC="debug" pytest -s -rxX --random-order test

src/deserialize/cache.rs

@@ -22,9 +22,9 @@ impl CachedKey {
             ptr: ptr as *mut c_void,
         }
     }
-
     pub fn get(&mut self) -> *mut pyo3_ffi::PyObject {
         let ptr = self.ptr as *mut pyo3_ffi::PyObject;
+        debug_assert!(ffi!(Py_REFCNT(ptr)) >= 1);
         ffi!(Py_INCREF(ptr));
         ptr
     }

src/deserialize/deserializer.rs

@@ -8,6 +8,7 @@ use std::ptr::NonNull;
 pub fn deserialize(
     ptr: *mut pyo3_ffi::PyObject,
 ) -> Result<NonNull<pyo3_ffi::PyObject>, DeserializeError<'static>> {
+    debug_assert!(ffi!(Py_REFCNT(ptr)) >= 1);
     let buffer = read_input_to_buf(ptr)?;
 
     if unlikely!(buffer.len() == 2) {

src/deserialize/json.rs

@@ -136,8 +136,10 @@ impl<'de> Visitor<'de> for JsonValue {
                     str_hash!(pykey),
                 )
             };
-            py_decref_without_destroy!(pykey);
-            py_decref_without_destroy!(pyval.as_ptr());
+            debug_assert!(ffi!(Py_REFCNT(pykey)) >= 2);
+            reverse_pydict_incref!(pykey);
+            debug_assert!(ffi!(Py_REFCNT(pyval.as_ptr())) >= 2);
+            reverse_pydict_incref!(pyval.as_ptr());
         }
         Ok(nonnull!(dict_ptr))
     }

src/deserialize/pyobject.rs

@@ -29,6 +29,8 @@ pub fn get_unicode_key(key_str: &str) -> *mut pyo3_ffi::PyObject {
         );
         pykey = entry.get();
     }
+    debug_assert!(ffi!(Py_REFCNT(pykey)) >= 1);
+    debug_assert!(unsafe { (*pykey.cast::<pyo3_ffi::PyASCIIObject>()).hash != -1 });
     pykey
 }
 

src/deserialize/utf8.rs

@@ -56,6 +56,7 @@ pub fn read_input_to_buf(
         buffer = unsafe { std::slice::from_raw_parts(as_str.as_ptr(), as_str.len()) };
     } else if unlikely!(is_type!(obj_type_ptr, MEMORYVIEW_TYPE)) {
         let membuf = unsafe { PyMemoryView_GET_BUFFER(ptr) };
+        debug_assert!(ffi!(Py_REFCNT(membuf as *mut pyo3_ffi::PyObject)) >= 1);
         if unsafe { pyo3_ffi::PyBuffer_IsContiguous(membuf, b'C' as c_char) == 0 } {
             return Err(DeserializeError::invalid(Cow::Borrowed(
                 "Input type memoryview must be a C contiguous buffer",

src/deserialize/yyjson.rs

@@ -185,8 +185,8 @@ fn parse_yy_object(elem: *mut yyjson_val) -> NonNull<pyo3_ffi::PyObject> {
             let _ = unsafe {
                 pyo3_ffi::_PyDict_SetItem_KnownHash(dict, pykey, pyval, str_hash!(pykey))
             };
-            py_decref_without_destroy!(pykey);
-            py_decref_without_destroy!(pyval);
+            reverse_pydict_incref!(pykey);
+            reverse_pydict_incref!(pyval);
         }
         nonnull!(dict)
     }

src/ffi/immortal.rs

@@ -0,0 +1,27 @@
+// SPDX-License-Identifier: (Apache-2.0 OR MIT)
+// copied from PyO3 when 0.19.2 was latest
+
+#[allow(non_snake_case)]
+#[cfg(all(Py_3_12, target_pointer_width = "32"))]
+pub const _Py_IMMORTAL_REFCNT: pyo3_ffi::Py_ssize_t = {
+    if cfg!(target_pointer_width = "64") {
+        std::os::raw::c_uint::MAX as pyo3_ffi::Py_ssize_t
+    } else {
+        // for 32-bit systems, use the lower 30 bits (see comment in CPython's object.h)
+        (std::os::raw::c_uint::MAX >> 2) as pyo3_ffi::Py_ssize_t
+    }
+};
+
+#[inline(always)]
+#[allow(non_snake_case)]
+#[cfg(all(Py_3_12, target_pointer_width = "64"))]
+pub unsafe fn _Py_IsImmortal(op: *mut pyo3_ffi::PyObject) -> std::os::raw::c_int {
+    (((*op).ob_refcnt.ob_refcnt as i32) < 0) as std::os::raw::c_int
+}
+
+#[inline(always)]
+#[allow(non_snake_case)]
+#[cfg(all(Py_3_12, target_pointer_width = "32"))]
+pub unsafe fn _Py_IsImmortal(op: *mut pyo3_ffi::PyObject) -> std::os::raw::c_int {
+    ((*op).ob_refcnt.ob_refcnt == _Py_IMMORTAL_REFCNT) as std::os::raw::c_int
+}

src/ffi/mod.rs

@@ -4,12 +4,16 @@ mod buffer;
 mod bytes;
 mod dict;
 mod fragment;
+#[cfg(Py_3_12)]
+mod immortal;
 mod list;
 mod long;
 
 pub use buffer::*;
 pub use bytes::*;
 pub use dict::*;
 pub use fragment::{orjson_fragmenttype_new, Fragment};
+#[cfg(Py_3_12)]
+pub use immortal::_Py_IsImmortal;
 pub use list::PyListIter;
 pub use long::{pylong_is_unsigned, pylong_is_zero};