gh-81057: Move the Allocators to _PyRuntimeState (gh-99217)

The global allocators were stored in 3 static global variables: _PyMem_Raw, _PyMem, and _PyObject.  State for the "small block" allocator was stored in another 13.  That makes a total of 16 global variables. We are moving all 16 to the _PyRuntimeState struct as part of the work for gh-81057.  (If PEP 684 is accepted then we will follow up by moving them all to PyInterpreterState.)

#81057

Include/internal/pycore_obmalloc_init.h

@@ -0,0 +1,68 @@
+#ifndef Py_INTERNAL_OBMALLOC_INIT_H
+#define Py_INTERNAL_OBMALLOC_INIT_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef Py_BUILD_CORE
+#  error "this header requires Py_BUILD_CORE define"
+#endif
+
+
+/****************************************************/
+/* the default object allocator's state initializer */
+
+#define PTA(pools, x) \
+    ((poolp )((uint8_t *)&(pools.used[2*(x)]) - 2*sizeof(pymem_block *)))
+#define PT(p, x)   PTA(p, x), PTA(p, x)
+
+#define PT_8(p, start) \
+    PT(p, start), \
+    PT(p, start+1), \
+    PT(p, start+2), \
+    PT(p, start+3), \
+    PT(p, start+4), \
+    PT(p, start+5), \
+    PT(p, start+6), \
+    PT(p, start+7)
+
+#if NB_SMALL_SIZE_CLASSES <= 8
+#  define _obmalloc_pools_INIT(p) \
+    { PT_8(p, 0) }
+#elif NB_SMALL_SIZE_CLASSES <= 16
+#  define _obmalloc_pools_INIT(p) \
+    { PT_8(p, 0), PT_8(p, 8) }
+#elif NB_SMALL_SIZE_CLASSES <= 24
+#  define _obmalloc_pools_INIT(p) \
+    { PT_8(p, 0), PT_8(p, 8), PT_8(p, 16) }
+#elif NB_SMALL_SIZE_CLASSES <= 32
+#  define _obmalloc_pools_INIT(p) \
+    { PT_8(p, 0), PT_8(p, 8), PT_8(p, 16), PT_8(p, 24) }
+#elif NB_SMALL_SIZE_CLASSES <= 40
+#  define _obmalloc_pools_INIT(p) \
+    { PT_8(p, 0), PT_8(p, 8), PT_8(p, 16), PT_8(p, 24), PT_8(p, 32) }
+#elif NB_SMALL_SIZE_CLASSES <= 48
+#  define _obmalloc_pools_INIT(p) \
+    { PT_8(p, 0), PT_8(p, 8), PT_8(p, 16), PT_8(p, 24), PT_8(p, 32), PT_8(p, 40) }
+#elif NB_SMALL_SIZE_CLASSES <= 56
+#  define _obmalloc_pools_INIT(p) \
+    { PT_8(p, 0), PT_8(p, 8), PT_8(p, 16), PT_8(p, 24), PT_8(p, 32), PT_8(p, 40), PT_8(p, 48) }
+#elif NB_SMALL_SIZE_CLASSES <= 64
+#  define _obmalloc_pools_INIT(p) \
+    { PT_8(p, 0), PT_8(p, 8), PT_8(p, 16), PT_8(p, 24), PT_8(p, 32), PT_8(p, 40), PT_8(p, 48), PT_8(p, 56) }
+#else
+#  error "NB_SMALL_SIZE_CLASSES should be less than 64"
+#endif
+
+#define _obmalloc_state_INIT(obmalloc) \
+    { \
+        .pools = { \
+            .used = _obmalloc_pools_INIT(obmalloc.pools), \
+        }, \
+    }
+
+
+#ifdef __cplusplus
+}
+#endif
+#endif  // !Py_INTERNAL_OBMALLOC_INIT_H

Include/internal/pycore_pymem.h

@@ -11,6 +11,27 @@ extern "C" {
 #include "pymem.h"      // PyMemAllocatorName
 
 
+typedef struct {
+    /* We tag each block with an API ID in order to tag API violations */
+    char api_id;
+    PyMemAllocatorEx alloc;
+} debug_alloc_api_t;
+
+struct _pymem_allocators {
+    struct {
+        PyMemAllocatorEx raw;
+        PyMemAllocatorEx mem;
+        PyMemAllocatorEx obj;
+    } standard;
+    struct {
+        debug_alloc_api_t raw;
+        debug_alloc_api_t mem;
+        debug_alloc_api_t obj;
+    } debug;
+    PyObjectArenaAllocator obj_arena;
+};
+
+
 /* Set the memory allocator of the specified domain to the default.
    Save the old allocator into *old_alloc if it's non-NULL.
    Return on success, or return -1 if the domain is unknown. */
@@ -94,20 +115,6 @@ struct _PyTraceMalloc_Config {
 
 PyAPI_DATA(struct _PyTraceMalloc_Config) _Py_tracemalloc_config;
 
-/* Allocate memory directly from the O/S virtual memory system,
- * where supported. Otherwise fallback on malloc */
-void *_PyObject_VirtualAlloc(size_t size);
-void _PyObject_VirtualFree(void *, size_t size);
-
-/* This function returns the number of allocated memory blocks, regardless of size */
-PyAPI_FUNC(Py_ssize_t) _Py_GetAllocatedBlocks(void);
-
-/* Macros */
-#ifdef WITH_PYMALLOC
-// Export the symbol for the 3rd party guppy3 project
-PyAPI_FUNC(int) _PyObject_DebugMallocStats(FILE *out);
-#endif
-
 #ifdef __cplusplus
 }
 #endif

Include/internal/pycore_pymem_init.h

@@ -0,0 +1,85 @@
+#ifndef Py_INTERNAL_PYMEM_INIT_H
+#define Py_INTERNAL_PYMEM_INIT_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef Py_BUILD_CORE
+#  error "this header requires Py_BUILD_CORE define"
+#endif
+
+#include "pycore_pymem.h"
+
+
+/********************************/
+/* the allocators' initializers */
+
+extern void * _PyMem_RawMalloc(void *, size_t);
+extern void * _PyMem_RawCalloc(void *, size_t, size_t);
+extern void * _PyMem_RawRealloc(void *, void *, size_t);
+extern void _PyMem_RawFree(void *, void *);
+#define PYRAW_ALLOC {NULL, _PyMem_RawMalloc, _PyMem_RawCalloc, _PyMem_RawRealloc, _PyMem_RawFree}
+
+#ifdef WITH_PYMALLOC
+extern void* _PyObject_Malloc(void *, size_t);
+extern void* _PyObject_Calloc(void *, size_t, size_t);
+extern void _PyObject_Free(void *, void *);
+extern void* _PyObject_Realloc(void *, void *, size_t);
+#  define PYOBJ_ALLOC {NULL, _PyObject_Malloc, _PyObject_Calloc, _PyObject_Realloc, _PyObject_Free}
+#else
+# define PYOBJ_ALLOC PYRAW_ALLOC
+#endif  // WITH_PYMALLOC
+
+#define PYMEM_ALLOC PYOBJ_ALLOC
+
+extern void* _PyMem_DebugRawMalloc(void *, size_t);
+extern void* _PyMem_DebugRawCalloc(void *, size_t, size_t);
+extern void* _PyMem_DebugRawRealloc(void *, void *, size_t);
+extern void _PyMem_DebugRawFree(void *, void *);
+
+extern void* _PyMem_DebugMalloc(void *, size_t);
+extern void* _PyMem_DebugCalloc(void *, size_t, size_t);
+extern void* _PyMem_DebugRealloc(void *, void *, size_t);
+extern void _PyMem_DebugFree(void *, void *);
+
+#define PYDBGRAW_ALLOC(runtime) \
+    {&(runtime).allocators.debug.raw, _PyMem_DebugRawMalloc, _PyMem_DebugRawCalloc, _PyMem_DebugRawRealloc, _PyMem_DebugRawFree}
+#define PYDBGMEM_ALLOC(runtime) \
+    {&(runtime).allocators.debug.mem, _PyMem_DebugMalloc, _PyMem_DebugCalloc, _PyMem_DebugRealloc, _PyMem_DebugFree}
+#define PYDBGOBJ_ALLOC(runtime) \
+    {&(runtime).allocators.debug.obj, _PyMem_DebugMalloc, _PyMem_DebugCalloc, _PyMem_DebugRealloc, _PyMem_DebugFree}
+
+extern void * _PyMem_ArenaAlloc(void *, size_t);
+extern void _PyMem_ArenaFree(void *, void *, size_t);
+
+#ifdef Py_DEBUG
+# define _pymem_allocators_standard_INIT(runtime) \
+    { \
+        PYDBGRAW_ALLOC(runtime), \
+        PYDBGMEM_ALLOC(runtime), \
+        PYDBGOBJ_ALLOC(runtime), \
+    }
+#else
+# define _pymem_allocators_standard_INIT(runtime) \
+    { \
+        PYRAW_ALLOC, \
+        PYMEM_ALLOC, \
+        PYOBJ_ALLOC, \
+    }
+#endif
+
+#define _pymem_allocators_debug_INIT \
+   { \
+       {'r', PYRAW_ALLOC}, \
+       {'m', PYMEM_ALLOC}, \
+       {'o', PYOBJ_ALLOC}, \
+   }
+
+#  define _pymem_allocators_obj_arena_INIT \
+    { NULL, _PyMem_ArenaAlloc, _PyMem_ArenaFree }
+
+
+#ifdef __cplusplus
+}
+#endif
+#endif  // !Py_INTERNAL_PYMEM_INIT_H

Include/internal/pycore_runtime.h

@@ -13,6 +13,8 @@ extern "C" {
 #include "pycore_global_objects.h"  // struct _Py_global_objects
 #include "pycore_import.h"          // struct _import_runtime_state
 #include "pycore_interp.h"          // PyInterpreterState
+#include "pycore_pymem.h"           // struct _pymem_allocators
+#include "pycore_obmalloc.h"        // struct obmalloc_state
 #include "pycore_unicodeobject.h"   // struct _Py_unicode_runtime_ids
 
 struct _getargs_runtime_state {
@@ -86,6 +88,9 @@ typedef struct pyruntimestate {
        to access it, don't access it directly. */
     _Py_atomic_address _finalizing;
 
+    struct _pymem_allocators allocators;
+    struct _obmalloc_state obmalloc;
+
     struct pyinterpreters {
         PyThread_type_lock mutex;
         /* The linked list of interpreters, newest first. */

Include/internal/pycore_runtime_init.h

@@ -9,20 +9,28 @@ extern "C" {
 #endif
 
 #include "pycore_object.h"
+#include "pycore_pymem_init.h"
+#include "pycore_obmalloc_init.h"
 
 
 /* The static initializers defined here should only be used
    in the runtime init code (in pystate.c and pylifecycle.c). */
 
 
-#define _PyRuntimeState_INIT \
+#define _PyRuntimeState_INIT(runtime) \
     { \
         .gilstate = { \
             .check_enabled = 1, \
             /* A TSS key must be initialized with Py_tss_NEEDS_INIT \
                in accordance with the specification. */ \
             .autoTSSkey = Py_tss_NEEDS_INIT, \
         }, \
+        .allocators = { \
+            _pymem_allocators_standard_INIT(runtime), \
+            _pymem_allocators_debug_INIT, \
+            _pymem_allocators_obj_arena_INIT, \
+        }, \
+        .obmalloc = _obmalloc_state_INIT(runtime.obmalloc), \
         .interpreters = { \
             /* This prevents interpreters from getting created \
               until _PyInterpreterState_Enable() is called. */ \

Makefile.pre.in

@@ -1650,12 +1650,15 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/internal/pycore_moduleobject.h \
 		$(srcdir)/Include/internal/pycore_namespace.h \
 		$(srcdir)/Include/internal/pycore_object.h \
+		$(srcdir)/Include/internal/pycore_obmalloc.h \
+		$(srcdir)/Include/internal/pycore_obmalloc_init.h \
 		$(srcdir)/Include/internal/pycore_pathconfig.h \
 		$(srcdir)/Include/internal/pycore_pyarena.h \
 		$(srcdir)/Include/internal/pycore_pyerrors.h \
 		$(srcdir)/Include/internal/pycore_pyhash.h \
 		$(srcdir)/Include/internal/pycore_pylifecycle.h \
 		$(srcdir)/Include/internal/pycore_pymem.h \
+		$(srcdir)/Include/internal/pycore_pymem_init.h \
 		$(srcdir)/Include/internal/pycore_pystate.h \
 		$(srcdir)/Include/internal/pycore_range.h \
 		$(srcdir)/Include/internal/pycore_runtime.h \

Misc/NEWS.d/next/Core and Builtins/2022-11-07-14-16-59.gh-issue-81057.3uKlLQ.rst

@@ -0,0 +1,3 @@
+The 18 global C variables holding the state of the allocators have been
+moved to ``_PyRuntimeState``.  This is a strictly internal change with no
+change in behavior.