GH-101578: Normalize the current exception (GH-101607)

* Make sure that the current exception is always normalized.

* Remove redundant type and traceback fields for the current exception.

* Add new API functions: PyErr_GetRaisedException, PyErr_SetRaisedException

* Add new API functions: PyException_GetArgs, PyException_SetArgs

Doc/c-api/exceptions.rst

@@ -400,8 +400,61 @@ Querying the error indicator
    recursively in subtuples) are searched for a match.
 
 
+.. c:function:: PyObject *PyErr_GetRaisedException(void)
+
+   Returns the exception currently being raised, clearing the exception at
+   the same time. Do not confuse this with the exception currently being
+   handled which can be accessed with  :c:func:`PyErr_GetHandledException`.
+
+   .. note::
+
+      This function is normally only used by code that needs to catch exceptions or
+      by code that needs to save and restore the error indicator temporarily, e.g.::
+
+         {
+            PyObject *exc = PyErr_GetRaisedException();
+
+            /* ... code that might produce other errors ... */
+
+            PyErr_SetRaisedException(exc);
+         }
+
+   .. versionadded:: 3.12
+
+
+.. c:function:: void PyErr_SetRaisedException(PyObject *exc)
+
+   Sets the exception currently being raised ``exc``.
+   If the exception is already set, it is cleared first.
+
+   ``exc`` must be a valid exception.
+   (Violating this rules will cause subtle problems later.)
+   This call consumes a reference to the ``exc`` object: you must own a
+   reference to that object before the call and after the call you no longer own
+   that reference.
+   (If you don't understand this, don't use this function. I warned you.)
+
+   .. note::
+
+      This function is normally only used by code that needs to save and restore the
+      error indicator temporarily.  Use :c:func:`PyErr_GetRaisedException` to save
+      the current exception, e.g.::
+
+         {
+            PyObject *exc = PyErr_GetRaisedException();
+
+            /* ... code that might produce other errors ... */
+
+            PyErr_SetRaisedException(exc);
+         }
+
+   .. versionadded:: 3.12
+
+
 .. c:function:: void PyErr_Fetch(PyObject **ptype, PyObject **pvalue, PyObject **ptraceback)
 
+    As of 3.12, this function is deprecated. Use :c:func:`PyErr_GetRaisedException` instead.
+
    Retrieve the error indicator into three variables whose addresses are passed.
    If the error indicator is not set, set all three variables to ``NULL``.  If it is
    set, it will be cleared and you own a reference to each object retrieved.  The
@@ -421,10 +474,14 @@ Querying the error indicator
             PyErr_Restore(type, value, traceback);
          }
 
+   .. deprecated:: 3.12
+
 
 .. c:function:: void PyErr_Restore(PyObject *type, PyObject *value, PyObject *traceback)
 
-   Set  the error indicator from the three objects.  If the error indicator is
+    As of 3.12, this function is deprecated. Use :c:func:`PyErr_SetRaisedException` instead.
+
+   Set the error indicator from the three objects.  If the error indicator is
    already set, it is cleared first.  If the objects are ``NULL``, the error
    indicator is cleared.  Do not pass a ``NULL`` type and non-``NULL`` value or
    traceback.  The exception type should be a class.  Do not pass an invalid
@@ -440,9 +497,15 @@ Querying the error indicator
       error indicator temporarily.  Use :c:func:`PyErr_Fetch` to save the current
       error indicator.
 
+   .. deprecated:: 3.12
+
 
 .. c:function:: void PyErr_NormalizeException(PyObject **exc, PyObject **val, PyObject **tb)
 
+   As of 3.12, this function is deprecated.
+   Use :c:func:`PyErr_GetRaisedException` instead of :c:func:`PyErr_Fetch` to avoid
+   any possible de-normalization.
+
    Under certain circumstances, the values returned by :c:func:`PyErr_Fetch` below
    can be "unnormalized", meaning that ``*exc`` is a class object but ``*val`` is
    not an instance of the  same class.  This function can be used to instantiate
@@ -459,6 +522,8 @@ Querying the error indicator
            PyException_SetTraceback(val, tb);
          }
 
+   .. deprecated:: 3.12
+
 
 .. c:function:: PyObject* PyErr_GetHandledException(void)
 
@@ -704,6 +769,18 @@ Exception Objects
    :attr:`__suppress_context__` is implicitly set to ``True`` by this function.
 
 
+.. c:function:: PyObject* PyException_GetArgs(PyObject *ex)
+
+   Return args of the given exception as a new reference,
+   as accessible from Python through :attr:`args`.
+
+
+.. c:function:: void PyException_SetArgs(PyObject *ex, PyObject *args)
+
+   Set the args of the given exception,
+   as accessible from Python through :attr:`args`.
+
+
 .. _unicodeexceptions:
 
 Unicode Exception Objects

Include/cpython/pyerrors.h

@@ -99,6 +99,7 @@ PyAPI_FUNC(void) _PyErr_GetExcInfo(PyThreadState *, PyObject **, PyObject **, Py
 /* Context manipulation (PEP 3134) */
 
 PyAPI_FUNC(void) _PyErr_ChainExceptions(PyObject *, PyObject *, PyObject *);
+PyAPI_FUNC(void) _PyErr_ChainExceptions1(PyObject *);
 
 /* Like PyErr_Format(), but saves current exception as __context__ and
    __cause__.

Include/cpython/pystate.h

@@ -166,9 +166,7 @@ struct _ts {
     PyObject *c_traceobj;
 
     /* The exception currently being raised */
-    PyObject *curexc_type;
-    PyObject *curexc_value;
-    PyObject *curexc_traceback;
+    PyObject *current_exception;
 
     /* Pointer to the top of the exception stack for the exceptions
      * we may be currently handling.  (See _PyErr_StackItem above.)

Include/internal/pycore_pyerrors.h

@@ -20,7 +20,10 @@ extern void _PyErr_FiniTypes(PyInterpreterState *);
 static inline PyObject* _PyErr_Occurred(PyThreadState *tstate)
 {
     assert(tstate != NULL);
-    return tstate->curexc_type;
+    if (tstate->current_exception == NULL) {
+        return NULL;
+    }
+    return (PyObject *)Py_TYPE(tstate->current_exception);
 }
 
 static inline void _PyErr_ClearExcState(_PyErr_StackItem *exc_state)
@@ -37,10 +40,16 @@ PyAPI_FUNC(void) _PyErr_Fetch(
     PyObject **value,
     PyObject **traceback);
 
+extern PyObject *
+_PyErr_GetRaisedException(PyThreadState *tstate);
+
 PyAPI_FUNC(int) _PyErr_ExceptionMatches(
     PyThreadState *tstate,
     PyObject *exc);
 
+void
+_PyErr_SetRaisedException(PyThreadState *tstate, PyObject *exc);
+
 PyAPI_FUNC(void) _PyErr_Restore(
     PyThreadState *tstate,
     PyObject *type,

Include/pyerrors.h

@@ -18,6 +18,8 @@ PyAPI_FUNC(PyObject *) PyErr_Occurred(void);
 PyAPI_FUNC(void) PyErr_Clear(void);
 PyAPI_FUNC(void) PyErr_Fetch(PyObject **, PyObject **, PyObject **);
 PyAPI_FUNC(void) PyErr_Restore(PyObject *, PyObject *, PyObject *);
+PyAPI_FUNC(PyObject *) PyErr_GetRaisedException(void);
+PyAPI_FUNC(void) PyErr_SetRaisedException(PyObject *);
 #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030b0000
 PyAPI_FUNC(PyObject*) PyErr_GetHandledException(void);
 PyAPI_FUNC(void) PyErr_SetHandledException(PyObject *);
@@ -51,6 +53,10 @@ PyAPI_FUNC(void) PyException_SetCause(PyObject *, PyObject *);
 PyAPI_FUNC(PyObject *) PyException_GetContext(PyObject *);
 PyAPI_FUNC(void) PyException_SetContext(PyObject *, PyObject *);
 
+
+PyAPI_FUNC(PyObject *) PyException_GetArgs(PyObject *);
+PyAPI_FUNC(void) PyException_SetArgs(PyObject *, PyObject *);
+
 /* */
 
 #define PyExceptionClass_Check(x)                                       \

Lib/test/test_capi/test_misc.py

@@ -1553,5 +1553,44 @@ def func2(x=None):
         self.do_test(func2)
 
 
+class Test_ErrSetAndRestore(unittest.TestCase):
+
+    def test_err_set_raised(self):
+        with self.assertRaises(ValueError):
+            _testcapi.err_set_raised(ValueError())
+        v = ValueError()
+        try:
+            _testcapi.err_set_raised(v)
+        except ValueError as ex:
+            self.assertIs(v, ex)
+
+    def test_err_restore(self):
+        with self.assertRaises(ValueError):
+            _testcapi.err_restore(ValueError)
+        with self.assertRaises(ValueError):
+            _testcapi.err_restore(ValueError, 1)
+        with self.assertRaises(ValueError):
+            _testcapi.err_restore(ValueError, 1, None)
+        with self.assertRaises(ValueError):
+            _testcapi.err_restore(ValueError, ValueError())
+        try:
+            _testcapi.err_restore(KeyError, "hi")
+        except KeyError as k:
+            self.assertEqual("hi", k.args[0])
+        try:
+            1/0
+        except Exception as e:
+            tb = e.__traceback__
+        with self.assertRaises(ValueError):
+            _testcapi.err_restore(ValueError, 1, tb)
+        with self.assertRaises(TypeError):
+            _testcapi.err_restore(ValueError, 1, 0)
+        try:
+            _testcapi.err_restore(ValueError, 1, tb)
+        except ValueError as v:
+            self.assertEqual(1, v.args[0])
+            self.assertIs(tb, v.__traceback__.tb_next)
+
+
 if __name__ == "__main__":
     unittest.main()

Lib/test/test_exceptions.py

@@ -347,6 +347,7 @@ def test_capi2():
                 _testcapi.raise_exception(BadException, 0)
             except RuntimeError as err:
                 exc, err, tb = sys.exc_info()
+                tb = tb.tb_next
                 co = tb.tb_frame.f_code
                 self.assertEqual(co.co_name, "__init__")
                 self.assertTrue(co.co_filename.endswith('test_exceptions.py'))
@@ -1415,8 +1416,8 @@ def gen():
     @cpython_only
     def test_recursion_normalizing_infinite_exception(self):
         # Issue #30697. Test that a RecursionError is raised when
-        # PyErr_NormalizeException() maximum recursion depth has been
-        # exceeded.
+        # maximum recursion depth has been exceeded when creating
+        # an exception
         code = """if 1:
             import _testcapi
             try:
@@ -1426,8 +1427,7 @@ def test_recursion_normalizing_infinite_exception(self):
         """
         rc, out, err = script_helper.assert_python_failure("-c", code)
         self.assertEqual(rc, 1)
-        self.assertIn(b'RecursionError: maximum recursion depth exceeded '
-                      b'while normalizing an exception', err)
+        self.assertIn(b'RecursionError: maximum recursion depth exceeded', err)
         self.assertIn(b'Done.', out)
 
 

Misc/NEWS.d/next/C API/2023-02-06-16-14-30.gh-issue-101578.PW5fA9.rst

@@ -0,0 +1,13 @@
+Add new C-API functions for saving and restoring the current exception:
+``PyErr_GetRaisedException`` and ``PyErr_SetRaisedException``.
+These functions take and return a single exception rather than
+the triple of  ``PyErr_Fetch`` and ``PyErr_Restore``.
+This is less error prone and a bit more efficient.
+
+The three arguments forms of saving and restoring the
+current exception: ``PyErr_Fetch`` and ``PyErr_Restore``
+are deprecated.
+
+Also add ``PyException_GetArgs`` and ``PyException_SetArgs``
+as convenience functions to help dealing with
+exceptions in the C API.

Misc/stable_abi.toml

@@ -2333,6 +2333,15 @@
     added = '3.12'
 [function.PyVectorcall_Call]
     added = '3.12'
+[function.PyErr_GetRaisedException]
+    added = '3.12'
+[function.PyErr_SetRaisedException]
+    added = '3.12'
+[function.PyException_GetArgs]
+    added = '3.12'
+[function.PyException_SetArgs]
+    added = '3.12'
+
 [typedef.vectorcallfunc]
     added = '3.12'
 [function.PyObject_Vectorcall]