Support frame eval in Python 3.9. Fixes microsoft#441

src/debugpy/_vendored/pydevd/_pydevd_frame_eval/.gitignore

@@ -1,2 +1,3 @@
 /pydevd_frame_evaluator.*.so
 /pydevd_frame_evaluator.*.pyd
+/pydevd_frame_evaluator.pyx

src/debugpy/_vendored/pydevd/_pydevd_frame_eval/pydevd_frame_evaluator.pxd

@@ -61,7 +61,9 @@ cdef extern from "Python.h":
     object PyObject_GetAttrString(object o, char *attr_name)
 
 cdef extern from "pystate.h":
-    ctypedef PyObject* _PyFrameEvalFunction(PyFrameObject *frame, int exc)
+    # ctypedef PyObject* _PyFrameEvalFunction(PyThreadState* tstate, PyFrameObject *frame, int exc)
+    # ctypedef PyObject* _PyFrameEvalFunction(PyFrameObject *frame, int exc)
+    ctypedef PyObject* _PyFrameEvalFunction(...)
 
     ctypedef struct PyInterpreterState:
         PyInterpreterState *next
@@ -96,8 +98,28 @@ cdef extern from "pystate.h":
     PyThreadState *PyThreadState_Get()
 
 cdef extern from "ceval.h":
+    '''
+#if PY_VERSION_HEX >= 0x03090000
+PyObject * noop(PyFrameObject *frame, int exc) {
+    return NULL;
+}
+#define CALL_EvalFrameDefault_38(a, b)    noop(a, b)
+#define CALL_EvalFrameDefault_39(a, b, c)    _PyEval_EvalFrameDefault(a, b, c)
+#else
+PyObject * noop(PyThreadState* tstate, PyFrameObject *frame, int exc) {
+    return NULL;
+}
+#define CALL_EvalFrameDefault_39(a, b, c)    noop(a, b, c)
+#define CALL_EvalFrameDefault_38(a, b)    _PyEval_EvalFrameDefault(a, b)
+#endif
+    '''
+
     int _PyEval_RequestCodeExtraIndex(freefunc)
     PyFrameObject *PyEval_GetFrame()
     PyObject* PyEval_CallFunction(PyObject *callable, const char *format, ...)
 
-    PyObject* _PyEval_EvalFrameDefault(PyFrameObject *frame, int exc)
+    # PyObject* _PyEval_EvalFrameDefault(PyThreadState* tstate, PyFrameObject *frame, int exc)
+    # PyObject* _PyEval_EvalFrameDefault(PyFrameObject *frame, int exc)
+    PyObject* _PyEval_EvalFrameDefault(...)
+    PyObject* CALL_EvalFrameDefault_38(PyFrameObject *frame, int exc)  # Actually a macro.
+    PyObject* CALL_EvalFrameDefault_39(PyThreadState* tstate, PyFrameObject *frame, int exc)  # Actually a macro.

src/debugpy/_vendored/pydevd/_pydevd_frame_eval/pydevd_frame_evaluator.pyx → src/debugpy/_vendored/pydevd/_pydevd_frame_eval/pydevd_frame_evaluator.template.pyx

@@ -186,7 +186,7 @@ cdef ThreadInfo get_thread_info():
             # but this is a good point to initialize it.
             global _code_extra_index
             if _code_extra_index == -1:
-                _code_extra_index = _PyEval_RequestCodeExtraIndex(release_co_extra)
+                _code_extra_index = <int> _PyEval_RequestCodeExtraIndex(release_co_extra)
 
             thread_info.initialize_if_possible()
         finally:
@@ -209,7 +209,7 @@ def get_func_code_info_py(thread_info, frame, code_obj) -> FuncCodeInfo:
     return get_func_code_info(<ThreadInfo> thread_info, <PyFrameObject *> frame, <PyCodeObject *> code_obj)
 
 
-_code_extra_index: Py_SIZE = -1
+cdef int _code_extra_index = -1
 
 cdef FuncCodeInfo get_func_code_info(ThreadInfo thread_info, PyFrameObject * frame_obj, PyCodeObject * code_obj):
     '''
@@ -462,19 +462,39 @@ cdef generate_code_with_breakpoints(object code_obj_py, dict breakpoints):
 
     return breakpoint_found, code_obj_py
 
+import sys
 
+cdef bint IS_PY_39_OWNARDS = sys.version_info[:2] >= (3, 9)
+
+def frame_eval_func():
+    cdef PyThreadState *state = PyThreadState_Get()
+    if IS_PY_39_OWNARDS:
+        state.interp.eval_frame = <_PyFrameEvalFunction *> get_bytecode_while_frame_eval_39
+    else:
+        state.interp.eval_frame = <_PyFrameEvalFunction *> get_bytecode_while_frame_eval_38
+    dummy_tracing_holder.set_trace_func(dummy_trace_dispatch)
+
+
+def stop_frame_eval():
+    cdef PyThreadState *state = PyThreadState_Get()
+    state.interp.eval_frame = _PyEval_EvalFrameDefault
+
+# During the build we'll generate 2 versions of the code below so that we're compatible with
+# Python 3.9, which receives a "PyThreadState* tstate" as the first parameter and Python 3.6-3.8
+# which doesn't.
+### TEMPLATE_START
 cdef PyObject * get_bytecode_while_frame_eval(PyFrameObject * frame_obj, int exc):
     '''
     This function makes the actual evaluation and changes the bytecode to a version
     where programmatic breakpoints are added.
     '''
     if GlobalDebuggerHolder is None or _thread_local_info is None or exc:
         # Sometimes during process shutdown these global variables become None
-        return _PyEval_EvalFrameDefault(frame_obj, exc)
+        return CALL_EvalFrameDefault
 
     # co_filename: str = <str>frame_obj.f_code.co_filename
     # if co_filename.endswith('threading.py'):
-    #     return _PyEval_EvalFrameDefault(frame_obj, exc)
+    #     return CALL_EvalFrameDefault
 
     cdef ThreadInfo thread_info
     cdef int STATE_SUSPEND = 2
@@ -489,21 +509,21 @@ cdef PyObject * get_bytecode_while_frame_eval(PyFrameObject * frame_obj, int exc
     except:
         thread_info = get_thread_info()
         if thread_info is None:
-            return _PyEval_EvalFrameDefault(frame_obj, exc)
+            return CALL_EvalFrameDefault
 
     if thread_info.inside_frame_eval:
-        return _PyEval_EvalFrameDefault(frame_obj, exc)
+        return CALL_EvalFrameDefault
 
     if not thread_info.fully_initialized:
         thread_info.initialize_if_possible()
         if not thread_info.fully_initialized:
-            return _PyEval_EvalFrameDefault(frame_obj, exc)
+            return CALL_EvalFrameDefault
 
     # Can only get additional_info when fully initialized.
     cdef PyDBAdditionalThreadInfo additional_info = thread_info.additional_info
     if thread_info.is_pydevd_thread or additional_info.is_tracing:
         # Make sure that we don't trace pydevd threads or inside our own calls.
-        return _PyEval_EvalFrameDefault(frame_obj, exc)
+        return CALL_EvalFrameDefault
 
     # frame = <object> frame_obj
     # DEBUG = frame.f_code.co_filename.endswith('_debugger_case_tracing.py')
@@ -515,7 +535,7 @@ cdef PyObject * get_bytecode_while_frame_eval(PyFrameObject * frame_obj, int exc
     try:
         main_debugger: object = GlobalDebuggerHolder.global_dbg
         if main_debugger is None:
-            return _PyEval_EvalFrameDefault(frame_obj, exc)
+            return CALL_EvalFrameDefault
         frame = <object> frame_obj
 
         if thread_info.thread_trace_func is None:
@@ -582,15 +602,5 @@ cdef PyObject * get_bytecode_while_frame_eval(PyFrameObject * frame_obj, int exc
         thread_info.inside_frame_eval -= 1
         additional_info.is_tracing = False
 
-    return _PyEval_EvalFrameDefault(frame_obj, exc)
-
-
-def frame_eval_func():
-    cdef PyThreadState *state = PyThreadState_Get()
-    state.interp.eval_frame = get_bytecode_while_frame_eval
-    dummy_tracing_holder.set_trace_func(dummy_trace_dispatch)
-
-
-def stop_frame_eval():
-    cdef PyThreadState *state = PyThreadState_Get()
-    state.interp.eval_frame = _PyEval_EvalFrameDefault
+    return CALL_EvalFrameDefault
+### TEMPLATE_END

src/debugpy/_vendored/pydevd/build_tools/build.py

@@ -104,13 +104,13 @@ def build():
         # set VS100COMNTOOLS=C:\Program Files (x86)\Microsoft Visual Studio 9.0\Common7\Tools
 
         env = os.environ.copy()
-        if sys.version_info[:2] in ((2, 6), (2, 7), (3, 5), (3, 6), (3, 7), (3, 8)):
+        if sys.version_info[:2] in ((2, 6), (2, 7), (3, 5), (3, 6), (3, 7), (3, 8), (3, 9)):
             import setuptools  # We have to import it first for the compiler to be found
             from distutils import msvc9compiler
 
             if sys.version_info[:2] in ((2, 6), (2, 7)):
                 vcvarsall = msvc9compiler.find_vcvarsall(9.0)
-            elif sys.version_info[:2] in ((3, 5), (3, 6), (3, 7), (3, 8)):
+            elif sys.version_info[:2] in ((3, 5), (3, 6), (3, 7), (3, 8), (3, 9)):
                 vcvarsall = msvc9compiler.find_vcvarsall(14.0)
             if vcvarsall is None or not os.path.exists(vcvarsall):
                 raise RuntimeError('Error finding vcvarsall.')

src/debugpy/_vendored/pydevd/setup_cython.py

@@ -41,9 +41,61 @@ def process_args():
     return extension_folder, target_pydevd_name, target_frame_eval, force_cython
 
 
-def build_extension(dir_name, extension_name, target_pydevd_name, force_cython, extended=False, has_pxd=False):
+def process_template_lines(template_lines):
+    # Create 2 versions of the template, one for Python 3.8 and another for Python 3.9
+    for version in ('38', '39'):
+        yield '### WARNING: GENERATED CODE, DO NOT EDIT!'
+        yield '### WARNING: GENERATED CODE, DO NOT EDIT!'
+        yield '### WARNING: GENERATED CODE, DO NOT EDIT!'
+
+        for line in template_lines:
+            if version == '38':
+                line = line.replace('get_bytecode_while_frame_eval(PyFrameObject * frame_obj, int exc)', 'get_bytecode_while_frame_eval_38(PyFrameObject * frame_obj, int exc)')
+                line = line.replace('CALL_EvalFrameDefault', 'CALL_EvalFrameDefault_38(frame_obj, exc)')
+            else:  # 3.9
+                line = line.replace('get_bytecode_while_frame_eval(PyFrameObject * frame_obj, int exc)', 'get_bytecode_while_frame_eval_39(PyThreadState* tstate, PyFrameObject * frame_obj, int exc)')
+                line = line.replace('CALL_EvalFrameDefault', 'CALL_EvalFrameDefault_39(tstate, frame_obj, exc)')
+
+            yield line
+
+        yield '### WARNING: GENERATED CODE, DO NOT EDIT!'
+        yield '### WARNING: GENERATED CODE, DO NOT EDIT!'
+        yield '### WARNING: GENERATED CODE, DO NOT EDIT!'
+        yield ''
+        yield ''
+
+
+def process_template_file(contents):
+    ret = []
+    template_lines = []
+
+    append_to = ret
+    for line in contents.splitlines(keepends=False):
+        if line.strip() == '### TEMPLATE_START':
+            append_to = template_lines
+        elif line.strip() == '### TEMPLATE_END':
+            append_to = ret
+            for line in process_template_lines(template_lines):
+                ret.append(line)
+        else:
+            append_to.append(line)
+
+    return '\n'.join(ret)
+
+
+def build_extension(dir_name, extension_name, target_pydevd_name, force_cython, extended=False, has_pxd=False, template=False):
     pyx_file = os.path.join(os.path.dirname(__file__), dir_name, "%s.pyx" % (extension_name,))
 
+    if template:
+        pyx_template_file = os.path.join(os.path.dirname(__file__), dir_name, "%s.template.pyx" % (extension_name,))
+        with open(pyx_template_file, 'r') as stream:
+            contents = stream.read()
+
+        contents = process_template_file(contents)
+
+        with open(pyx_file, 'w') as stream:
+            stream.write(contents)
+
     if target_pydevd_name != extension_name:
         # It MUST be there in this case!
         # (otherwise we'll have unresolved externals because the .c file had another name initially).
@@ -73,17 +125,23 @@ def build_extension(dir_name, extension_name, target_pydevd_name, force_cython,
                     pass
             from Cython.Build import cythonize  # @UnusedImport
             # Generate the .c files in cythonize (will not compile at this point).
-            cythonize([
-                "%s/%s.pyx" % (dir_name, target_pydevd_name,),
-            ])
 
-            # This is needed in CPython 3.8 to access PyInterpreterState.eval_frame.
-            # i.e.: we change #include "pystate.h" to also #include "internal/pycore_pystate.h"
-            # if compiling on Python 3.8.
+            target = "%s/%s.pyx" % (dir_name, target_pydevd_name,)
+            cythonize([target])
+
+            # Hacks needed in CPython 3.8 and 3.9 to access PyInterpreterState.eval_frame.
             for c_file in c_files:
                 with open(c_file, 'r') as stream:
                     c_file_contents = stream.read()
 
+                if '#include "internal/pycore_gc.h"' not in c_file_contents:
+                    c_file_contents = c_file_contents.replace('#include "Python.h"', '''#include "Python.h"
+#if PY_VERSION_HEX >= 0x03090000
+#include "internal/pycore_gc.h"
+#include "internal/pycore_interp.h"
+#endif
+''')
+
                 if '#include "internal/pycore_pystate.h"' not in c_file_contents:
                     c_file_contents = c_file_contents.replace('#include "pystate.h"', '''#include "pystate.h"
 #if PY_VERSION_HEX >= 0x03080000
@@ -149,7 +207,7 @@ def build_extension(dir_name, extension_name, target_pydevd_name, force_cython,
     extension_name = "pydevd_frame_evaluator"
     if target_frame_eval is None:
         target_frame_eval = extension_name
-    build_extension("_pydevd_frame_eval", extension_name, target_frame_eval, force_cython, extension_folder, True)
+    build_extension("_pydevd_frame_eval", extension_name, target_frame_eval, force_cython, extension_folder, True, template=True)
 
 if extension_folder:
     os.chdir(extension_folder)