Native thread tracing in frame eval mode. Fixes #444

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_cython.pyx

@@ -1323,6 +1323,9 @@ def fix_top_level_trace_and_get_trace_func(py_db, frame):
                 force_only_unhandled_tracer = True
                 break
 
+        elif name == 'pydevd_tracing':
+            return None, False
+
         elif f_unhandled.f_back is None:
             break
 

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_process_net_command.py

@@ -446,7 +446,7 @@ def cmd_set_property_trace(self, py_db, cmd_id, seq, text):
         if text:
             splitted = text.split(';')
             if len(splitted) >= 3:
-                if py_db.disable_property_trace is False and splitted[0] == 'true':
+                if not py_db.disable_property_trace and splitted[0] == 'true':
                     # Replacing property by custom property only when the debugger starts
                     pydevd_traceproperty.replace_builtin_property()
                     py_db.disable_property_trace = True

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_trace_dispatch_regular.py

@@ -128,6 +128,9 @@ def fix_top_level_trace_and_get_trace_func(py_db, frame):
                 force_only_unhandled_tracer = True
                 break
 
+        elif name == 'pydevd_tracing':
+            return None, False
+
         elif f_unhandled.f_back is None:
             break
 

src/debugpy/_vendored/pydevd/_pydevd_frame_eval/pydevd_frame_evaluator.pxd

@@ -26,6 +26,7 @@ cdef extern from *:
 
 cdef extern from "frameobject.h":
     ctypedef struct PyFrameObject:
+        PyFrameObject *f_back
         PyCodeObject *f_code       # code segment
         PyObject *f_builtins       # builtin symbol table (PyDictObject)
         PyObject *f_globals        # global symbol table (PyDictObject) */
@@ -45,7 +46,7 @@ cdef extern from "frameobject.h":
         PyObject *f_localsplus[1];
 
 cdef extern from "release_mem.h":
-    void release_co_extra(void *) 
+    void release_co_extra(void *)
 
 cdef extern from "code.h":
     ctypedef void freefunc(void *)

src/debugpy/_vendored/pydevd/_pydevd_frame_eval/pydevd_frame_evaluator.pyx

@@ -9,6 +9,7 @@ from _pydevd_bundle.pydevd_trace_dispatch import fix_top_level_trace_and_get_tra
 
 from _pydevd_bundle.pydevd_additional_thread_info import _set_additional_thread_info_lock
 from _pydevd_bundle.pydevd_cython cimport PyDBAdditionalThreadInfo
+from pydevd_tracing import SetTrace
 
 _get_ident = threading.get_ident  # Note this is py3 only, if py2 needed to be supported, _get_ident would be needed.
 _thread_local_info = threading.local()
@@ -26,6 +27,7 @@ cdef class ThreadInfo:
     cdef public int inside_frame_eval
     cdef public bint fully_initialized
     cdef public object thread_trace_func
+    cdef bint _can_create_dummy_thread
 
     # Note: whenever get_func_code_info is called, this value is reset (we're using
     # it as a thread-local value info).
@@ -34,13 +36,56 @@ cdef class ThreadInfo:
     cdef public bint force_stay_in_untraced_mode
 
     def __init__(self):
+        cdef PyFrameObject * frame_obj
+        frame_obj = PyEval_GetFrame()
+
+        # Places that create a ThreadInfo should verify that
+        # a current Python frame is being executed!
+        assert frame_obj != NULL
+
         self.additional_info = None
         self.is_pydevd_thread = False
         self.inside_frame_eval = 0
         self.fully_initialized = False
         self.thread_trace_func = None
 
-    def initialize_if_possible(self):
+        # Get the root (if it's not a Thread initialized from the threading
+        # module, create the dummy thread entry so that we can debug it --
+        # otherwise, we have to wait for the threading module itself to
+        # create the Thread entry).
+        while frame_obj.f_back != NULL:
+            frame_obj = frame_obj.f_back
+
+        basename = <str> frame_obj.f_code.co_filename
+        i = basename.rfind('/')
+        j = basename.rfind('\\')
+        if j > i:
+            i = j
+        if i >= 0:
+            basename = basename[i + 1:]
+        # remove ext
+        i = basename.rfind('.')
+        if i >= 0:
+            basename = basename[:i]
+
+        co_name = <str> frame_obj.f_code.co_name
+
+        # In these cases we cannot create a dummy thread (an actual
+        # thread will be created later or tracing will already be set).
+        if basename == 'threading' and co_name in ('__bootstrap', '_bootstrap', '__bootstrap_inner', '_bootstrap_inner'):
+            self._can_create_dummy_thread = False
+        elif basename == 'pydev_monkey' and co_name == '__call__':
+            self._can_create_dummy_thread = False
+        elif basename == 'pydevd' and co_name in ('run', 'main', '_exec'):
+            self._can_create_dummy_thread = False
+        elif basename == 'pydevd_tracing':
+            self._can_create_dummy_thread = False
+        else:
+            self._can_create_dummy_thread = True
+
+        # print('Can create dummy thread for thread started in: %s %s' % (basename, co_name))
+
+    cdef initialize_if_possible(self):
         # Don't call threading.currentThread because if we're too early in the process
         # we may create a dummy thread.
         self.inside_frame_eval += 1
@@ -49,7 +94,13 @@ cdef class ThreadInfo:
             thread_ident = _get_ident()
             t = _thread_active.get(thread_ident)
             if t is None:
-                return  # Cannot initialize until thread becomes active.
+                if self._can_create_dummy_thread:
+                    # Initialize the dummy thread and set the tracing (both are needed to
+                    # actually stop on breakpoints).
+                    t = threading.current_thread()
+                    SetTrace(dummy_trace_dispatch)
+                else:
+                    return  # Cannot initialize until thread becomes active.
 
             if getattr(t, 'is_pydev_daemon_thread', False):
                 self.is_pydevd_thread = True
@@ -117,11 +168,15 @@ cdef ThreadInfo get_thread_info():
     May return None if the thread is still not active.
     '''
     cdef ThreadInfo thread_info
+    cdef PyFrameObject * frame_obj
     try:
         # Note: changing to a `dict[thread.ident] = thread_info` had almost no
         # effect in the performance.
         thread_info = _thread_local_info.thread_info
     except:
+        frame_obj = PyEval_GetFrame()
+        if frame_obj == NULL:
+            return None
         thread_info = ThreadInfo()
         thread_info.inside_frame_eval += 1
         try:
@@ -315,12 +370,16 @@ cdef class _CacheValue(object):
         self.breakpoints_hit_at_lines = breakpoints_hit_at_lines
         self.code_lines_as_set = set(code_line_info.line_to_offset)
 
-    def compute_force_stay_in_untraced_mode(self, breakpoints):
+    cpdef compute_force_stay_in_untraced_mode(self, breakpoints):
         '''
         :param breakpoints:
             set(breakpoint_lines) or dict(breakpoint_line->breakpoint info)
         :return tuple(breakpoint_found, force_stay_in_untraced_mode)
         '''
+        cdef bint force_stay_in_untraced_mode
+        cdef bint breakpoint_found
+        cdef set target_breakpoints
+
         force_stay_in_untraced_mode = False
 
         target_breakpoints = self.code_lines_as_set.intersection(breakpoints)

src/debugpy/_vendored/pydevd/appveyor.yml

@@ -28,21 +28,14 @@ environment:
     - PYTHON_FOLDER: "C:\\Python35-x64"
       PYDEVD_USE_CYTHON: YES
       TRIO: 1
-#     - PYTHON_FOLDER: "C:\\Python35-x64"
-#       PYDEVD_USE_CYTHON: NO
 
-#     - PYTHON_FOLDER: "C:\\Python36-x64"
-#       PYDEVD_USE_CYTHON: YES
-    - PYTHON_FOLDER: "C:\\Python36-x64"
-      PYDEVD_USE_CYTHON: NO
+    - PYTHON_FOLDER: "C:\\Python38-x64"
+      PYDEVD_USE_CYTHON: YES
       TRIO: 1
 
-    - PYTHON_FOLDER: "C:\\Python37-x64"
+    - PYTHON_FOLDER: "C:\\Python38-x64"
       PYDEVD_USE_CYTHON: YES
       TRIO: 1
-    - PYTHON_FOLDER: "C:\\Python37-x64"
-      PYDEVD_USE_CYTHON: NO
-      TRIO: 1
 
     # Disable IronPython tests (must be investigated in appveyor).
     #- PYTHON_FOLDER: "C:\\Python36-x64"
@@ -70,8 +63,8 @@ install:
   - cmd: IF "%TEST_IRONPYTHON%"=="" (%PYTHON_EXE% -m pip install ipython --no-warn-script-location)
   - cmd: IF "%TEST_IRONPYTHON%"=="" (%PYTHON_EXE% -m pip install untangle --no-warn-script-location)
   - cmd: IF "%TEST_IRONPYTHON%"=="" IF %PYTHON_FOLDER%=="C:\\Python27" (%PYTHON_EXE% -m pip install django>=1.7,<1.8)
-  - cmd: IF "%TEST_IRONPYTHON%"=="" IF %PYTHON_FOLDER%=="C:\\Python37-x64" (%PYTHON_EXE% -m pip install cherrypy)
-  - cmd: IF "%TEST_IRONPYTHON%"=="" IF %PYTHON_FOLDER%=="C:\\Python37-x64" (%PYTHON_EXE% -m pip install trio)
+  - cmd: IF "%TEST_IRONPYTHON%"=="" IF %PYTHON_FOLDER%=="C:\\Python38-x64" (%PYTHON_EXE% -m pip install cherrypy)
+  - cmd: IF "%TEST_IRONPYTHON%"=="" IF %PYTHON_FOLDER%=="C:\\Python38-x64" (%PYTHON_EXE% -m pip install trio)
   - cmd: IF "%TEST_IRONPYTHON%"=="" (%PYTHON_EXE% -m pip install scapy==2.4.0 --no-warn-script-location)
   - cmd: "set PYTHONPATH=%PYTHONPATH%;%APPVEYOR_BUILD_FOLDER%"