Show full stack trace on exception with __cause__ or __context__. Fixes

microsoft#391

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_comm.py

@@ -87,6 +87,7 @@
 from _pydevd_bundle.pydevd_thread_lifecycle import pydevd_find_thread_by_id, resume_threads
 from _pydevd_bundle.pydevd_dont_trace_files import PYDEV_FILE
 import dis
+from _pydevd_bundle.pydevd_frame_utils import create_frames_list_from_exception_cause
 try:
     from urllib import quote_plus, unquote_plus  # @UnresolvedImport
 except:
@@ -1196,33 +1197,59 @@ def build_exception_info_response(dbg, thread_id, request_seq, set_additional_th
     additional_info = set_additional_thread_info(thread)
     topmost_frame = additional_info.get_topmost_frame(thread)
 
-    frames = []
+    frames_lst_container = []
     exc_type = None
     exc_desc = None
     current_paused_frame_name = ''
+
+    source_path = ''  # This is an extra bit of data used by Visual Studio
+    stack_str_lst = []
+
     if topmost_frame is not None:
         try:
-            frames_list = dbg.suspended_frames_manager.get_frames_list(thread_id)
-            if frames_list is not None:
-                exc_type = frames_list.exc_type
-                exc_desc = frames_list.exc_desc
-                trace_obj = frames_list.trace_obj
-                for frame_id, frame, method_name, original_filename, filename_in_utf8, lineno, _applied_mapping, show_as_current_frame in \
-                    iter_visible_frames_info(dbg, frames_list):
-
-                    line_text = linecache.getline(original_filename, lineno)
-
-                    # Never filter out plugin frames!
-                    if not getattr(frame, 'IS_PLUGIN_FRAME', False):
-                        if dbg.is_files_filter_enabled and dbg.apply_files_filter(frame, original_filename, False):
-                            continue
+            try:
+                frames_list = dbg.suspended_frames_manager.get_frames_list(thread_id)
+                memo = set()
+                while frames_list is not None and len(frames_list):
+                    frames = []
+                    frames_lst_container.append(frames)
+
+                    exc_type = frames_list.exc_type
+                    exc_desc = frames_list.exc_desc
+                    trace_obj = frames_list.trace_obj
+                    frame = None
+
+                    for frame_id, frame, method_name, original_filename, filename_in_utf8, lineno, _applied_mapping, show_as_current_frame in \
+                        iter_visible_frames_info(dbg, frames_list):
+
+                        line_text = linecache.getline(original_filename, lineno)
+
+                        # Never filter out plugin frames!
+                        if not getattr(frame, 'IS_PLUGIN_FRAME', False):
+                            if dbg.is_files_filter_enabled and dbg.apply_files_filter(frame, original_filename, False):
+                                continue
+
+                        if show_as_current_frame:
+                            current_paused_frame_name = method_name
+                            method_name += ' (Current frame)'
+                        frames.append((filename_in_utf8, lineno, method_name, line_text))
+
+                    if not source_path and frames:
+                        source_path = frames[0][0]
+
+                    stack_str = ''.join(traceback.format_list(frames[-max_frames:]))
+                    stack_str += frames_list.exc_context_msg
+                    stack_str_lst.append(stack_str)
 
-                    if show_as_current_frame:
-                        current_paused_frame_name = method_name
-                        method_name += ' (Current frame)'
-                    frames.append((filename_in_utf8, lineno, method_name, line_text))
+                    frames_list = create_frames_list_from_exception_cause(trace_obj, None, exc_type, exc_desc, memo)
+                    if frames_list is None or not frames_list:
+                        break
+
+            except:
+                pydev_log.exception('Error on build_exception_info_response.')
         finally:
             topmost_frame = None
+    full_stack_str = ''.join(reversed(stack_str_lst))
 
     name = 'exception: type unknown'
     if exc_type is not None:
@@ -1247,11 +1274,6 @@ def build_exception_info_response(dbg, thread_id, request_seq, set_additional_th
     if current_paused_frame_name:
         name += '       (note: full exception trace is shown but execution is paused at: %s)' % (current_paused_frame_name,)
 
-    stack_str = ''.join(traceback.format_list(frames[-max_frames:]))
-
-    # This is an extra bit of data used by Visual Studio
-    source_path = frames[0][0] if frames else ''
-
     if thread.stop_reason == CMD_STEP_CAUGHT_EXCEPTION:
         break_mode = pydevd_schema.ExceptionBreakMode.ALWAYS
     else:
@@ -1268,8 +1290,10 @@ def build_exception_info_response(dbg, thread_id, request_seq, set_additional_th
             details=pydevd_schema.ExceptionDetails(
                 message=description,
                 typeName=name,
-                stackTrace=stack_str,
-                source=source_path
+                stackTrace=full_stack_str,
+                source=source_path,
+                # Note: ExceptionDetails actually accepts an 'innerException', but
+                # when passing it, VSCode is not showing the stack trace at all.
             )
         )
     )

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_frame.py

@@ -414,9 +414,11 @@ def _handle_exception(self, frame, event, arg, exception_type):
 
                 stopped = True
                 main_debugger.send_caught_exception_stack(thread, arg, id(frame))
-                self.set_suspend(thread, CMD_STEP_CAUGHT_EXCEPTION)
-                self.do_wait_suspend(thread, frame, event, arg, exception_type=exception_type)
-                main_debugger.send_caught_exception_stack_proceeded(thread)
+                try:
+                    self.set_suspend(thread, CMD_STEP_CAUGHT_EXCEPTION)
+                    self.do_wait_suspend(thread, frame, event, arg, exception_type=exception_type)
+                finally:
+                    main_debugger.send_caught_exception_stack_proceeded(thread)
             except:
                 pydev_log.exception()
 

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_frame_utils.py

@@ -1,7 +1,5 @@
-from _pydevd_bundle.pydevd_constants import IS_PY3K, EXCEPTION_TYPE_USER_UNHANDLED, \
-    EXCEPTION_TYPE_UNHANDLED
+from _pydevd_bundle.pydevd_constants import EXCEPTION_TYPE_USER_UNHANDLED, EXCEPTION_TYPE_UNHANDLED
 from _pydev_bundle import pydev_log
-import sys
 
 
 class Frame(object):
@@ -95,6 +93,9 @@ def __init__(self):
         # executing frame).
         self.current_frame = None
 
+        # This is to know whether an exception was extracted from a __cause__ or __context__.
+        self.exc_context_msg = ''
+
     def append(self, frame):
         self._frames.append(frame)
 
@@ -132,6 +133,77 @@ def __repr__(self):
     __str__ = __repr__
 
 
+class _DummyFrameWrapper(object):
+
+    def __init__(self, frame, f_lineno, f_back):
+        self._base_frame = frame
+        self.f_lineno = f_lineno
+        self.f_back = f_back
+        self.f_trace = None
+        original_code = frame.f_code
+        self.f_code = FCode(original_code.co_name , original_code.co_filename)
+
+    @property
+    def f_locals(self):
+        return self._base_frame.f_locals
+
+    @property
+    def f_globals(self):
+        return self._base_frame.f_globals
+
+
+_cause_message = (
+    "\nThe above exception was the direct cause "
+    "of the following exception:\n\n")
+
+_context_message = (
+    "\nDuring handling of the above exception, "
+    "another exception occurred:\n\n")
+
+
+def create_frames_list_from_exception_cause(trace_obj, frame, exc_type, exc_desc, memo):
+    lst = []
+    msg = '<Unknown context>'
+    try:
+        exc_cause = getattr(exc_desc, '__cause__', None)
+        msg = _cause_message
+    except Exception:
+        exc_cause = None
+
+    if exc_cause is None:
+        try:
+            exc_cause = getattr(exc_desc, '__context__', None)
+            msg = _context_message
+        except Exception:
+            exc_cause = None
+
+    if exc_cause is None or id(exc_cause) in memo:
+        return None
+
+    # The traceback module does this, so, let's play safe here too...
+    memo.add(id(exc_cause))
+
+    tb = exc_cause.__traceback__
+    frames_list = FramesList()
+    frames_list.exc_type = type(exc_cause)
+    frames_list.exc_desc = exc_cause
+    frames_list.trace_obj = tb
+    frames_list.exc_context_msg = msg
+
+    while tb is not None:
+        # Note: we don't use the actual tb.tb_frame because if the cause of the exception
+        # uses the same frame object, the id(frame) would be the same and the frame_id_to_lineno
+        # would be wrong as the same frame needs to appear with 2 different lines.
+        lst.append((_DummyFrameWrapper(tb.tb_frame, tb.tb_lineno, None), tb.tb_lineno))
+        tb = tb.tb_next
+
+    for tb_frame, tb_lineno in lst:
+        frames_list.append(tb_frame)
+        frames_list.frame_id_to_lineno[id(tb_frame)] = tb_lineno
+
+    return frames_list
+
+
 def create_frames_list_from_traceback(trace_obj, frame, exc_type, exc_desc, exception_type=None):
     '''
     :param trace_obj:
@@ -158,6 +230,36 @@ def create_frames_list_from_traceback(trace_obj, frame, exc_type, exc_desc, exce
         lst.append((tb.tb_frame, tb.tb_lineno))
         tb = tb.tb_next
 
+    curr = exc_desc
+    memo = set()
+    while True:
+        initial = curr
+        try:
+            curr = getattr(initial, '__cause__', None)
+        except Exception:
+            curr = None
+
+        if curr is None:
+            try:
+                curr = getattr(initial, '__context__', None)
+            except Exception:
+                curr = None
+
+        if curr is None or id(curr) in memo:
+            break
+
+        # The traceback module does this, so, let's play safe here too...
+        memo.add(id(curr))
+
+        tb = getattr(curr, '__traceback__', None)
+
+        while tb is not None:
+            # Note: we don't use the actual tb.tb_frame because if the cause of the exception
+            # uses the same frame object, the id(frame) would be the same and the frame_id_to_lineno
+            # would be wrong as the same frame needs to appear with 2 different lines.
+            lst.append((_DummyFrameWrapper(tb.tb_frame, tb.tb_lineno, None), tb.tb_lineno))
+            tb = tb.tb_next
+
     frames_list = None
 
     for tb_frame, tb_lineno in reversed(lst):

src/debugpy/_vendored/pydevd/pydevd.py

@@ -1990,8 +1990,12 @@ def do_stop_on_unhandled_exception(self, thread, frame, frames_byid, arg):
         pydev_log.debug("We are stopping in unhandled exception.")
         try:
             add_exception_to_frame(frame, arg)
-            self.set_suspend(thread, CMD_ADD_EXCEPTION_BREAK)
-            self.do_wait_suspend(thread, frame, 'exception', arg, EXCEPTION_TYPE_UNHANDLED)
+            self.send_caught_exception_stack(thread, arg, id(frame))
+            try:
+                self.set_suspend(thread, CMD_ADD_EXCEPTION_BREAK)
+                self.do_wait_suspend(thread, frame, 'exception', arg, EXCEPTION_TYPE_UNHANDLED)
+            except:
+                self.send_caught_exception_stack_proceeded(thread)
         except:
             pydev_log.exception("We've got an error while stopping in unhandled exception: %s.", arg[0])
         finally:

src/debugpy/_vendored/pydevd/tests_python/debugger_unittest.py

@@ -1323,6 +1323,9 @@ def wait_for_list_threads(self, seq):
     def wait_for_get_thread_stack_message(self):
         return self.wait_for_message(CMD_GET_THREAD_STACK)
 
+    def wait_for_curr_exc_stack(self):
+        return self.wait_for_message(CMD_SEND_CURR_EXCEPTION_TRACE)
+
     def wait_for_json_message(self, accept_message, unquote_msg=True, timeout=None):
         last = self.wait_for_message(accept_message, unquote_msg, expect_xml=False, timeout=timeout)
         json_msg = last.split('\t', 2)[-1]  # We have something as: CMD\tSEQ\tJSON

src/debugpy/_vendored/pydevd/tests_python/resources/_debugger_case_raise_with_cause.py

@@ -0,0 +1,23 @@
+def method2():
+    raise RuntimeError('TEST SUCEEDED')
+
+
+def method():
+    method2()
+
+
+def handle(e):
+    raise Exception('another while handling')
+
+
+def foobar():
+    try:
+        try:
+            method()
+        except Exception as e:
+            handle(e)
+    except Exception as e:
+        raise RuntimeError from e
+
+
+foobar()

src/debugpy/_vendored/pydevd/tests_python/test_debugger.py

@@ -1640,6 +1640,50 @@ def test_unhandled_exceptions_get_stack(case_setup_unhandled_exceptions):
         writer.finished_ok = True
 
 
+@pytest.mark.skipif(not IS_PY36_OR_GREATER, reason='Requires Python 3.')
+def test_case_throw_exc_reason_xml(case_setup):
+
+    def check_test_suceeded_msg(self, stdout, stderr):
+        return 'TEST SUCEEDED' in ''.join(stderr)
+
+    def additional_output_checks(writer, stdout, stderr):
+        assert "raise RuntimeError('TEST SUCEEDED')" in stderr
+        assert "raise RuntimeError from e" in stderr
+        assert "raise Exception('another while handling')" in stderr
+
+    with case_setup.test_file(
+            '_debugger_case_raise_with_cause.py',
+            EXPECTED_RETURNCODE=1,
+            check_test_suceeded_msg=check_test_suceeded_msg,
+            additional_output_checks=additional_output_checks
+        ) as writer:
+
+        writer.write_add_exception_breakpoint_with_policy('Exception', "0", "1", "0")
+        writer.write_make_initial_run()
+
+        el = writer.wait_for_curr_exc_stack()
+        name_and_lines = []
+        for frame in el.thread.frame:
+            name_and_lines.append((frame['name'], frame['line']))
+
+        assert name_and_lines == [
+            ('method2', '2'),
+            ('method', '6'),
+            ('foobar', '16'),
+            ('handle', '10'),
+            ('foobar', '18'),
+            ('foobar', '20'),
+            ('<module>', '23'),
+        ]
+
+        hit = writer.wait_for_breakpoint_hit(REASON_UNCAUGHT_EXCEPTION)
+        writer.write_get_thread_stack(hit.thread_id)
+
+        writer.write_run_thread(hit.thread_id)
+
+        writer.finished_ok = True
+
+
 @pytest.mark.skipif(not IS_CPYTHON, reason='Only for Python.')
 def test_case_get_next_statement_targets(case_setup):
     with case_setup.test_file('_debugger_case_get_next_statement_targets.py') as writer: