From 958af7e2f71fbdda5f8e06a108ca269935b1ab2d Mon Sep 17 00:00:00 2001 From: Fabio Zadrozny Date: Fri, 22 May 2020 14:49:41 -0300 Subject: [PATCH] Don't crash if running under CPython debug build. Fixes #152 --- src/debugpy/_vendored/pydevd/.travis.yml | 9 +-- .../.travis/install_and_run_debug_py.sh | 35 +++++++++++ .../pydevd_additional_thread_info.py | 14 ++--- .../pydevd/_pydevd_bundle/pydevd_constants.py | 42 +++++++++++-- .../_pydevd_bundle/pydevd_trace_dispatch.py | 16 ++--- .../pydevd_frame_eval_main.py | 17 ++---- .../_vendored/pydevd/build_tools/build.py | 9 +-- .../pydevd_attach_to_process/attach_script.py | 60 ------------------- .../_vendored/pydevd/pydevd_tracing.py | 9 ++- .../pydevd/tests_python/check_debug_python.py | 38 ++++++++++++ 10 files changed, 143 insertions(+), 106 deletions(-) create mode 100644 src/debugpy/_vendored/pydevd/.travis/install_and_run_debug_py.sh create mode 100644 src/debugpy/_vendored/pydevd/tests_python/check_debug_python.py diff --git a/src/debugpy/_vendored/pydevd/.travis.yml b/src/debugpy/_vendored/pydevd/.travis.yml index d6055b08a..7c93e7523 100644 --- a/src/debugpy/_vendored/pydevd/.travis.yml +++ b/src/debugpy/_vendored/pydevd/.travis.yml @@ -57,12 +57,12 @@ matrix: - PYDEVD_USE_CYTHON=NO - PYDEVD_TEST_VM=CPYTHON - # Python 3.7 + # Python 3.8 debug - python: 2.7 env: - - PYDEVD_PYTHON_VERSION=3.7 - - PYDEVD_USE_CYTHON=YES - - PYDEVD_TEST_VM=CPYTHON + - PYDEVD_PYTHON_VERSION=3.8 + - PYDEVD_TEST_VM=CPYTHON_DEBUG + - PYDEVD_USE_CONDA=NO - python: 3.8 env: @@ -124,6 +124,7 @@ install: # On local machine with jython: c:\bin\jython2.7.0\bin\jython.exe -Dpython.path=.;jython_test_deps/ant.jar;jython_test_deps/junit.jar -m pytest # On remove machine with python: c:\bin\python27\python.exe -m pytest script: + - if [[ ("$PYDEVD_TEST_VM" == "CPYTHON_DEBUG") ]]; then ./.travis/install_and_run_debug_py.sh; fi - if [[ ("$PYDEVD_TEST_VM" == "CPYTHON") ]]; then ./.travis/run_python_pytest.sh; fi - if [ "$PYDEVD_TEST_VM" == "PYPY" ]; then source activate build_env; pypy3 -m pytest -n auto; fi - if [ "$PYDEVD_TEST_VM" == "JYTHON" ]; then jython -Dpython.path=.:jython_test_deps/ant.jar:jython_test_deps/junit.jar -m pytest --tb=native; fi diff --git a/src/debugpy/_vendored/pydevd/.travis/install_and_run_debug_py.sh b/src/debugpy/_vendored/pydevd/.travis/install_and_run_debug_py.sh new file mode 100644 index 000000000..659bdb051 --- /dev/null +++ b/src/debugpy/_vendored/pydevd/.travis/install_and_run_debug_py.sh @@ -0,0 +1,35 @@ +# Build the cython extensions (to check that we don't crash when they're there in debug mode). +python setup_cython.py build_ext --inplace + +curl -L https://www.python.org/ftp/python/3.8.3/Python-3.8.3.tgz -o Python-3.8.3.tgz +tar -xzf Python-3.8.3.tgz +cd Python-3.8.3 +mkdir debug +cd debug +../configure --with-pydebug +make + +curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py +./python get-pip.py + +./python -m pip install "pytest" +./python -m pip install "psutil" +./python -m pip install "untangle" + +# Check that it worked. +./python -c "import pytest" +./python -c "import psutil" +./python -c "import untangle" + +cd .. +cd .. +ls -la + +./Python-3.8.3/debug/python -c "import sys;assert hasattr(sys,'gettotalrefcount')" + +cd tests_python + +# Although we compiled cython, all we're checking is that we don't crash (since it was built for the release env). +../Python-3.8.3/debug/python -m pytest test_debugger_json.py -k "test_case_json_change_breaks or test_remote_debugger_basic" +export PYTHONPATH=.. +../Python-3.8.3/debug/python -c "import check_debug_python;check_debug_python.check() " diff --git a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_additional_thread_info.py b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_additional_thread_info.py index 5e0d7dd77..ac866735b 100644 --- a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_additional_thread_info.py +++ b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_additional_thread_info.py @@ -1,23 +1,19 @@ # Defines which version of the PyDBAdditionalThreadInfo we'll use. +from _pydevd_bundle.pydevd_constants import ENV_FALSE_LOWER_VALUES, USE_CYTHON_FLAG, \ + ENV_TRUE_LOWER_VALUES -import os -use_cython = os.getenv('PYDEVD_USE_CYTHON', None) - -if use_cython == 'YES': +if USE_CYTHON_FLAG in ENV_TRUE_LOWER_VALUES: # We must import the cython version if forcing cython from _pydevd_bundle.pydevd_cython_wrapper import PyDBAdditionalThreadInfo, set_additional_thread_info, _set_additional_thread_info_lock # @UnusedImport -elif use_cython == 'NO': +elif USE_CYTHON_FLAG in ENV_FALSE_LOWER_VALUES: # Use the regular version if not forcing cython from _pydevd_bundle.pydevd_additional_thread_info_regular import PyDBAdditionalThreadInfo, set_additional_thread_info, _set_additional_thread_info_lock # @UnusedImport @Reimport -elif use_cython is None: +else: # Regular: use fallback if not found (message is already given elsewhere). try: from _pydevd_bundle.pydevd_cython_wrapper import PyDBAdditionalThreadInfo, set_additional_thread_info, _set_additional_thread_info_lock except ImportError: from _pydevd_bundle.pydevd_additional_thread_info_regular import PyDBAdditionalThreadInfo, set_additional_thread_info, _set_additional_thread_info_lock # @UnusedImport -else: - raise RuntimeError('Unexpected value for PYDEVD_USE_CYTHON: %s (accepted: YES, NO)' % (use_cython,)) - diff --git a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_constants.py b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_constants.py index 49d3879d6..13e6107be 100644 --- a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_constants.py +++ b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_constants.py @@ -173,7 +173,23 @@ def version_str(v): except AttributeError: PY_IMPL_NAME = '' -SUPPORT_GEVENT = os.getenv('GEVENT_SUPPORT', 'False') in ('True', 'true', '1') +ENV_TRUE_LOWER_VALUES = ('yes', 'true', '1') +ENV_FALSE_LOWER_VALUES = ('no', 'false', '0') + + +def is_true_in_env(env_key): + if isinstance(env_key, tuple): + # If a tuple, return True if any of those ends up being true. + for v in env_key: + if is_true_in_env(v): + return True + return False + else: + return os.getenv(env_key, '').lower() in ENV_TRUE_LOWER_VALUES + + +# If true in env, use gevent mode. +SUPPORT_GEVENT = is_true_in_env('GEVENT_SUPPORT') GEVENT_SUPPORT_NOT_SET_MSG = os.getenv( 'GEVENT_SUPPORT_NOT_SET_MSG', @@ -187,14 +203,32 @@ def version_str(v): INTERACTIVE_MODE_AVAILABLE = sys.platform in ('darwin', 'win32') or os.getenv('DISPLAY') is not None -SHOW_COMPILE_CYTHON_COMMAND_LINE = os.getenv('PYDEVD_SHOW_COMPILE_CYTHON_COMMAND_LINE', 'False') == 'True' +# If true in env, forces cython to be used (raises error if not available). +# If false in env, disables it. +# If not specified, uses default heuristic to determine if it should be loaded. +USE_CYTHON_FLAG = os.getenv('PYDEVD_USE_CYTHON') + +# Use to disable loading the lib to set tracing to all threads (default is using heuristics based on where we're running). +LOAD_NATIVE_LIB_FLAG = os.getenv('PYDEVD_LOAD_NATIVE_LIB', '').lower() + +if USE_CYTHON_FLAG is not None: + USE_CYTHON_FLAG = USE_CYTHON_FLAG.lower() + if USE_CYTHON_FLAG not in ENV_TRUE_LOWER_VALUES and USE_CYTHON_FLAG not in ENV_FALSE_LOWER_VALUES: + raise RuntimeError('Unexpected value for PYDEVD_USE_CYTHON: %s (enable with one of: %s, disable with one of: %s)' % ( + USE_CYTHON_FLAG, ENV_TRUE_LOWER_VALUES, ENV_FALSE_LOWER_VALUES)) + +else: + if not CYTHON_SUPPORTED: + USE_CYTHON_FLAG = 'no' + +SHOW_COMPILE_CYTHON_COMMAND_LINE = is_true_in_env('PYDEVD_SHOW_COMPILE_CYTHON_COMMAND_LINE') -LOAD_VALUES_ASYNC = os.getenv('PYDEVD_LOAD_VALUES_ASYNC', 'False') == 'True' +LOAD_VALUES_ASYNC = is_true_in_env('PYDEVD_LOAD_VALUES_ASYNC') DEFAULT_VALUE = "__pydevd_value_async" ASYNC_EVAL_TIMEOUT_SEC = 60 NEXT_VALUE_SEPARATOR = "__pydev_val__" BUILTINS_MODULE_NAME = '__builtin__' if IS_PY2 else 'builtins' -SHOW_DEBUG_INFO_ENV = os.getenv('PYCHARM_DEBUG') == 'True' or os.getenv('PYDEV_DEBUG') == 'True' or os.getenv('PYDEVD_DEBUG') == 'True' +SHOW_DEBUG_INFO_ENV = is_true_in_env(('PYCHARM_DEBUG', 'PYDEV_DEBUG', 'PYDEVD_DEBUG')) if SHOW_DEBUG_INFO_ENV: # show debug info before the debugger start diff --git a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_trace_dispatch.py b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_trace_dispatch.py index c08742a47..40a683770 100644 --- a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_trace_dispatch.py +++ b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_trace_dispatch.py @@ -2,16 +2,12 @@ # Should give warning only here if cython is not available but supported. import os -from _pydevd_bundle.pydevd_constants import CYTHON_SUPPORTED +from _pydevd_bundle.pydevd_constants import USE_CYTHON_FLAG, ENV_TRUE_LOWER_VALUES, \ + ENV_FALSE_LOWER_VALUES from _pydev_bundle import pydev_log -use_cython = os.getenv('PYDEVD_USE_CYTHON', None) dirname = os.path.dirname(os.path.dirname(__file__)) USING_CYTHON = False -# Do not show incorrect warning for .egg files for Remote debugger -if not CYTHON_SUPPORTED or dirname.endswith('.egg'): - # Do not try to import cython extensions if cython isn't supported - use_cython = 'NO' def delete_old_compiled_extensions(): @@ -35,16 +31,16 @@ def delete_old_compiled_extensions(): "\"%s\" and \"%s\"" % (_pydevd_bundle_ext_dir, _pydevd_frame_eval_ext_dir)) -if use_cython == 'YES': +if USE_CYTHON_FLAG in ENV_TRUE_LOWER_VALUES: # We must import the cython version if forcing cython from _pydevd_bundle.pydevd_cython_wrapper import trace_dispatch, global_cache_skips, global_cache_frame_skips, fix_top_level_trace_and_get_trace_func USING_CYTHON = True -elif use_cython == 'NO': +elif USE_CYTHON_FLAG in ENV_FALSE_LOWER_VALUES: # Use the regular version if not forcing cython from _pydevd_bundle.pydevd_trace_dispatch_regular import trace_dispatch, global_cache_skips, global_cache_frame_skips, fix_top_level_trace_and_get_trace_func # @UnusedImport -elif use_cython is None: +else: # Regular: use fallback if not found and give message to user try: from _pydevd_bundle.pydevd_cython_wrapper import trace_dispatch, global_cache_skips, global_cache_frame_skips, fix_top_level_trace_and_get_trace_func @@ -63,6 +59,4 @@ def delete_old_compiled_extensions(): except ImportError: from _pydevd_bundle.pydevd_trace_dispatch_regular import trace_dispatch, global_cache_skips, global_cache_frame_skips, fix_top_level_trace_and_get_trace_func # @UnusedImport pydev_log.show_compile_cython_command_line() -else: - raise RuntimeError('Unexpected value for PYDEVD_USE_CYTHON: %s (accepted: YES, NO)' % (use_cython,)) diff --git a/src/debugpy/_vendored/pydevd/_pydevd_frame_eval/pydevd_frame_eval_main.py b/src/debugpy/_vendored/pydevd/_pydevd_frame_eval/pydevd_frame_eval_main.py index d7b321de6..a8c53ca22 100644 --- a/src/debugpy/_vendored/pydevd/_pydevd_frame_eval/pydevd_frame_eval_main.py +++ b/src/debugpy/_vendored/pydevd/_pydevd_frame_eval/pydevd_frame_eval_main.py @@ -1,31 +1,29 @@ import os -import sys from _pydev_bundle import pydev_log from _pydevd_bundle.pydevd_trace_dispatch import USING_CYTHON - -IS_PY36_OR_GREATER = sys.version_info >= (3, 6) +from _pydevd_bundle.pydevd_constants import USE_CYTHON_FLAG, ENV_FALSE_LOWER_VALUES, \ + ENV_TRUE_LOWER_VALUES, IS_PY36_OR_GREATER frame_eval_func = None stop_frame_eval = None dummy_trace_dispatch = None clear_thread_local_info = None -use_cython = os.getenv('PYDEVD_USE_CYTHON', None) USING_FRAME_EVAL = False # "NO" means we should not use frame evaluation, 'YES' we should use it (and fail if not there) and unspecified uses if possible. -use_frame_eval = os.environ.get('PYDEVD_USE_FRAME_EVAL', None) +use_frame_eval = os.environ.get('PYDEVD_USE_FRAME_EVAL', '').lower() -if use_frame_eval == 'NO' or use_cython == 'NO' or not USING_CYTHON: +if use_frame_eval in ENV_FALSE_LOWER_VALUES or USE_CYTHON_FLAG in ENV_FALSE_LOWER_VALUES or not USING_CYTHON: pass -elif use_frame_eval == 'YES': +elif use_frame_eval in ENV_TRUE_LOWER_VALUES: # Fail if unable to use from _pydevd_frame_eval.pydevd_frame_eval_cython_wrapper import frame_eval_func, stop_frame_eval, dummy_trace_dispatch, clear_thread_local_info USING_FRAME_EVAL = True -elif use_frame_eval is None: +else: # Try to use if possible if IS_PY36_OR_GREATER: try: @@ -33,6 +31,3 @@ USING_FRAME_EVAL = True except ImportError: pydev_log.show_compile_cython_command_line() - -else: - raise RuntimeError('Unexpected value for PYDEVD_USE_FRAME_EVAL: %s (accepted: YES, NO)' % (use_frame_eval,)) diff --git a/src/debugpy/_vendored/pydevd/build_tools/build.py b/src/debugpy/_vendored/pydevd/build_tools/build.py index bf34feb35..6ae56431d 100644 --- a/src/debugpy/_vendored/pydevd/build_tools/build.py +++ b/src/debugpy/_vendored/pydevd/build_tools/build.py @@ -158,10 +158,11 @@ def build(): if __name__ == '__main__': - use_cython = os.getenv('PYDEVD_USE_CYTHON', None) - if use_cython == 'YES': + use_cython = os.getenv('PYDEVD_USE_CYTHON', '').lower() + # Note: don't import pydevd during build (so, accept just yes/no in this case). + if use_cython == 'yes': build() - elif use_cython == 'NO': + elif use_cython == 'no': remove_binaries(['.pyd', '.so']) elif use_cython is None: # Regular process @@ -170,5 +171,5 @@ def build(): generate_cython_module() build() else: - raise RuntimeError('Unexpected value for PYDEVD_USE_CYTHON: %s (accepted: YES, NO)' % (use_cython,)) + raise RuntimeError('Unexpected value for PYDEVD_USE_CYTHON: %s (accepted: yes, no)' % (use_cython,)) diff --git a/src/debugpy/_vendored/pydevd/pydevd_attach_to_process/attach_script.py b/src/debugpy/_vendored/pydevd/pydevd_attach_to_process/attach_script.py index 60faa1dd0..301de7599 100644 --- a/src/debugpy/_vendored/pydevd/pydevd_attach_to_process/attach_script.py +++ b/src/debugpy/_vendored/pydevd/pydevd_attach_to_process/attach_script.py @@ -1,65 +1,5 @@ -def load_python_helper_lib(): - import sys - try: - import ctypes - except ImportError: - ctypes = None - - # Note: we cannot use import platform because it may end up importing threading, - # but that should be ok because at this point we can only be in CPython (other - # implementations wouldn't get to this point in the attach process). - # IS_CPYTHON = platform.python_implementation() == 'CPython' - IS_CPYTHON = True - - import os - IS_64BIT_PROCESS = sys.maxsize > (2 ** 32) - IS_WINDOWS = sys.platform == 'win32' - IS_LINUX = sys.platform in ('linux', 'linux2') - IS_MAC = sys.platform == 'darwin' - - if not IS_CPYTHON or ctypes is None or sys.version_info[:2] > (3, 7): - return None - - if IS_WINDOWS: - if IS_64BIT_PROCESS: - suffix = 'amd64' - else: - suffix = 'x86' - - filename = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'pydevd_attach_to_process', 'attach_%s.dll' % (suffix,)) - - elif IS_LINUX: - if IS_64BIT_PROCESS: - suffix = 'amd64' - else: - suffix = 'x86' - - filename = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'pydevd_attach_to_process', 'attach_linux_%s.so' % (suffix,)) - - elif IS_MAC: - if IS_64BIT_PROCESS: - suffix = 'x86_64.dylib' - else: - suffix = 'x86.dylib' - - filename = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'pydevd_attach_to_process', 'attach_%s' % (suffix,)) - - else: - return None - - if not os.path.exists(filename): - return None - - try: - # Load as pydll so that we don't release the gil. - lib = ctypes.pydll.LoadLibrary(filename) - return lib - except: - return None - - def get_main_thread_instance(threading): if hasattr(threading, 'main_thread'): return threading.main_thread() diff --git a/src/debugpy/_vendored/pydevd/pydevd_tracing.py b/src/debugpy/_vendored/pydevd/pydevd_tracing.py index 983517a14..b7e05a0ee 100644 --- a/src/debugpy/_vendored/pydevd/pydevd_tracing.py +++ b/src/debugpy/_vendored/pydevd/pydevd_tracing.py @@ -1,6 +1,6 @@ - from _pydevd_bundle.pydevd_constants import get_frame, IS_CPYTHON, IS_64BIT_PROCESS, IS_WINDOWS, \ - IS_LINUX, IS_MAC, IS_PY2, DebugInfoHolder, ForkSafeLock + IS_LINUX, IS_MAC, IS_PY2, DebugInfoHolder, ForkSafeLock, LOAD_NATIVE_LIB_FLAG, \ + ENV_FALSE_LOWER_VALUES from _pydev_imps._pydev_saved_modules import thread, threading from _pydev_bundle import pydev_log, pydev_monkey from os.path import os @@ -109,7 +109,7 @@ def restore_sys_set_trace_func(): def load_python_helper_lib(): - if not IS_CPYTHON or ctypes is None or sys.version_info[:2] > (3, 8): + if not IS_CPYTHON or ctypes is None or sys.version_info[:2] > (3, 8) or hasattr(sys, 'gettotalrefcount') or LOAD_NATIVE_LIB_FLAG in ENV_FALSE_LOWER_VALUES: return None if IS_WINDOWS: @@ -159,8 +159,11 @@ def load_python_helper_lib(): def set_trace_to_threads(tracing_func): lib = load_python_helper_lib() if lib is None: # This is the case if it's not CPython. + pydev_log.info('Unable to load helper lib to set tracing to all threads (unsupported python vm).') return -1 + pydev_log.info('Successfully Loaded helper lib to set tracing to all threads.') + ret = 0 set_trace_func = TracingFunctionHolder._original_tracing or sys.settrace diff --git a/src/debugpy/_vendored/pydevd/tests_python/check_debug_python.py b/src/debugpy/_vendored/pydevd/tests_python/check_debug_python.py new file mode 100644 index 000000000..b81861d5b --- /dev/null +++ b/src/debugpy/_vendored/pydevd/tests_python/check_debug_python.py @@ -0,0 +1,38 @@ +import sys +import threading +from _pydev_bundle import pydev_log + + +def check(): + with pydev_log.log_context(3, sys.stderr): + assert hasattr(sys, 'gettotalrefcount') + import pydevd_tracing + + proceed1 = threading.Event() + proceed2 = threading.Event() + + class SomeThread(threading.Thread): + + def run(self): + proceed1.set() + proceed2.wait() + + t = SomeThread() + t.start() + proceed1.wait() + try: + + def some_func(frame, event, arg): + return some_func + + pydevd_tracing.set_trace_to_threads(some_func) + finally: + proceed2.set() + + lib = pydevd_tracing.load_python_helper_lib() + assert lib is None + print('Finished OK') + + +if __name__ == '__main__': + check()