From 24e2a712d80a592c134ed0a7f48c205689c97fbf Mon Sep 17 00:00:00 2001 From: Fabio Zadrozny Date: Fri, 4 Dec 2020 16:39:43 -0300 Subject: [PATCH] Support running non-ascii files on Python 2. Fixes #206 --- .../pydevd/pydevd_plugins/__init__.py | 13 ++-- .../pydevd_plugins/extensions/__init__.py | 12 ++-- .../extensions/types/__init__.py | 12 ++-- .../my_extensions/pydevd_plugins/__init__.py | 12 ++-- .../pydevd_plugins/extensions/__init__.py | 12 ++-- src/debugpy/server/api.py | 2 +- src/debugpy/server/cli.py | 62 ++++++++++++------- tests/debug/targets.py | 6 +- tests/debugpy/test_run.py | 43 +++++++++++-- 9 files changed, 116 insertions(+), 58 deletions(-) diff --git a/src/debugpy/_vendored/pydevd/pydevd_plugins/__init__.py b/src/debugpy/_vendored/pydevd/pydevd_plugins/__init__.py index afff0c07d..f471804c7 100644 --- a/src/debugpy/_vendored/pydevd/pydevd_plugins/__init__.py +++ b/src/debugpy/_vendored/pydevd/pydevd_plugins/__init__.py @@ -1,5 +1,8 @@ -try: - __import__('pkg_resources').declare_namespace(__name__) -except ImportError: - import pkgutil - __path__ = pkgutil.extend_path(__path__, __name__) +import warnings +with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=DeprecationWarning) + try: + __import__('pkg_resources').declare_namespace(__name__) + except ImportError: + import pkgutil + __path__ = pkgutil.extend_path(__path__, __name__) diff --git a/src/debugpy/_vendored/pydevd/pydevd_plugins/extensions/__init__.py b/src/debugpy/_vendored/pydevd/pydevd_plugins/extensions/__init__.py index afff0c07d..274d7bc4f 100644 --- a/src/debugpy/_vendored/pydevd/pydevd_plugins/extensions/__init__.py +++ b/src/debugpy/_vendored/pydevd/pydevd_plugins/extensions/__init__.py @@ -1,5 +1,7 @@ -try: - __import__('pkg_resources').declare_namespace(__name__) -except ImportError: - import pkgutil - __path__ = pkgutil.extend_path(__path__, __name__) +import warnings +with warnings.catch_warnings(): + try: + __import__('pkg_resources').declare_namespace(__name__) + except ImportError: + import pkgutil + __path__ = pkgutil.extend_path(__path__, __name__) diff --git a/src/debugpy/_vendored/pydevd/pydevd_plugins/extensions/types/__init__.py b/src/debugpy/_vendored/pydevd/pydevd_plugins/extensions/types/__init__.py index afff0c07d..274d7bc4f 100644 --- a/src/debugpy/_vendored/pydevd/pydevd_plugins/extensions/types/__init__.py +++ b/src/debugpy/_vendored/pydevd/pydevd_plugins/extensions/types/__init__.py @@ -1,5 +1,7 @@ -try: - __import__('pkg_resources').declare_namespace(__name__) -except ImportError: - import pkgutil - __path__ = pkgutil.extend_path(__path__, __name__) +import warnings +with warnings.catch_warnings(): + try: + __import__('pkg_resources').declare_namespace(__name__) + except ImportError: + import pkgutil + __path__ = pkgutil.extend_path(__path__, __name__) diff --git a/src/debugpy/_vendored/pydevd/tests_python/my_extensions/pydevd_plugins/__init__.py b/src/debugpy/_vendored/pydevd/tests_python/my_extensions/pydevd_plugins/__init__.py index afff0c07d..274d7bc4f 100644 --- a/src/debugpy/_vendored/pydevd/tests_python/my_extensions/pydevd_plugins/__init__.py +++ b/src/debugpy/_vendored/pydevd/tests_python/my_extensions/pydevd_plugins/__init__.py @@ -1,5 +1,7 @@ -try: - __import__('pkg_resources').declare_namespace(__name__) -except ImportError: - import pkgutil - __path__ = pkgutil.extend_path(__path__, __name__) +import warnings +with warnings.catch_warnings(): + try: + __import__('pkg_resources').declare_namespace(__name__) + except ImportError: + import pkgutil + __path__ = pkgutil.extend_path(__path__, __name__) diff --git a/src/debugpy/_vendored/pydevd/tests_python/my_extensions/pydevd_plugins/extensions/__init__.py b/src/debugpy/_vendored/pydevd/tests_python/my_extensions/pydevd_plugins/extensions/__init__.py index afff0c07d..274d7bc4f 100644 --- a/src/debugpy/_vendored/pydevd/tests_python/my_extensions/pydevd_plugins/extensions/__init__.py +++ b/src/debugpy/_vendored/pydevd/tests_python/my_extensions/pydevd_plugins/extensions/__init__.py @@ -1,5 +1,7 @@ -try: - __import__('pkg_resources').declare_namespace(__name__) -except ImportError: - import pkgutil - __path__ = pkgutil.extend_path(__path__, __name__) +import warnings +with warnings.catch_warnings(): + try: + __import__('pkg_resources').declare_namespace(__name__) + except ImportError: + import pkgutil + __path__ = pkgutil.extend_path(__path__, __name__) diff --git a/src/debugpy/server/api.py b/src/debugpy/server/api.py index d86813281..3f52633a4 100644 --- a/src/debugpy/server/api.py +++ b/src/debugpy/server/api.py @@ -135,7 +135,7 @@ def debug(address, **kwargs): debugpy_path = os.path.dirname(absolute_path(debugpy.__file__)) settrace_kwargs["dont_trace_start_patterns"] = (debugpy_path,) - settrace_kwargs["dont_trace_end_patterns"] = ("debugpy_launcher.py",) + settrace_kwargs["dont_trace_end_patterns"] = (str("debugpy_launcher.py"),) try: return func(address, settrace_kwargs, **kwargs) diff --git a/src/debugpy/server/cli.py b/src/debugpy/server/cli.py index 3b40ff5fd..b073f91eb 100644 --- a/src/debugpy/server/cli.py +++ b/src/debugpy/server/cli.py @@ -43,7 +43,7 @@ class Options(object): address = None log_to = None log_to_stderr = False - target = None + target = None # unicode target_kind = None wait_for_client = False adapter_access_token = None @@ -141,7 +141,20 @@ def set_config(arg, it): def set_target(kind, parser=(lambda x: x), positional=False): def do(arg, it): options.target_kind = kind - options.target = parser(arg if positional else next(it)) + target = parser(arg if positional else next(it)) + + if isinstance(target, bytes): + # target may be the code, so, try some additional encodings... + try: + target = target.decode(sys.getfilesystemencoding()) + except UnicodeDecodeError: + try: + target = target.decode("utf-8") + except UnicodeDecodeError: + import locale + + target = target.decode(locale.getpreferredencoding(False)) + options.target = target return do @@ -233,6 +246,7 @@ def start_debugging(argv_0): # because they use it to report the "process" event. Thus, we can't rely on # run_path() and run_module() doing that, even though they will eventually. sys.argv[0] = compat.filename_str(argv_0) + log.debug("sys.argv after patching: {0!r}", sys.argv) debugpy.configure(options.config) @@ -249,43 +263,48 @@ def start_debugging(argv_0): def run_file(): - start_debugging(options.target) + target = options.target + start_debugging(target) + + target_as_str = compat.filename_str(target) # run_path has one difference with invoking Python from command-line: # if the target is a file (rather than a directory), it does not add its # parent directory to sys.path. Thus, importing other modules from the # same directory is broken unless sys.path is patched here. - if os.path.isfile(options.target): - dir = os.path.dirname(options.target) + + if os.path.isfile(target_as_str): + dir = os.path.dirname(target_as_str) sys.path.insert(0, dir) else: - log.debug("Not a file: {0!r}", options.target) + log.debug("Not a file: {0!r}", target) log.describe_environment("Pre-launch environment:") - log.info("Running file {0!r}", options.target) - runpy.run_path(options.target, run_name=compat.force_str("__main__")) + log.info("Running file {0!r}", target) + runpy.run_path(target_as_str, run_name=compat.force_str("__main__")) def run_module(): # Add current directory to path, like Python itself does for -m. This must # be in place before trying to use find_spec below to resolve submodules. - sys.path.insert(0, "") + sys.path.insert(0, str("")) # We want to do the same thing that run_module() would do here, without # actually invoking it. On Python 3, it's exposed as a public API, but # on Python 2, we have to invoke a private function in runpy for this. # Either way, if it fails to resolve for any reason, just leave argv as is. argv_0 = sys.argv[0] + target_as_str = compat.filename_str(options.target) try: if sys.version_info >= (3,): from importlib.util import find_spec - spec = find_spec(options.target) + spec = find_spec(target_as_str) if spec is not None: argv_0 = spec.origin else: - _, _, _, argv_0 = runpy._get_module_details(options.target) + _, _, _, argv_0 = runpy._get_module_details(target_as_str) except Exception: log.swallow_exception("Error determining module path for sys.argv") @@ -294,14 +313,9 @@ def run_module(): # On Python 2, module name must be a non-Unicode string, because it ends up # a part of module's __package__, and Python will refuse to run the module # if __package__ is Unicode. - target = ( - compat.filename_bytes(options.target) - if sys.version_info < (3,) - else options.target - ) log.describe_environment("Pre-launch environment:") - log.info("Running module {0!r}", target) + log.info("Running module {0!r}", options.target) # Docs say that runpy.run_module is equivalent to -m, but it's not actually # the case for packages - -m sets __name__ to "__main__", but run_module sets @@ -312,17 +326,17 @@ def run_module(): run_module_as_main = runpy._run_module_as_main except AttributeError: log.warning("runpy._run_module_as_main is missing, falling back to run_module.") - runpy.run_module(target, alter_sys=True) + runpy.run_module(target_as_str, alter_sys=True) else: - run_module_as_main(target, alter_argv=True) + run_module_as_main(target_as_str, alter_argv=True) def run_code(): # Add current directory to path, like Python itself does for -c. - sys.path.insert(0, "") - code = compile(options.target, "", "exec") + sys.path.insert(0, str("")) + code = compile(options.target, str(""), str("exec")) - start_debugging("-c") + start_debugging(str("-c")) log.describe_environment("Pre-launch environment:") log.info("Running code:\n\n{0}", options.target) @@ -404,7 +418,7 @@ def main(): try: parse_argv() except Exception as exc: - print(HELP + "\nError: " + str(exc), file=sys.stderr) + print(str(HELP) + str("\nError: ") + str(exc), file=sys.stderr) sys.exit(2) if options.log_to is not None: @@ -415,7 +429,7 @@ def main(): api.ensure_logging() log.info( - "sys.argv before parsing: {0!r}\n" " after parsing: {1!r}", + str("sys.argv before parsing: {0!r}\n" " after parsing: {1!r}"), original_argv, sys.argv, ) diff --git a/tests/debug/targets.py b/tests/debug/targets.py index a9d88f9b3..2189574c0 100644 --- a/tests/debug/targets.py +++ b/tests/debug/targets.py @@ -6,7 +6,7 @@ import py -from debugpy.common import fmt +from debugpy.common import fmt, compat from tests.patterns import some @@ -79,10 +79,10 @@ class Program(Target): pytest_id = "program" def __repr__(self): - return fmt("program {0!j}", self.filename) + return fmt("program {0!j}", compat.filename(self.filename.strpath)) def configure(self, session): - session.config["program"] = self.filename + session.config["program"] = compat.filename(self.filename.strpath) session.config["args"] = self.args def cli(self, env): diff --git a/tests/debugpy/test_run.py b/tests/debugpy/test_run.py index 47d9011ac..df0f8a2f6 100644 --- a/tests/debugpy/test_run.py +++ b/tests/debugpy/test_run.py @@ -1,3 +1,4 @@ +# coding: utf-8 # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See LICENSE in the project root # for license information. @@ -162,7 +163,7 @@ def make_custompy(tmpdir, id=""): @pytest.mark.parametrize("debuggee_custompy", [None, "launcher"]) @pytest.mark.parametrize("launcher_custompy", [None, "debuggee"]) def test_custom_python( - pyfile, tmpdir, run, target, debuggee_custompy, launcher_custompy, + pyfile, tmpdir, run, target, debuggee_custompy, launcher_custompy ): @pyfile def code_to_debug(): @@ -236,10 +237,7 @@ def code_to_debug(): with run(session, target(code_to_debug)): pass - assert backchannel.receive() == [ - "-O" in python_cmd, - "-B" in python_cmd, - ] + assert backchannel.receive() == ["-O" in python_cmd, "-B" in python_cmd] @pytest.mark.parametrize("run", runners.all) @@ -284,3 +282,38 @@ def code_to_debug(): using_frame_eval = backchannel.receive() assert using_frame_eval == (frame_eval == "yes") + + +@pytest.mark.skipif( + not sys.version_info[0] >= 3, + reason="Test structure must still be updated to properly support Python 2 with unicode", +) +@pytest.mark.parametrize("run", [runners.all_launch[0]]) +def test_unicode_dir(tmpdir, run, target): + from debugpy.common import compat + + unicode_chars = "รก" + + directory = os.path.join(compat.force_unicode(str(tmpdir), "ascii"), unicode_chars) + directory = compat.filename_str(directory) + os.makedirs(directory) + + code_to_debug = os.path.join(directory, compat.filename_str("experiment.py")) + with open(code_to_debug, "wb") as stream: + stream.write( + b""" +import debuggee +from debuggee import backchannel + +debuggee.setup() +backchannel.send('ok') +""" + ) + + with debug.Session() as session: + backchannel = session.open_backchannel() + with run(session, target(compat.filename_str(code_to_debug))): + pass + + received = backchannel.receive() + assert received == "ok"