From 9eda118f9be64c237e0c90c148a10381bf3d190d Mon Sep 17 00:00:00 2001 From: krassowski <5832902+krassowski@users.noreply.github.com> Date: Wed, 14 Feb 2024 13:30:30 +0000 Subject: [PATCH] Disable frozen modules by default, add a toggle --- ipykernel/kernelspec.py | 57 +++++++++++++++++++++++++++++++--------- tests/test_kernelspec.py | 48 +++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 13 deletions(-) diff --git a/ipykernel/kernelspec.py b/ipykernel/kernelspec.py index a1d508c3..c0cc9ffe 100644 --- a/ipykernel/kernelspec.py +++ b/ipykernel/kernelspec.py @@ -8,6 +8,7 @@ import errno import json import os +import platform import shutil import stat import sys @@ -19,11 +20,6 @@ from traitlets import Unicode from traitlets.config import Application -try: - from .debugger import _is_debugpy_available -except ImportError: - _is_debugpy_available = False - pjoin = os.path.join KERNEL_NAME = "python%i" % sys.version_info[0] @@ -36,6 +32,7 @@ def make_ipkernel_cmd( mod: str = "ipykernel_launcher", executable: str | None = None, extra_arguments: list[str] | None = None, + python_arguments: list[str] | None = None, ) -> list[str]: """Build Popen command list for launching an IPython kernel. @@ -55,16 +52,18 @@ def make_ipkernel_cmd( if executable is None: executable = sys.executable extra_arguments = extra_arguments or [] - arguments = [executable, "-m", mod, "-f", "{connection_file}"] - arguments.extend(extra_arguments) - - return arguments + python_arguments = python_arguments or [] + return [executable, *python_arguments, "-m", mod, "-f", "{connection_file}", *extra_arguments] -def get_kernel_dict(extra_arguments: list[str] | None = None) -> dict[str, Any]: +def get_kernel_dict( + extra_arguments: list[str] | None = None, python_arguments: list[str] | None = None +) -> dict[str, Any]: """Construct dict for kernel.json""" return { - "argv": make_ipkernel_cmd(extra_arguments=extra_arguments), + "argv": make_ipkernel_cmd( + extra_arguments=extra_arguments, python_arguments=python_arguments + ), "display_name": "Python %i (ipykernel)" % sys.version_info[0], "language": "python", "metadata": {"debugger": True}, @@ -75,6 +74,7 @@ def write_kernel_spec( path: Path | str | None = None, overrides: dict[str, Any] | None = None, extra_arguments: list[str] | None = None, + python_arguments: list[str] | None = None, ) -> str: """Write a kernel spec directory to `path` @@ -95,7 +95,7 @@ def write_kernel_spec( Path(path).chmod(mask | stat.S_IWUSR) # write kernel.json - kernel_dict = get_kernel_dict(extra_arguments) + kernel_dict = get_kernel_dict(extra_arguments, python_arguments) if overrides: kernel_dict.update(overrides) @@ -113,6 +113,7 @@ def install( prefix: str | None = None, profile: str | None = None, env: dict[str, str] | None = None, + frozen_modules: bool = False, ) -> str: """Install the IPython kernelspec for Jupyter @@ -137,6 +138,12 @@ def install( A dictionary of extra environment variables for the kernel. These will be added to the current environment variables before the kernel is started + frozen_modules : bool, optional + Whether to use frozen modules for potentially faster kernel startup. + Using frozen modules prevents debugging inside of some built-in + Python modules, such as io, abc, posixpath, ntpath, or stat. + The frozen modules are used in CPython for faster interpreter startup. + Ignored for cPython <3.11 and for other Python implementations. Returns ------- @@ -145,6 +152,9 @@ def install( if kernel_spec_manager is None: kernel_spec_manager = KernelSpecManager() + if env is None: + env = {} + if (kernel_name != KERNEL_NAME) and (display_name is None): # kernel_name is specified and display_name is not # default display_name to kernel_name @@ -159,9 +169,24 @@ def install( overrides["display_name"] = "Python %i [profile=%s]" % (sys.version_info[0], profile) else: extra_arguments = None + + python_arguments = None + + # addresses the debugger warning from debugpy about frozen modules + if sys.version_info >= (3, 11) and platform.python_implementation() == "CPython": + if not frozen_modules: + # disable frozen modules + python_arguments = ["-Xfrozen_modules=off"] + elif "PYDEVD_DISABLE_FILE_VALIDATION" not in env: + # user opted-in to have frozen modules, and we warned them about + # consequences for the - disable the debugger warning + env["PYDEVD_DISABLE_FILE_VALIDATION"] = "1" + if env: overrides["env"] = env - path = write_kernel_spec(overrides=overrides, extra_arguments=extra_arguments) + path = write_kernel_spec( + overrides=overrides, extra_arguments=extra_arguments, python_arguments=python_arguments + ) dest = kernel_spec_manager.install_kernel_spec( path, kernel_name=kernel_name, user=user, prefix=prefix ) @@ -236,6 +261,12 @@ def start(self) -> None: metavar=("ENV", "VALUE"), help="Set environment variables for the kernel.", ) + parser.add_argument( + "--frozen_modules", + action="store_true", + help="Enable frozen modules for potentially faster startup." + " This has a downside of preventing the debugger from navigating to certain built-in modules.", + ) opts = parser.parse_args(self.argv) if opts.env: opts.env = dict(opts.env) diff --git a/tests/test_kernelspec.py b/tests/test_kernelspec.py index e9210964..c3b62b21 100644 --- a/tests/test_kernelspec.py +++ b/tests/test_kernelspec.py @@ -3,6 +3,7 @@ import json import os +import platform import shutil import sys import tempfile @@ -22,6 +23,7 @@ ) pjoin = os.path.join +is_cpython = platform.python_implementation() == "CPython" def test_make_ipkernel_cmd(): @@ -144,3 +146,49 @@ def test_install_env(tmp_path, env): assert spec["env"][k] == v else: assert "env" not in spec + + +@pytest.mark.skipif(sys.version_info < (3, 11) or not is_cpython, reason="requires cPython 3.11") +def test_install_frozen_modules_on(): + system_jupyter_dir = tempfile.mkdtemp() + + with mock.patch("jupyter_client.kernelspec.SYSTEM_JUPYTER_PATH", [system_jupyter_dir]): + install(frozen_modules=True) + + spec_file = os.path.join(system_jupyter_dir, "kernels", KERNEL_NAME, "kernel.json") + with open(spec_file) as f: + spec = json.load(f) + assert spec["env"]["PYDEVD_DISABLE_FILE_VALIDATION"] == "1" + assert "-Xfrozen_modules=off" not in spec["argv"] + + +@pytest.mark.skipif(sys.version_info < (3, 11) or not is_cpython, reason="requires cPython 3.11") +def test_install_frozen_modules_off(): + system_jupyter_dir = tempfile.mkdtemp() + + with mock.patch("jupyter_client.kernelspec.SYSTEM_JUPYTER_PATH", [system_jupyter_dir]): + install(frozen_modules=False) + + spec_file = os.path.join(system_jupyter_dir, "kernels", KERNEL_NAME, "kernel.json") + with open(spec_file) as f: + spec = json.load(f) + assert "env" not in spec + assert spec["argv"][1] == "-Xfrozen_modules=off" + + +@pytest.mark.skipif( + sys.version_info >= (3, 11) or is_cpython, + reason="checks versions older than 3.11 and other Python implementations", +) +def test_install_frozen_modules_no_op(): + # ensure we do not add add Xfrozen_modules on older Python versions + # (although cPython does not error out on unknown X options as of 3.8) + system_jupyter_dir = tempfile.mkdtemp() + + with mock.patch("jupyter_client.kernelspec.SYSTEM_JUPYTER_PATH", [system_jupyter_dir]): + install(frozen_modules=False) + + spec_file = os.path.join(system_jupyter_dir, "kernels", KERNEL_NAME, "kernel.json") + with open(spec_file) as f: + spec = json.load(f) + assert "-Xfrozen_modules=off" not in spec["argv"]