Disable frozen modules by default, add a toggle (#1213)

ipykernel/kernelspec.py

@@ -8,6 +8,7 @@
 import errno
 import json
 import os
+import platform
 import shutil
 import stat
 import sys
@@ -19,11 +20,6 @@
 from traitlets import Unicode
 from traitlets.config import Application
 
-try:
-    from .debugger import _is_debugpy_available
-except ImportError:
-    _is_debugpy_available = False
-
 pjoin = os.path.join
 
 KERNEL_NAME = "python%i" % sys.version_info[0]
@@ -36,6 +32,7 @@ def make_ipkernel_cmd(
     mod: str = "ipykernel_launcher",
     executable: str | None = None,
     extra_arguments: list[str] | None = None,
+    python_arguments: list[str] | None = None,
 ) -> list[str]:
     """Build Popen command list for launching an IPython kernel.
 
@@ -55,16 +52,18 @@ def make_ipkernel_cmd(
     if executable is None:
         executable = sys.executable
     extra_arguments = extra_arguments or []
-    arguments = [executable, "-m", mod, "-f", "{connection_file}"]
-    arguments.extend(extra_arguments)
-
-    return arguments
+    python_arguments = python_arguments or []
+    return [executable, *python_arguments, "-m", mod, "-f", "{connection_file}", *extra_arguments]
 
 
-def get_kernel_dict(extra_arguments: list[str] | None = None) -> dict[str, Any]:
+def get_kernel_dict(
+    extra_arguments: list[str] | None = None, python_arguments: list[str] | None = None
+) -> dict[str, Any]:
     """Construct dict for kernel.json"""
     return {
-        "argv": make_ipkernel_cmd(extra_arguments=extra_arguments),
+        "argv": make_ipkernel_cmd(
+            extra_arguments=extra_arguments, python_arguments=python_arguments
+        ),
         "display_name": "Python %i (ipykernel)" % sys.version_info[0],
         "language": "python",
         "metadata": {"debugger": True},
@@ -75,6 +74,7 @@ def write_kernel_spec(
     path: Path | str | None = None,
     overrides: dict[str, Any] | None = None,
     extra_arguments: list[str] | None = None,
+    python_arguments: list[str] | None = None,
 ) -> str:
     """Write a kernel spec directory to `path`
 
@@ -95,7 +95,7 @@ def write_kernel_spec(
         Path(path).chmod(mask | stat.S_IWUSR)
 
     # write kernel.json
-    kernel_dict = get_kernel_dict(extra_arguments)
+    kernel_dict = get_kernel_dict(extra_arguments, python_arguments)
 
     if overrides:
         kernel_dict.update(overrides)
@@ -113,6 +113,7 @@ def install(
     prefix: str | None = None,
     profile: str | None = None,
     env: dict[str, str] | None = None,
+    frozen_modules: bool = False,
 ) -> str:
     """Install the IPython kernelspec for Jupyter
 
@@ -137,6 +138,12 @@ def install(
         A dictionary of extra environment variables for the kernel.
         These will be added to the current environment variables before the
         kernel is started
+    frozen_modules : bool, optional
+        Whether to use frozen modules for potentially faster kernel startup.
+        Using frozen modules prevents debugging inside of some built-in
+        Python modules, such as io, abc, posixpath, ntpath, or stat.
+        The frozen modules are used in CPython for faster interpreter startup.
+        Ignored for cPython <3.11 and for other Python implementations.
 
     Returns
     -------
@@ -145,6 +152,9 @@ def install(
     if kernel_spec_manager is None:
         kernel_spec_manager = KernelSpecManager()
 
+    if env is None:
+        env = {}
+
     if (kernel_name != KERNEL_NAME) and (display_name is None):
         # kernel_name is specified and display_name is not
         # default display_name to kernel_name
@@ -159,9 +169,24 @@ def install(
             overrides["display_name"] = "Python %i [profile=%s]" % (sys.version_info[0], profile)
     else:
         extra_arguments = None
+
+    python_arguments = None
+
+    # addresses the debugger warning from debugpy about frozen modules
+    if sys.version_info >= (3, 11) and platform.python_implementation() == "CPython":
+        if not frozen_modules:
+            # disable frozen modules
+            python_arguments = ["-Xfrozen_modules=off"]
+        elif "PYDEVD_DISABLE_FILE_VALIDATION" not in env:
+            # user opted-in to have frozen modules, and we warned them about
+            # consequences for the - disable the debugger warning
+            env["PYDEVD_DISABLE_FILE_VALIDATION"] = "1"
+
     if env:
         overrides["env"] = env
-    path = write_kernel_spec(overrides=overrides, extra_arguments=extra_arguments)
+    path = write_kernel_spec(
+        overrides=overrides, extra_arguments=extra_arguments, python_arguments=python_arguments
+    )
     dest = kernel_spec_manager.install_kernel_spec(
         path, kernel_name=kernel_name, user=user, prefix=prefix
     )
@@ -236,6 +261,12 @@ def start(self) -> None:
             metavar=("ENV", "VALUE"),
             help="Set environment variables for the kernel.",
         )
+        parser.add_argument(
+            "--frozen_modules",
+            action="store_true",
+            help="Enable frozen modules for potentially faster startup."
+            " This has a downside of preventing the debugger from navigating to certain built-in modules.",
+        )
         opts = parser.parse_args(self.argv)
         if opts.env:
             opts.env = dict(opts.env)

tests/test_kernelspec.py

@@ -3,6 +3,7 @@
 
 import json
 import os
+import platform
 import shutil
 import sys
 import tempfile
@@ -22,6 +23,7 @@
 )
 
 pjoin = os.path.join
+is_cpython = platform.python_implementation() == "CPython"
 
 
 def test_make_ipkernel_cmd():
@@ -144,3 +146,49 @@ def test_install_env(tmp_path, env):
             assert spec["env"][k] == v
     else:
         assert "env" not in spec
+
+
+@pytest.mark.skipif(sys.version_info < (3, 11) or not is_cpython, reason="requires cPython 3.11")
+def test_install_frozen_modules_on():
+    system_jupyter_dir = tempfile.mkdtemp()
+
+    with mock.patch("jupyter_client.kernelspec.SYSTEM_JUPYTER_PATH", [system_jupyter_dir]):
+        install(frozen_modules=True)
+
+    spec_file = os.path.join(system_jupyter_dir, "kernels", KERNEL_NAME, "kernel.json")
+    with open(spec_file) as f:
+        spec = json.load(f)
+    assert spec["env"]["PYDEVD_DISABLE_FILE_VALIDATION"] == "1"
+    assert "-Xfrozen_modules=off" not in spec["argv"]
+
+
+@pytest.mark.skipif(sys.version_info < (3, 11) or not is_cpython, reason="requires cPython 3.11")
+def test_install_frozen_modules_off():
+    system_jupyter_dir = tempfile.mkdtemp()
+
+    with mock.patch("jupyter_client.kernelspec.SYSTEM_JUPYTER_PATH", [system_jupyter_dir]):
+        install(frozen_modules=False)
+
+    spec_file = os.path.join(system_jupyter_dir, "kernels", KERNEL_NAME, "kernel.json")
+    with open(spec_file) as f:
+        spec = json.load(f)
+    assert "env" not in spec
+    assert spec["argv"][1] == "-Xfrozen_modules=off"
+
+
+@pytest.mark.skipif(
+    sys.version_info >= (3, 11) or is_cpython,
+    reason="checks versions older than 3.11 and other Python implementations",
+)
+def test_install_frozen_modules_no_op():
+    # ensure we do not add add Xfrozen_modules on older Python versions
+    # (although cPython does not error out on unknown X options as of 3.8)
+    system_jupyter_dir = tempfile.mkdtemp()
+
+    with mock.patch("jupyter_client.kernelspec.SYSTEM_JUPYTER_PATH", [system_jupyter_dir]):
+        install(frozen_modules=False)
+
+    spec_file = os.path.join(system_jupyter_dir, "kernels", KERNEL_NAME, "kernel.json")
+    with open(spec_file) as f:
+        spec = json.load(f)
+    assert "-Xfrozen_modules=off" not in spec["argv"]