pythongh-86298: Ensure that __loader__ and __spec__.loader agree in w…

…arnings.warn_explicit() (pythonGH-97803)

In `_warnings.c`, in the C equivalent of `warnings.warn_explicit()`, if the module globals are given (and not None), the warning will attempt to get the source line for the issued warning.  To do this, it needs the module's loader.

Previously, it would only look up `__loader__` in the module globals.  In python#86298 we want to defer to the `__spec__.loader` if available.

The first step on this journey is to check that `loader == __spec__.loader` and issue another warning if it is not.  This commit does that.

Since this is a PoC, only manual testing for now.

```python
# /tmp/foo.py
import warnings

import bar

warnings.warn_explicit(
    'warning!',
    RuntimeWarning,
    'bar.py', 2,
    module='bar knee',
    module_globals=bar.__dict__,
    )
```

```python
# /tmp/bar.py
import sys
import os
import pathlib

# __loader__ = pathlib.Path()
```

Then running this: `./python.exe -Wdefault /tmp/foo.py`

Produces:

```
bar.py:2: RuntimeWarning: warning!
  import os
```

Uncomment the `__loader__ = ` line in `bar.py` and try it again:

```
sys:1: ImportWarning: Module bar; __loader__ != __spec__.loader (<_frozen_importlib_external.SourceFileLoader object at 0x109f7dfa0> != PosixPath('.'))
bar.py:2: RuntimeWarning: warning!
  import os
```

Automerge-Triggered-By: GH:warsaw

Doc/reference/import.rst

@@ -558,6 +558,11 @@ listed below.
    It is **strongly** recommended that you rely on :attr:`__spec__`
    instead instead of this attribute.
 
+   .. versionchanged:: 3.12
+      The value of ``__loader__`` is expected to be the same as
+      ``__spec__.loader``.  The use of ``__loader__`` is deprecated and slated
+      for removal in Python 3.14.
+
 .. attribute:: __package__
 
    The module's ``__package__`` attribute may be set.  Its value must
@@ -568,6 +573,9 @@ listed below.
    submodules, to the parent package's name.  See :pep:`366` for further
    details.
 
+   This attribute is used instead of ``__name__`` to calculate explicit
+   relative imports for main modules, as defined in :pep:`366`.
+
    It is **strongly** recommended that you rely on :attr:`__spec__`
    instead instead of this attribute.
 

Lib/importlib/_bootstrap_external.py

@@ -862,6 +862,54 @@ def spec_from_file_location(name, location=None, *, loader=None,
     return spec
 
 
+def _bless_my_loader(module_globals):
+    """Helper function for _warnings.c
+
+    See GH#97850 for details.
+    """
+    # 2022-10-06(warsaw): For now, this helper is only used in _warnings.c and
+    # that use case only has the module globals.  This function could be
+    # extended to accept either that or a module object.  However, in the
+    # latter case, it would be better to raise certain exceptions when looking
+    # at a module, which should have either a __loader__ or __spec__.loader.
+    # For backward compatibility, it is possible that we'll get an empty
+    # dictionary for the module globals, and that cannot raise an exception.
+    if not isinstance(module_globals, dict):
+        return None
+
+    missing = object()
+    loader = module_globals.get('__loader__', None)
+    spec = module_globals.get('__spec__', missing)
+
+    if loader is None:
+        if spec is missing:
+            # If working with a module:
+            # raise AttributeError('Module globals is missing a __spec__')
+            return None
+        elif spec is None:
+            raise ValueError('Module globals is missing a __spec__.loader')
+
+    spec_loader = getattr(spec, 'loader', missing)
+
+    if spec_loader in (missing, None):
+        if loader is None:
+            exc = AttributeError if spec_loader is missing else ValueError
+            raise exc('Module globals is missing a __spec__.loader')
+        _warnings.warn(
+            'Module globals is missing a __spec__.loader',
+            DeprecationWarning)
+        spec_loader = loader
+
+    assert spec_loader is not None
+    if loader is not None and loader != spec_loader:
+        _warnings.warn(
+            'Module globals; __loader__ != __spec__.loader',
+            DeprecationWarning)
+        return loader
+
+    return spec_loader
+
+
 # Loaders #####################################################################
 
 class WindowsRegistryFinder:

Lib/test/test_importlib/import_/test_helpers.py

@@ -2,7 +2,9 @@
 
 from importlib import _bootstrap_external, machinery
 import os.path
+from types import ModuleType, SimpleNamespace
 import unittest
+import warnings
 
 from .. import util
 
@@ -67,5 +69,116 @@ def test_no_loader_no_spec_but_source(self):
 
 FrozenFixUpModuleTests, SourceFixUpModuleTests = util.test_both(FixUpModuleTests)
 
+
+class TestBlessMyLoader(unittest.TestCase):
+    # GH#86298 is part of the migration away from module attributes and toward
+    # __spec__ attributes.  There are several cases to test here.  This will
+    # have to change in Python 3.14 when we actually remove/ignore __loader__
+    # in favor of requiring __spec__.loader.
+
+    def test_gh86298_no_loader_and_no_spec(self):
+        bar = ModuleType('bar')
+        del bar.__loader__
+        del bar.__spec__
+        # 2022-10-06(warsaw): For backward compatibility with the
+        # implementation in _warnings.c, this can't raise an
+        # AttributeError.  See _bless_my_loader() in _bootstrap_external.py
+        # If working with a module:
+        ## self.assertRaises(
+        ##     AttributeError, _bootstrap_external._bless_my_loader,
+        ##     bar.__dict__)
+        self.assertIsNone(_bootstrap_external._bless_my_loader(bar.__dict__))
+
+    def test_gh86298_loader_is_none_and_no_spec(self):
+        bar = ModuleType('bar')
+        bar.__loader__ = None
+        del bar.__spec__
+        # 2022-10-06(warsaw): For backward compatibility with the
+        # implementation in _warnings.c, this can't raise an
+        # AttributeError.  See _bless_my_loader() in _bootstrap_external.py
+        # If working with a module:
+        ## self.assertRaises(
+        ##     AttributeError, _bootstrap_external._bless_my_loader,
+        ##     bar.__dict__)
+        self.assertIsNone(_bootstrap_external._bless_my_loader(bar.__dict__))
+
+    def test_gh86298_no_loader_and_spec_is_none(self):
+        bar = ModuleType('bar')
+        del bar.__loader__
+        bar.__spec__ = None
+        self.assertRaises(
+            ValueError,
+            _bootstrap_external._bless_my_loader, bar.__dict__)
+
+    def test_gh86298_loader_is_none_and_spec_is_none(self):
+        bar = ModuleType('bar')
+        bar.__loader__ = None
+        bar.__spec__ = None
+        self.assertRaises(
+            ValueError,
+            _bootstrap_external._bless_my_loader, bar.__dict__)
+
+    def test_gh86298_loader_is_none_and_spec_loader_is_none(self):
+        bar = ModuleType('bar')
+        bar.__loader__ = None
+        bar.__spec__ = SimpleNamespace(loader=None)
+        self.assertRaises(
+            ValueError,
+            _bootstrap_external._bless_my_loader, bar.__dict__)
+
+    def test_gh86298_no_spec(self):
+        bar = ModuleType('bar')
+        bar.__loader__ = object()
+        del bar.__spec__
+        with warnings.catch_warnings():
+            self.assertWarns(
+                DeprecationWarning,
+                _bootstrap_external._bless_my_loader, bar.__dict__)
+
+    def test_gh86298_spec_is_none(self):
+        bar = ModuleType('bar')
+        bar.__loader__ = object()
+        bar.__spec__ = None
+        with warnings.catch_warnings():
+            self.assertWarns(
+                DeprecationWarning,
+                _bootstrap_external._bless_my_loader, bar.__dict__)
+
+    def test_gh86298_no_spec_loader(self):
+        bar = ModuleType('bar')
+        bar.__loader__ = object()
+        bar.__spec__ = SimpleNamespace()
+        with warnings.catch_warnings():
+            self.assertWarns(
+                DeprecationWarning,
+                _bootstrap_external._bless_my_loader, bar.__dict__)
+
+    def test_gh86298_loader_and_spec_loader_disagree(self):
+        bar = ModuleType('bar')
+        bar.__loader__ = object()
+        bar.__spec__ = SimpleNamespace(loader=object())
+        with warnings.catch_warnings():
+            self.assertWarns(
+                DeprecationWarning,
+                _bootstrap_external._bless_my_loader, bar.__dict__)
+
+    def test_gh86298_no_loader_and_no_spec_loader(self):
+        bar = ModuleType('bar')
+        del bar.__loader__
+        bar.__spec__ = SimpleNamespace()
+        self.assertRaises(
+            AttributeError,
+            _bootstrap_external._bless_my_loader, bar.__dict__)
+
+    def test_gh86298_no_loader_with_spec_loader_okay(self):
+        bar = ModuleType('bar')
+        del bar.__loader__
+        loader = object()
+        bar.__spec__ = SimpleNamespace(loader=loader)
+        self.assertEqual(
+            _bootstrap_external._bless_my_loader(bar.__dict__),
+            loader)
+
+
 if __name__ == "__main__":
     unittest.main()

Misc/NEWS.d/next/Core and Builtins/2022-10-04-14-04-40.gh-issue-86298.QVM7G1.rst

@@ -0,0 +1,3 @@
+In cases where ``warnings.warn_explicit()`` consults the module's loader, an
+``DeprecationWarning`` is issued when ``m.__loader__`` differs from
+``m.__spec__.loader``.

Python/_warnings.c

@@ -977,19 +977,26 @@ warnings_warn_impl(PyObject *module, PyObject *message, PyObject *category,
 static PyObject *
 get_source_line(PyInterpreterState *interp, PyObject *module_globals, int lineno)
 {
+    PyObject *external;
     PyObject *loader;
     PyObject *module_name;
     PyObject *get_source;
     PyObject *source;
     PyObject *source_list;
     PyObject *source_line;
 
-    /* Check/get the requisite pieces needed for the loader. */
-    loader = _PyDict_GetItemWithError(module_globals, &_Py_ID(__loader__));
+    /* stolen from import.c */
+    external = PyObject_GetAttrString(interp->importlib, "_bootstrap_external");
+    if (external == NULL) {
+        return NULL;
+    }
+
+    loader = PyObject_CallMethod(external, "_bless_my_loader", "O", module_globals, NULL);
+    Py_DECREF(external);
     if (loader == NULL) {
         return NULL;
     }
-    Py_INCREF(loader);
+
     module_name = _PyDict_GetItemWithError(module_globals, &_Py_ID(__name__));
     if (!module_name) {
         Py_DECREF(loader);