Merge remote-tracking branch 'upstream/master' into TypeVar-01-founda…

…tion

.pre-commit-config.yaml

@@ -4,11 +4,11 @@ repos:
     hooks:
       - id: black
   - repo: https://github.com/pycqa/isort
-    rev: 5.11.5  # must match test-requirements.txt
+    rev: 5.12.0  # must match test-requirements.txt
     hooks:
       - id: isort
   - repo: https://github.com/pycqa/flake8
-    rev: 5.0.4  # must match test-requirements.txt
+    rev: 6.0.0  # must match test-requirements.txt
     hooks:
       - id: flake8
         additional_dependencies:

docs/source/error_code_list2.rst

@@ -347,3 +347,47 @@ silence the error:
 
     async def g() -> None:
         _ = asyncio.create_task(f())  # No error
+
+Check that ``# type: ignore`` comment is used [unused-ignore]
+-------------------------------------------------------------
+
+If you use :option:`--enable-error-code unused-ignore <mypy --enable-error-code>`,
+or :option:`--warn-unused-ignores <mypy --warn-unused-ignores>`
+mypy generates an error if you don't use a ``# type: ignore`` comment, i.e. if
+there is a comment, but there would be no error generated by mypy on this line
+anyway.
+
+Example:
+
+.. code-block:: python
+
+    # Use "mypy --warn-unused-ignores ..."
+
+    def add(a: int, b: int) -> int:
+        # Error: unused "type: ignore" comment
+        return a + b  # type: ignore
+
+Note that due to a specific nature of this comment, the only way to selectively
+silence it, is to include the error code explicitly. Also note that this error is
+not shown if the ``# type: ignore`` is not used due to code being statically
+unreachable (e.g. due to platform or version checks).
+
+Example:
+
+.. code-block:: python
+
+    # Use "mypy --warn-unused-ignores ..."
+
+    import sys
+
+    try:
+        # The "[unused-ignore]" is needed to get a clean mypy run
+        # on both Python 3.8, and 3.9 where this module was added
+        import graphlib  # type: ignore[import,unused-ignore]
+    except ImportError:
+        pass
+
+    if sys.version_info >= (3, 9):
+        # The following will not generate an error on either
+        # Python 3.8, or Python 3.9
+        42 + "testing..."  # type: ignore

mypy/applytype.py

@@ -8,6 +8,7 @@
 from mypy.types import (
     AnyType,
     CallableType,
+    Instance,
     Parameters,
     ParamSpecType,
     PartialType,
@@ -75,7 +76,6 @@ def apply_generic_arguments(
     report_incompatible_typevar_value: Callable[[CallableType, Type, str, Context], None],
     context: Context,
     skip_unsatisfied: bool = False,
-    allow_erased_callables: bool = False,
 ) -> CallableType:
     """Apply generic type arguments to a callable type.
 
@@ -119,15 +119,9 @@ def apply_generic_arguments(
         star_index = callable.arg_kinds.index(ARG_STAR)
         callable = callable.copy_modified(
             arg_types=(
-                [
-                    expand_type(at, id_to_type, allow_erased_callables)
-                    for at in callable.arg_types[:star_index]
-                ]
+                [expand_type(at, id_to_type) for at in callable.arg_types[:star_index]]
                 + [callable.arg_types[star_index]]
-                + [
-                    expand_type(at, id_to_type, allow_erased_callables)
-                    for at in callable.arg_types[star_index + 1 :]
-                ]
+                + [expand_type(at, id_to_type) for at in callable.arg_types[star_index + 1 :]]
             )
         )
 
@@ -155,30 +149,38 @@ def apply_generic_arguments(
                 assert False, f"mypy bug: unimplemented case, {expanded_tuple}"
         elif isinstance(unpacked_type, TypeVarTupleType):
             expanded_tvt = expand_unpack_with_variables(var_arg.typ, id_to_type)
-            assert isinstance(expanded_tvt, list)
-            for t in expanded_tvt:
-                assert not isinstance(t, UnpackType)
-            callable = replace_starargs(callable, expanded_tvt)
+            if isinstance(expanded_tvt, list):
+                for t in expanded_tvt:
+                    assert not isinstance(t, UnpackType)
+                callable = replace_starargs(callable, expanded_tvt)
+            else:
+                assert isinstance(expanded_tvt, Instance)
+                assert expanded_tvt.type.fullname == "builtins.tuple"
+                callable = callable.copy_modified(
+                    arg_types=(
+                        callable.arg_types[:star_index]
+                        + [expanded_tvt.args[0]]
+                        + callable.arg_types[star_index + 1 :]
+                    )
+                )
         else:
             assert False, "mypy bug: unhandled case applying unpack"
     else:
         callable = callable.copy_modified(
-            arg_types=[
-                expand_type(at, id_to_type, allow_erased_callables) for at in callable.arg_types
-            ]
+            arg_types=[expand_type(at, id_to_type) for at in callable.arg_types]
         )
 
     # Apply arguments to TypeGuard if any.
     if callable.type_guard is not None:
-        type_guard = expand_type(callable.type_guard, id_to_type, allow_erased_callables)
+        type_guard = expand_type(callable.type_guard, id_to_type)
     else:
         type_guard = None
 
     # The callable may retain some type vars if only some were applied.
     remaining_tvars = [tv for tv in tvars if tv.id not in id_to_type]
 
     return callable.copy_modified(
-        ret_type=expand_type(callable.ret_type, id_to_type, allow_erased_callables),
+        ret_type=expand_type(callable.ret_type, id_to_type),
         variables=remaining_tvars,
         type_guard=type_guard,
     )

mypy/binder.py

@@ -51,6 +51,9 @@ def __init__(self, id: int, conditional_frame: bool = False) -> None:
         # need this field.
         self.suppress_unreachable_warnings = False
 
+    def __repr__(self) -> str:
+        return f"Frame({self.id}, {self.types}, {self.unreachable}, {self.conditional_frame})"
+
 
 Assigns = DefaultDict[Expression, List[Tuple[Type, Optional[Type]]]]
 
@@ -63,7 +66,7 @@ class ConditionalTypeBinder:
 
     ```
     class A:
-        a = None          # type: Union[int, str]
+        a: Union[int, str] = None
     x = A()
     lst = [x]
     reveal_type(x.a)      # Union[int, str]
@@ -446,6 +449,7 @@ def top_frame_context(self) -> Iterator[Frame]:
         assert len(self.frames) == 1
         yield self.push_frame()
         self.pop_frame(True, 0)
+        assert len(self.frames) == 1
 
 
 def get_declaration(expr: BindableExpression) -> Type | None:

mypy/build.py

@@ -2238,6 +2238,7 @@ def semantic_analysis_pass1(self) -> None:
         analyzer = SemanticAnalyzerPreAnalysis()
         with self.wrap_context():
             analyzer.visit_file(self.tree, self.xpath, self.id, options)
+        self.manager.errors.set_unreachable_lines(self.xpath, self.tree.unreachable_lines)
         # TODO: Do this while constructing the AST?
         self.tree.names = SymbolTable()
         if not self.tree.is_stub:
@@ -2572,7 +2573,10 @@ def dependency_lines(self) -> list[int]:
         return [self.dep_line_map.get(dep, 1) for dep in self.dependencies + self.suppressed]
 
     def generate_unused_ignore_notes(self) -> None:
-        if self.options.warn_unused_ignores:
+        if (
+            self.options.warn_unused_ignores
+            or codes.UNUSED_IGNORE in self.options.enabled_error_codes
+        ) and codes.UNUSED_IGNORE not in self.options.disabled_error_codes:
             # If this file was initially loaded from the cache, it may have suppressed
             # dependencies due to imports with ignores on them. We need to generate
             # those errors to avoid spuriously flagging them as unused ignores.

mypy/checker.py

@@ -26,7 +26,7 @@
 
 import mypy.checkexpr
 from mypy import errorcodes as codes, message_registry, nodes, operators
-from mypy.binder import ConditionalTypeBinder, get_declaration
+from mypy.binder import ConditionalTypeBinder, Frame, get_declaration
 from mypy.checkmember import (
     MemberContext,
     analyze_decorator_or_funcbase_access,
@@ -41,7 +41,7 @@
 from mypy.errors import Errors, ErrorWatcher, report_internal_error
 from mypy.expandtype import expand_self_type, expand_type, expand_type_by_instance
 from mypy.join import join_types
-from mypy.literals import Key, literal, literal_hash
+from mypy.literals import Key, extract_var_from_literal_hash, literal, literal_hash
 from mypy.maptype import map_instance_to_supertype
 from mypy.meet import is_overlapping_erased_types, is_overlapping_types
 from mypy.message_registry import ErrorMessage
@@ -134,6 +134,7 @@
     is_final_node,
 )
 from mypy.options import Options
+from mypy.patterns import AsPattern, StarredPattern
 from mypy.plugin import CheckerPluginInterface, Plugin
 from mypy.scope import Scope
 from mypy.semanal import is_trivial_body, refers_to_fullname, set_callable_name
@@ -151,7 +152,7 @@
     restrict_subtype_away,
     unify_generic_callable,
 )
-from mypy.traverser import all_return_statements, has_return_statement
+from mypy.traverser import TraverserVisitor, all_return_statements, has_return_statement
 from mypy.treetransform import TransformVisitor
 from mypy.typeanal import check_for_explicit_any, has_any_from_unimported_type, make_optional_type
 from mypy.typeops import (
@@ -1207,6 +1208,21 @@ def check_func_def(
 
             # Type check body in a new scope.
             with self.binder.top_frame_context():
+                # Copy some type narrowings from an outer function when it seems safe enough
+                # (i.e. we can't find an assignment that might change the type of the
+                # variable afterwards).
+                new_frame: Frame | None = None
+                for frame in old_binder.frames:
+                    for key, narrowed_type in frame.types.items():
+                        key_var = extract_var_from_literal_hash(key)
+                        if key_var is not None and not self.is_var_redefined_in_outer_context(
+                            key_var, defn.line
+                        ):
+                            # It seems safe to propagate the type narrowing to a nested scope.
+                            if new_frame is None:
+                                new_frame = self.binder.push_frame()
+                            new_frame.types[key] = narrowed_type
+                            self.binder.declarations[key] = old_binder.declarations[key]
                 with self.scope.push_function(defn):
                     # We suppress reachability warnings when we use TypeVars with value
                     # restrictions: we only want to report a warning if a certain statement is
@@ -1218,6 +1234,8 @@ def check_func_def(
                         self.binder.suppress_unreachable_warnings()
                     self.accept(item.body)
                 unreachable = self.binder.is_unreachable()
+                if new_frame is not None:
+                    self.binder.pop_frame(True, 0)
 
             if not unreachable:
                 if defn.is_generator or is_named_instance(
@@ -1310,6 +1328,23 @@ def check_func_def(
 
             self.binder = old_binder
 
+    def is_var_redefined_in_outer_context(self, v: Var, after_line: int) -> bool:
+        """Can the variable be assigned to at module top level or outer function?
+
+        Note that this doesn't do a full CFG analysis but uses a line number based
+        heuristic that isn't correct in some (rare) cases.
+        """
+        outers = self.tscope.outer_functions()
+        if not outers:
+            # Top-level function -- outer context is top level, and we can't reason about
+            # globals
+            return True
+        for outer in outers:
+            if isinstance(outer, FuncDef):
+                if find_last_var_assignment_line(outer.body, v) >= after_line:
+                    return True
+        return False
+
     def check_unbound_return_typevar(self, typ: CallableType) -> None:
         """Fails when the return typevar is not defined in arguments."""
         if isinstance(typ.ret_type, TypeVarType) and typ.ret_type in typ.variables:
@@ -7630,3 +7665,80 @@ def collapse_walrus(e: Expression) -> Expression:
     if isinstance(e, AssignmentExpr):
         return e.target
     return e
+
+
+def find_last_var_assignment_line(n: Node, v: Var) -> int:
+    """Find the highest line number of a potential assignment to variable within node.
+
+    This supports local and global variables.
+
+    Return -1 if no assignment was found.
+    """
+    visitor = VarAssignVisitor(v)
+    n.accept(visitor)
+    return visitor.last_line
+
+
+class VarAssignVisitor(TraverserVisitor):
+    def __init__(self, v: Var) -> None:
+        self.last_line = -1
+        self.lvalue = False
+        self.var_node = v
+
+    def visit_assignment_stmt(self, s: AssignmentStmt) -> None:
+        self.lvalue = True
+        for lv in s.lvalues:
+            lv.accept(self)
+        self.lvalue = False
+
+    def visit_name_expr(self, e: NameExpr) -> None:
+        if self.lvalue and e.node is self.var_node:
+            self.last_line = max(self.last_line, e.line)
+
+    def visit_member_expr(self, e: MemberExpr) -> None:
+        old_lvalue = self.lvalue
+        self.lvalue = False
+        super().visit_member_expr(e)
+        self.lvalue = old_lvalue
+
+    def visit_index_expr(self, e: IndexExpr) -> None:
+        old_lvalue = self.lvalue
+        self.lvalue = False
+        super().visit_index_expr(e)
+        self.lvalue = old_lvalue
+
+    def visit_with_stmt(self, s: WithStmt) -> None:
+        self.lvalue = True
+        for lv in s.target:
+            if lv is not None:
+                lv.accept(self)
+        self.lvalue = False
+        s.body.accept(self)
+
+    def visit_for_stmt(self, s: ForStmt) -> None:
+        self.lvalue = True
+        s.index.accept(self)
+        self.lvalue = False
+        s.body.accept(self)
+        if s.else_body:
+            s.else_body.accept(self)
+
+    def visit_assignment_expr(self, e: AssignmentExpr) -> None:
+        self.lvalue = True
+        e.target.accept(self)
+        self.lvalue = False
+        e.value.accept(self)
+
+    def visit_as_pattern(self, p: AsPattern) -> None:
+        if p.pattern is not None:
+            p.pattern.accept(self)
+        if p.name is not None:
+            self.lvalue = True
+            p.name.accept(self)
+            self.lvalue = False
+
+    def visit_starred_pattern(self, p: StarredPattern) -> None:
+        if p.capture is not None:
+            self.lvalue = True
+            p.capture.accept(self)
+            self.lvalue = False