From 0760f2727c3fc0f098ac120e22edd4e767332b94 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 18 Sep 2018 20:48:24 +0100 Subject: [PATCH 1/9] Defer subclass methods if superclass has not been analyzed --- mypy/checker.py | 46 ++++++++++++++++++------------- mypy/semanal.py | 2 +- mypy/semanal_pass3.py | 2 +- mypy/server/aststrip.py | 2 +- mypy/server/update.py | 5 ++-- test-data/unit/check-classes.test | 36 ++++++++++++++++++++++++ 6 files changed, 69 insertions(+), 24 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 64b3313d40bd..00a62500a237 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -72,6 +72,7 @@ DEFAULT_LAST_PASS = 1 # type: Final # Pass numbers start at 0 +DeferredNodeType = Union[FuncDef, LambdaExpr, MypyFile, OverloadedFuncDef, Decorator] # A node which is postponed to be processed during the next pass. # This is used for both batch mode and fine-grained incremental mode. @@ -79,7 +80,7 @@ 'DeferredNode', [ # In batch mode only FuncDef and LambdaExpr are supported - ('node', Union[FuncDef, LambdaExpr, MypyFile, OverloadedFuncDef]), + ('node', DeferredNodeType), ('context_type_name', Optional[str]), # Name of the surrounding class (for error messages) ('active_typeinfo', Optional[TypeInfo]), # And its TypeInfo (for semantic analysis # self type handling) @@ -300,7 +301,7 @@ def check_second_pass(self, todo: Optional[List[DeferredNode]] = None) -> bool: else: assert not self.deferred_nodes self.deferred_nodes = [] - done = set() # type: Set[Union[FuncDef, LambdaExpr, MypyFile, OverloadedFuncDef]] + done = set() # type: Set[DeferredNodeType] for node, type_name, active_typeinfo in todo: if node in done: continue @@ -314,10 +315,7 @@ def check_second_pass(self, todo: Optional[List[DeferredNode]] = None) -> bool: self.tscope.leave() return True - def check_partial(self, node: Union[FuncDef, - LambdaExpr, - MypyFile, - OverloadedFuncDef]) -> None: + def check_partial(self, node: DeferredNodeType) -> None: if isinstance(node, MypyFile): self.check_top_level(node) else: @@ -338,6 +336,14 @@ def check_top_level(self, node: MypyFile) -> None: assert not self.current_node_deferred # TODO: Handle __all__ + def defer_node(self, node: DeferredNodeType, enclosing_class: TypeInfo) -> None: + if self.errors.type_name: + type_name = self.errors.type_name[-1] + else: + type_name = None + # Shouldn't we freeze the entire scope? + self.deferred_nodes.append(DeferredNode(node, type_name, enclosing_class)) + def handle_cannot_determine_type(self, name: str, context: Context) -> None: node = self.scope.top_non_lambda_function() if self.pass_num < self.last_pass and isinstance(node, FuncDef): @@ -345,13 +351,8 @@ def handle_cannot_determine_type(self, name: str, context: Context) -> None: # lambdas because they are coupled to the surrounding function # through the binder and the inferred type of the lambda, so it # would get messy. - if self.errors.type_name: - type_name = self.errors.type_name[-1] - else: - type_name = None - # Shouldn't we freeze the entire scope? enclosing_class = self.scope.enclosing_class() - self.deferred_nodes.append(DeferredNode(node, type_name, enclosing_class)) + self.defer_node(node, enclosing_class) # Set a marker so that we won't infer additional types in this # function. Any inferred types could be bogus, because there's at # least one type that we don't know. @@ -1257,10 +1258,11 @@ def check_method_override(self, defn: Union[FuncBase, Decorator]) -> None: """Check if function definition is compatible with base classes.""" # Check against definitions in base classes. for base in defn.info.mro[1:]: - self.check_method_or_accessor_override_for_base(defn, base) + if self.check_method_or_accessor_override_for_base(defn, base): + return def check_method_or_accessor_override_for_base(self, defn: Union[FuncBase, Decorator], - base: TypeInfo) -> None: + base: TypeInfo) -> bool: """Check if method definition is compatible with a base class.""" if base: name = defn.name() @@ -1277,7 +1279,8 @@ def check_method_or_accessor_override_for_base(self, defn: Union[FuncBase, Decor if name not in ('__init__', '__new__', '__init_subclass__'): # Check method override # (__init__, __new__, __init_subclass__ are special). - self.check_method_override_for_base_with_name(defn, name, base) + if self.check_method_override_for_base_with_name(defn, name, base): + return True if name in nodes.inplace_operator_methods: # Figure out the name of the corresponding operator method. method = '__' + name[3:] @@ -1285,11 +1288,11 @@ def check_method_or_accessor_override_for_base(self, defn: Union[FuncBase, Decor # always introduced safely if a base class defined __add__. # TODO can't come up with an example where this is # necessary; now it's "just in case" - self.check_method_override_for_base_with_name(defn, method, - base) + return self.check_method_override_for_base_with_name(defn, method, + base) def check_method_override_for_base_with_name( - self, defn: Union[FuncBase, Decorator], name: str, base: TypeInfo) -> None: + self, defn: Union[FuncBase, Decorator], name: str, base: TypeInfo) -> bool: base_attr = base.names.get(name) if base_attr: # The name of the method is defined in the base class. @@ -1317,7 +1320,11 @@ def check_method_override_for_base_with_name( original_type = base_attr.type original_node = base_attr.node if original_type is None: - if isinstance(original_node, FuncBase): + if self.pass_num < self.last_pass: + assert isinstance(defn, (FuncDef, OverloadedFuncDef, Decorator)) + self.defer_node(defn, defn.info) + return True + elif isinstance(original_node, FuncBase): original_type = self.function_type(original_node) elif isinstance(original_node, Decorator): original_type = self.function_type(original_node.func) @@ -1359,6 +1366,7 @@ def check_method_override_for_base_with_name( else: self.msg.signature_incompatible_with_supertype( defn.name(), name, base.name(), context) + return False def get_op_other_domain(self, tp: FunctionLike) -> Optional[Type]: if isinstance(tp, CallableType): diff --git a/mypy/semanal.py b/mypy/semanal.py index 1e24dd3ea1c1..08b12ba6457c 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -317,7 +317,7 @@ def visit_file(self, file_node: MypyFile, fnam: str, options: Options, del self.cur_mod_node del self.globals - def refresh_partial(self, node: Union[MypyFile, FuncItem, OverloadedFuncDef], + def refresh_partial(self, node: Union[MypyFile, FuncItem, OverloadedFuncDef, Decorator], patches: List[Tuple[int, Callable[[], None]]]) -> None: """Refresh a stale target in fine-grained incremental mode.""" self.patches = patches diff --git a/mypy/semanal_pass3.py b/mypy/semanal_pass3.py index c00da4d45f99..389071fb2033 100644 --- a/mypy/semanal_pass3.py +++ b/mypy/semanal_pass3.py @@ -73,7 +73,7 @@ def visit_file(self, file_node: MypyFile, fnam: str, options: Options, del self.cur_mod_node self.patches = [] - def refresh_partial(self, node: Union[MypyFile, FuncItem, OverloadedFuncDef], + def refresh_partial(self, node: Union[MypyFile, FuncItem, OverloadedFuncDef, Decorator], patches: List[Tuple[int, Callable[[], None]]]) -> None: """Refresh a stale target in fine-grained incremental mode.""" self.options = self.sem.options diff --git a/mypy/server/aststrip.py b/mypy/server/aststrip.py index 5314ca7f3f5b..cbfbe0d462e2 100644 --- a/mypy/server/aststrip.py +++ b/mypy/server/aststrip.py @@ -52,7 +52,7 @@ from mypy.typestate import TypeState -def strip_target(node: Union[MypyFile, FuncItem, OverloadedFuncDef]) -> None: +def strip_target(node: Union[MypyFile, FuncItem, OverloadedFuncDef, Decorator]) -> None: """Reset a fine-grained incremental target to state after semantic analysis pass 1. NOTE: Currently we opportunistically only reset changes that are known to otherwise diff --git a/mypy/server/update.py b/mypy/server/update.py index 9c306ca89f25..76fcd402dc38 100644 --- a/mypy/server/update.py +++ b/mypy/server/update.py @@ -1058,7 +1058,7 @@ def is_verbose(manager: BuildManager) -> bool: def target_from_node(module: str, - node: Union[FuncDef, MypyFile, OverloadedFuncDef, LambdaExpr] + node: Union[FuncDef, MypyFile, OverloadedFuncDef, LambdaExpr, Decorator] ) -> Optional[str]: """Return the target name corresponding to a deferred node. @@ -1079,4 +1079,5 @@ def target_from_node(module: str, else: return '%s.%s' % (module, node.name()) else: - assert False, "Lambda expressions can't be deferred in fine-grained incremental mode" + assert False, "Lambda expressions and decorators can't be deferred in" \ + " fine-grained incremental mode" diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index 14ed21207a00..ec9e775ee011 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -5117,3 +5117,39 @@ class C: def x(self) -> int: pass [builtins fixtures/property.pyi] [out] + +[case testCyclicDecorator] +import b +[file a.py] +import b +import c + +class A(b.B): + @c.deco + def meth(self) -> int: ... +[file b.py] +import a +import c + +class B: + @c.deco + def meth(self) -> int: ... +[file c.py] +from typing import TypeVar, Tuple, Callable +T = TypeVar('T') +def deco(f: Callable[..., T]) -> Callable[..., Tuple[T, int]]: ... +[out] + +[case testCyclicOverride] +import a +[file b.py] +import a +class Sub(a.Base): + def x(self) -> int: pass + +[file a.py] +import b +class Base: + def __init__(self): + self.x = 1 +[out] From 1fe32b59540d3bb59c3b0b79fd8dd85fea4d5d68 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 18 Sep 2018 20:54:54 +0100 Subject: [PATCH 2/9] Fix self-check --- mypy/checker.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mypy/checker.py b/mypy/checker.py index 00a62500a237..1a1d973868bb 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -336,7 +336,7 @@ def check_top_level(self, node: MypyFile) -> None: assert not self.current_node_deferred # TODO: Handle __all__ - def defer_node(self, node: DeferredNodeType, enclosing_class: TypeInfo) -> None: + def defer_node(self, node: DeferredNodeType, enclosing_class: Optional[TypeInfo]) -> None: if self.errors.type_name: type_name = self.errors.type_name[-1] else: @@ -1290,6 +1290,7 @@ def check_method_or_accessor_override_for_base(self, defn: Union[FuncBase, Decor # necessary; now it's "just in case" return self.check_method_override_for_base_with_name(defn, method, base) + return False def check_method_override_for_base_with_name( self, defn: Union[FuncBase, Decorator], name: str, base: TypeInfo) -> bool: From b03279afa2bb922e6cad7709458dd53951b06a5b Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 18 Sep 2018 21:13:23 +0100 Subject: [PATCH 3/9] Update docstring --- mypy/checker.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mypy/checker.py b/mypy/checker.py index 1a1d973868bb..31692f770173 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -1263,7 +1263,11 @@ def check_method_override(self, defn: Union[FuncBase, Decorator]) -> None: def check_method_or_accessor_override_for_base(self, defn: Union[FuncBase, Decorator], base: TypeInfo) -> bool: - """Check if method definition is compatible with a base class.""" + """Check if method definition is compatible with a base class. + + Return True if the node was deferred because one of the corresponding + superclass nodes is not ready. + """ if base: name = defn.name() base_attr = base.names.get(name) From 62a15562a5a3259518a6a29b71bc982f68d740fa Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 19 Sep 2018 16:24:08 +0100 Subject: [PATCH 4/9] Add some tests --- test-data/unit/check-classes.test | 192 +++++++++++++++++++++++++++++- 1 file changed, 191 insertions(+), 1 deletion(-) diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index ec9e775ee011..496ca220f014 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -5140,7 +5140,62 @@ T = TypeVar('T') def deco(f: Callable[..., T]) -> Callable[..., Tuple[T, int]]: ... [out] -[case testCyclicOverride] +[case testCyclicOverload] +import b +[file a.pyi] +import b +from typing import overload + +class A(b.B): + @overload + def meth(self, x: int) -> int: ... + @overload + def meth(self, x: str) -> str: ... +[file b.pyi] +import a +from typing import overload + +class B: + @overload + def meth(self, x: int) -> int: ... + @overload + def meth(self, x: str) -> str: ... +[out] + +[case testCyclicOverloadDoubleDeferred] +import b +[file a.py] +import b +from typing import overload, Union + +class A(b.B): + @overload + def meth(self, x: int) -> int: ... + @overload + def meth(self, x: str) -> str: ... + def meth(self, x) -> Union[int, str]: + reveal_type(other.x) # E: Revealed type is 'builtins.int' + return 0 + +other: Other +class Other: + def __init__(self) -> None: + self.x = f() +def f() -> int: ... +[file b.py] +import a +from typing import overload + +class B: + @overload + def meth(self, x: int) -> int: ... + @overload + def meth(self, x: str) -> str: ... + def meth(self, x): + pass +[out] + +[case testCyclicOverrideAny] import a [file b.py] import a @@ -5153,3 +5208,138 @@ class Base: def __init__(self): self.x = 1 [out] + +[case testCyclicOverrideChecked] +import a +[file b.py] +import a +class Sub(a.Base): + def x(self) -> int: pass # E: Signature of "x" incompatible with supertype "Base" + +[file a.py] +import b +class Base: + def __init__(self) -> None: + self.x = 1 +[out] + +[case testCyclicDecoratorDoubleDeferred] +import b +[file a.py] +import b +import c + +class A(b.B): + @c.deco + def meth(self) -> int: + reveal_type(other.x) # E: Revealed type is 'builtins.int' + return 0 + +other: Other +class Other: + def __init__(self) -> None: + self.x = f() +def f() -> int: ... +[file b.py] +import a +import c + +class B: + @c.deco + def meth(self) -> int: + pass +[file c.py] +from typing import TypeVar, Tuple, Callable +T = TypeVar('T') +def deco(f: Callable[..., T]) -> Callable[..., Tuple[T, int]]: ... +[out] + +[case testCyclicDecoratorSuper] +import b +[file a.py] +import b +import c + +class A(b.B): + @c.deco + def meth(self) -> int: + y = super().meth() + reveal_type(y) # E: Revealed type is 'Tuple[builtins.int*, builtins.int]' + return 0 +[file b.py] +import a +import c + +class B: + @c.deco + def meth(self) -> int: + pass +[file c.py] +from typing import TypeVar, Tuple, Callable +T = TypeVar('T') +def deco(f: Callable[..., T]) -> Callable[..., Tuple[T, int]]: ... +[out] + +[case testCyclicDecoratorBothDeferred] +import b +[file a.py] +import b +import c + +class A(b.B): + @c.deco + def meth(self) -> int: + pass +[file b.py] +import a +import c + +class B: + @c.deco + def meth(self) -> int: + reveal_type(other.x) # E: Revealed type is 'builtins.int' + return 0 + +other: Other +class Other: + def __init__(self) -> None: + self.x = f() +def f() -> int: ... +[file c.py] +from typing import TypeVar, Tuple, Callable +T = TypeVar('T') +def deco(f: Callable[..., T]) -> Callable[..., Tuple[T, int]]: ... +[out] + +[case testCyclicDecoratorSuperDeferred] +import b +[file a.py] +import b +import c + +class A(b.B): + @c.deco + def meth(self) -> int: + y = super().meth() + reveal_type(y) # E: Revealed type is 'Tuple[builtins.int*, builtins.int]' + reveal_type(other.x) # E: Revealed type is 'builtins.int' + return 0 + +other: Other +class Other: + def __init__(self) -> None: + self.x = f() +def f() -> int: ... +[file b.py] +import a +import c + +class B: + @c.deco + def meth(self) -> int: + pass +[file c.py] +from typing import TypeVar, Tuple, Callable +T = TypeVar('T') +def deco(f: Callable[..., T]) -> Callable[..., Tuple[T, int]]: ... +[out] From 8f6b4c98df2fa8e0a8b422ef9a8907a626e34e62 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 19 Sep 2018 16:25:38 +0100 Subject: [PATCH 5/9] Add some tests --- test-data/unit/check-classes.test | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index 496ca220f014..4121bd824d45 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -5223,6 +5223,27 @@ class Base: self.x = 1 [out] +[case testCyclicOverrideCheckedDecorator] +import a +[file b.py] +import a +import c +class Sub(a.Base): + @c.deco + def x(self) -> int: pass # E: Signature of "x" incompatible with supertype "Base" + +[file a.py] +import b +import c +class Base: + def __init__(self) -> None: + self.x = 1 +[file c.py] +from typing import TypeVar, Tuple, Callable +T = TypeVar('T') +def deco(f: Callable[..., T]) -> Callable[..., Tuple[T, int]]: ... +[out] + [case testCyclicDecoratorDoubleDeferred] import b [file a.py] From 002ca6781a99856ff2eba565307f2a9cc1d29c4e Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 19 Sep 2018 16:29:56 +0100 Subject: [PATCH 6/9] Add even more tests (overriding a var) --- test-data/unit/check-classes.test | 45 +++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index 4121bd824d45..a0000ba12e84 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -5244,6 +5244,51 @@ T = TypeVar('T') def deco(f: Callable[..., T]) -> Callable[..., Tuple[T, int]]: ... [out] +[case testCyclicOverrideCheckedDecoratorDeferred] +import a +[file b.py] +import a +import c +class Sub(a.Base): + @c.deco + def x(self) -> int: pass # E: Signature of "x" incompatible with supertype "Base" + +[file a.py] +import b +import c +class Base: + def __init__(self) -> None: + self.x = f() + +def f() -> int: ... +[file c.py] +from typing import TypeVar, Tuple, Callable +T = TypeVar('T') +def deco(f: Callable[..., T]) -> Callable[..., Tuple[T, int]]: ... +[out] + +[case testCyclicOverrideAnyDecoratorDeferred] +import a +[file b.py] +import a +import c +class Sub(a.Base): + @c.deco + def x(self) -> int: pass + +[file a.py] +import b +import c +class Base: + def __init__(self) -> None: + self.x = f() + +def f() -> int: ... +[file c.py] +from typing import Any, Callable +def deco(f: Callable[..., Any]) -> Any: ... +[out] + [case testCyclicDecoratorDoubleDeferred] import b [file a.py] From e11822b447eb4b99692d4df85a42781376475514 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 19 Sep 2018 17:04:05 +0100 Subject: [PATCH 7/9] Separate fine and coarse grained deferred nodes --- mypy/checker.py | 44 ++++++++++++++++++++++++++++------------- mypy/semanal.py | 2 +- mypy/semanal_pass3.py | 2 +- mypy/server/aststrip.py | 2 +- mypy/server/update.py | 25 +++++++++++------------ 5 files changed, 44 insertions(+), 31 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 31692f770173..f6e722cb12d7 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -5,7 +5,8 @@ from contextlib import contextmanager from typing import ( - Dict, Set, List, cast, Tuple, TypeVar, Union, Optional, NamedTuple, Iterator, Iterable, Any + Dict, Set, List, cast, Tuple, TypeVar, Union, Optional, NamedTuple, Iterator, Iterable, + Sequence ) from mypy.errors import Errors, report_internal_error @@ -72,20 +73,28 @@ DEFAULT_LAST_PASS = 1 # type: Final # Pass numbers start at 0 -DeferredNodeType = Union[FuncDef, LambdaExpr, MypyFile, OverloadedFuncDef, Decorator] +DeferredNodeType = Union[FuncDef, LambdaExpr, OverloadedFuncDef, Decorator] +FineDeferredNodeType = Union[FuncDef, MypyFile, OverloadedFuncDef] # A node which is postponed to be processed during the next pass. -# This is used for both batch mode and fine-grained incremental mode. DeferredNode = NamedTuple( 'DeferredNode', [ - # In batch mode only FuncDef and LambdaExpr are supported ('node', DeferredNodeType), ('context_type_name', Optional[str]), # Name of the surrounding class (for error messages) ('active_typeinfo', Optional[TypeInfo]), # And its TypeInfo (for semantic analysis # self type handling) ]) +# Same as above, but for fine grained mode targets. Only top-level functions/methods +# and module top levels are allowed as such. +FineDeferredNode = NamedTuple( + 'FineDeferredNode', + [ + ('node', FineDeferredNodeType), + ('context_type_name', Optional[str]), + ('active_typeinfo', Optional[TypeInfo]), + ]) # Data structure returned by find_isinstance_check representing # information learned from the truth or falsehood of a condition. The @@ -284,7 +293,8 @@ def check_first_pass(self) -> None: self.tscope.leave() - def check_second_pass(self, todo: Optional[List[DeferredNode]] = None) -> bool: + def check_second_pass(self, todo: Optional[Sequence[Union[DeferredNode, + FineDeferredNode]]] = None) -> bool: """Run second or following pass of type checking. This goes through deferred nodes, returning True if there were any. @@ -301,7 +311,7 @@ def check_second_pass(self, todo: Optional[List[DeferredNode]] = None) -> bool: else: assert not self.deferred_nodes self.deferred_nodes = [] - done = set() # type: Set[DeferredNodeType] + done = set() # type: Set[Union[DeferredNodeType, FineDeferredNodeType]] for node, type_name, active_typeinfo in todo: if node in done: continue @@ -315,7 +325,7 @@ def check_second_pass(self, todo: Optional[List[DeferredNode]] = None) -> bool: self.tscope.leave() return True - def check_partial(self, node: DeferredNodeType) -> None: + def check_partial(self, node: Union[DeferredNodeType, FineDeferredNodeType]) -> None: if isinstance(node, MypyFile): self.check_top_level(node) else: @@ -1254,14 +1264,16 @@ def expand_typevars(self, defn: FuncItem, else: return [(defn, typ)] - def check_method_override(self, defn: Union[FuncBase, Decorator]) -> None: + def check_method_override(self, defn: Union[FuncDef, OverloadedFuncDef, Decorator]) -> None: """Check if function definition is compatible with base classes.""" # Check against definitions in base classes. for base in defn.info.mro[1:]: if self.check_method_or_accessor_override_for_base(defn, base): return - def check_method_or_accessor_override_for_base(self, defn: Union[FuncBase, Decorator], + def check_method_or_accessor_override_for_base(self, defn: Union[FuncDef, + OverloadedFuncDef, + Decorator], base: TypeInfo) -> bool: """Check if method definition is compatible with a base class. @@ -1297,7 +1309,12 @@ def check_method_or_accessor_override_for_base(self, defn: Union[FuncBase, Decor return False def check_method_override_for_base_with_name( - self, defn: Union[FuncBase, Decorator], name: str, base: TypeInfo) -> bool: + self, defn: Union[FuncDef, OverloadedFuncDef, Decorator], + name: str, base: TypeInfo) -> bool: + """Check if overriding an attribute `name` of `base` with `defn` is valid. + + Return True if the supertype node was not analysed yet, and `defn` was deferred. + """ base_attr = base.names.get(name) if base_attr: # The name of the method is defined in the base class. @@ -1310,7 +1327,7 @@ def check_method_override_for_base_with_name( context = defn.func # Construct the type of the overriding method. - if isinstance(defn, FuncBase): + if isinstance(defn, (FuncDef, OverloadedFuncDef)): typ = self.function_type(defn) # type: Type override_class_or_static = defn.is_class or defn.is_static else: @@ -1326,16 +1343,15 @@ def check_method_override_for_base_with_name( original_node = base_attr.node if original_type is None: if self.pass_num < self.last_pass: - assert isinstance(defn, (FuncDef, OverloadedFuncDef, Decorator)) self.defer_node(defn, defn.info) return True - elif isinstance(original_node, FuncBase): + elif isinstance(original_node, (FuncDef, OverloadedFuncDef)): original_type = self.function_type(original_node) elif isinstance(original_node, Decorator): original_type = self.function_type(original_node.func) else: assert False, str(base_attr.node) - if isinstance(original_node, FuncBase): + if isinstance(original_node, (FuncDef, OverloadedFuncDef)): original_class_or_static = original_node.is_class or original_node.is_static elif isinstance(original_node, Decorator): fdef = original_node.func diff --git a/mypy/semanal.py b/mypy/semanal.py index 08b12ba6457c..b9f9328d9496 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -317,7 +317,7 @@ def visit_file(self, file_node: MypyFile, fnam: str, options: Options, del self.cur_mod_node del self.globals - def refresh_partial(self, node: Union[MypyFile, FuncItem, OverloadedFuncDef, Decorator], + def refresh_partial(self, node: Union[MypyFile, FuncDef, OverloadedFuncDef], patches: List[Tuple[int, Callable[[], None]]]) -> None: """Refresh a stale target in fine-grained incremental mode.""" self.patches = patches diff --git a/mypy/semanal_pass3.py b/mypy/semanal_pass3.py index 389071fb2033..9b8fab138c17 100644 --- a/mypy/semanal_pass3.py +++ b/mypy/semanal_pass3.py @@ -73,7 +73,7 @@ def visit_file(self, file_node: MypyFile, fnam: str, options: Options, del self.cur_mod_node self.patches = [] - def refresh_partial(self, node: Union[MypyFile, FuncItem, OverloadedFuncDef, Decorator], + def refresh_partial(self, node: Union[MypyFile, FuncDef, OverloadedFuncDef], patches: List[Tuple[int, Callable[[], None]]]) -> None: """Refresh a stale target in fine-grained incremental mode.""" self.options = self.sem.options diff --git a/mypy/server/aststrip.py b/mypy/server/aststrip.py index cbfbe0d462e2..e109ad4ab4f3 100644 --- a/mypy/server/aststrip.py +++ b/mypy/server/aststrip.py @@ -52,7 +52,7 @@ from mypy.typestate import TypeState -def strip_target(node: Union[MypyFile, FuncItem, OverloadedFuncDef, Decorator]) -> None: +def strip_target(node: Union[MypyFile, FuncDef, OverloadedFuncDef]) -> None: """Reset a fine-grained incremental target to state after semantic analysis pass 1. NOTE: Currently we opportunistically only reset changes that are known to otherwise diff --git a/mypy/server/update.py b/mypy/server/update.py index 76fcd402dc38..2b98683aa6f2 100644 --- a/mypy/server/update.py +++ b/mypy/server/update.py @@ -122,7 +122,7 @@ BuildManager, State, BuildSource, BuildResult, Graph, load_graph, process_fresh_modules, DEBUG_FINE_GRAINED, ) -from mypy.checker import DeferredNode +from mypy.checker import FineDeferredNode from mypy.errors import CompileError from mypy.nodes import ( MypyFile, FuncDef, TypeInfo, SymbolNode, Decorator, @@ -780,7 +780,7 @@ def find_targets_recursive( graph: Graph, triggers: Set[str], deps: Dict[str, Set[str]], - up_to_date_modules: Set[str]) -> Tuple[Dict[str, Set[DeferredNode]], + up_to_date_modules: Set[str]) -> Tuple[Dict[str, Set[FineDeferredNode]], Set[str], Set[TypeInfo]]: """Find names of all targets that need to reprocessed, given some triggers. @@ -788,7 +788,7 @@ def find_targets_recursive( * Dictionary from module id to a set of stale targets. * A set of module ids for unparsed modules with stale targets. """ - result = {} # type: Dict[str, Set[DeferredNode]] + result = {} # type: Dict[str, Set[FineDeferredNode]] worklist = triggers processed = set() # type: Set[str] stale_protos = set() # type: Set[TypeInfo] @@ -834,7 +834,7 @@ def find_targets_recursive( def reprocess_nodes(manager: BuildManager, graph: Dict[str, State], module_id: str, - nodeset: Set[DeferredNode], + nodeset: Set[FineDeferredNode], deps: Dict[str, Set[str]]) -> Set[str]: """Reprocess a set of nodes within a single module. @@ -850,7 +850,7 @@ def reprocess_nodes(manager: BuildManager, old_symbols = {name: names.copy() for name, names in old_symbols.items()} old_symbols_snapshot = snapshot_symbol_table(file_node.fullname(), file_node.names) - def key(node: DeferredNode) -> int: + def key(node: FineDeferredNode) -> int: # Unlike modules which are sorted by name within SCC, # nodes within the same module are sorted by line number, because # this is how they are processed in normal mode. @@ -959,7 +959,7 @@ def find_symbol_tables_recursive(prefix: str, symbols: SymbolTable) -> Dict[str, def update_deps(module_id: str, - nodes: List[DeferredNode], + nodes: List[FineDeferredNode], graph: Dict[str, State], deps: Dict[str, Set[str]], options: Options) -> None: @@ -977,7 +977,7 @@ def update_deps(module_id: str, def lookup_target(manager: BuildManager, - target: str) -> Tuple[List[DeferredNode], Optional[TypeInfo]]: + target: str) -> Tuple[List[FineDeferredNode], Optional[TypeInfo]]: """Look up a target by fully-qualified name. The first item in the return tuple is a list of deferred nodes that @@ -1025,7 +1025,7 @@ def not_found() -> None: # a deserialized TypeInfo with missing attributes. not_found() return [], None - result = [DeferredNode(file, None, None)] + result = [FineDeferredNode(file, None, None)] stale_info = None # type: Optional[TypeInfo] if node.is_protocol: stale_info = node @@ -1050,7 +1050,7 @@ def not_found() -> None: # context will be wrong and it could be a partially initialized deserialized node. not_found() return [], None - return [DeferredNode(node, active_class_name, active_class)], None + return [FineDeferredNode(node, active_class_name, active_class)], None def is_verbose(manager: BuildManager) -> bool: @@ -1058,7 +1058,7 @@ def is_verbose(manager: BuildManager) -> bool: def target_from_node(module: str, - node: Union[FuncDef, MypyFile, OverloadedFuncDef, LambdaExpr, Decorator] + node: Union[FuncDef, MypyFile, OverloadedFuncDef] ) -> Optional[str]: """Return the target name corresponding to a deferred node. @@ -1073,11 +1073,8 @@ def target_from_node(module: str, # Actually a reference to another module -- likely a stale dependency. return None return module - elif isinstance(node, (OverloadedFuncDef, FuncDef)): + else: # OverloadedFuncDef or FuncDef if node.info: return '%s.%s' % (node.info.fullname(), node.name()) else: return '%s.%s' % (module, node.name()) - else: - assert False, "Lambda expressions and decorators can't be deferred in" \ - " fine-grained incremental mode" From 250931c0ba6c81082d4a600b15d4d27ce6037cc5 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 19 Sep 2018 17:27:31 +0100 Subject: [PATCH 8/9] Add some clarifying comments and docstrings --- mypy/checker.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index f6e722cb12d7..56f3603a19ea 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -77,6 +77,9 @@ FineDeferredNodeType = Union[FuncDef, MypyFile, OverloadedFuncDef] # A node which is postponed to be processed during the next pass. +# In normal mode one can defer functions and methods (also decorated and/or overloaded) +# and lambda expressions. Nested functions can't be deferred -- only top-level functions +# and methods of classes not defined within a function can be deferred. DeferredNode = NamedTuple( 'DeferredNode', [ @@ -86,7 +89,7 @@ # self type handling) ]) -# Same as above, but for fine grained mode targets. Only top-level functions/methods +# Same as above, but for fine-grained mode targets. Only top-level functions/methods # and module top levels are allowed as such. FineDeferredNode = NamedTuple( 'FineDeferredNode', @@ -347,11 +350,20 @@ def check_top_level(self, node: MypyFile) -> None: # TODO: Handle __all__ def defer_node(self, node: DeferredNodeType, enclosing_class: Optional[TypeInfo]) -> None: + """Defer a node for processing during next type-checking pass. + + Args: + node: function/method being deferred + enclosing_class: for methods, the class where the method is defined + NOTE: this can't handle nested functions/methods. + """ if self.errors.type_name: type_name = self.errors.type_name[-1] else: type_name = None - # Shouldn't we freeze the entire scope? + # We don't freeze the entire scope since only top-level functions and methods + # can be deferred. Only module/class level scope information is needed. + # Module-level scope information is preserved in the TypeChecker instance. self.deferred_nodes.append(DeferredNode(node, type_name, enclosing_class)) def handle_cannot_determine_type(self, name: str, context: Context) -> None: @@ -1265,10 +1277,14 @@ def expand_typevars(self, defn: FuncItem, return [(defn, typ)] def check_method_override(self, defn: Union[FuncDef, OverloadedFuncDef, Decorator]) -> None: - """Check if function definition is compatible with base classes.""" + """Check if function definition is compatible with base classes. + + This may defer the method if a signature is not available in at least on base class. + """ # Check against definitions in base classes. for base in defn.info.mro[1:]: if self.check_method_or_accessor_override_for_base(defn, base): + # Node was deferred, we will have another attempt later. return def check_method_or_accessor_override_for_base(self, defn: Union[FuncDef, @@ -1343,6 +1359,8 @@ def check_method_override_for_base_with_name( original_node = base_attr.node if original_type is None: if self.pass_num < self.last_pass: + # If there are passes left, defer this node until next pass, + # otherwise try reconstructing the method type from available information. self.defer_node(defn, defn.info) return True elif isinstance(original_node, (FuncDef, OverloadedFuncDef)): From 132b2224992684e50686dfdfcfad1c23d4c77c15 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 20 Sep 2018 13:46:15 +0100 Subject: [PATCH 9/9] Address CR --- mypy/checker.py | 18 ++++++++++-------- mypy/server/update.py | 18 +++++++++--------- test-data/unit/check-classes.test | 12 ++++++------ 3 files changed, 25 insertions(+), 23 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 56f3603a19ea..b01b4bbf984c 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -74,7 +74,7 @@ DEFAULT_LAST_PASS = 1 # type: Final # Pass numbers start at 0 DeferredNodeType = Union[FuncDef, LambdaExpr, OverloadedFuncDef, Decorator] -FineDeferredNodeType = Union[FuncDef, MypyFile, OverloadedFuncDef] +FineGrainedDeferredNodeType = Union[FuncDef, MypyFile, OverloadedFuncDef] # A node which is postponed to be processed during the next pass. # In normal mode one can defer functions and methods (also decorated and/or overloaded) @@ -91,10 +91,10 @@ # Same as above, but for fine-grained mode targets. Only top-level functions/methods # and module top levels are allowed as such. -FineDeferredNode = NamedTuple( +FineGrainedDeferredNode = NamedTuple( 'FineDeferredNode', [ - ('node', FineDeferredNodeType), + ('node', FineGrainedDeferredNodeType), ('context_type_name', Optional[str]), ('active_typeinfo', Optional[TypeInfo]), ]) @@ -296,8 +296,10 @@ def check_first_pass(self) -> None: self.tscope.leave() - def check_second_pass(self, todo: Optional[Sequence[Union[DeferredNode, - FineDeferredNode]]] = None) -> bool: + def check_second_pass(self, + todo: Optional[Sequence[Union[DeferredNode, + FineGrainedDeferredNode]]] = None + ) -> bool: """Run second or following pass of type checking. This goes through deferred nodes, returning True if there were any. @@ -314,7 +316,7 @@ def check_second_pass(self, todo: Optional[Sequence[Union[DeferredNode, else: assert not self.deferred_nodes self.deferred_nodes = [] - done = set() # type: Set[Union[DeferredNodeType, FineDeferredNodeType]] + done = set() # type: Set[Union[DeferredNodeType, FineGrainedDeferredNodeType]] for node, type_name, active_typeinfo in todo: if node in done: continue @@ -328,7 +330,7 @@ def check_second_pass(self, todo: Optional[Sequence[Union[DeferredNode, self.tscope.leave() return True - def check_partial(self, node: Union[DeferredNodeType, FineDeferredNodeType]) -> None: + def check_partial(self, node: Union[DeferredNodeType, FineGrainedDeferredNodeType]) -> None: if isinstance(node, MypyFile): self.check_top_level(node) else: @@ -1279,7 +1281,7 @@ def expand_typevars(self, defn: FuncItem, def check_method_override(self, defn: Union[FuncDef, OverloadedFuncDef, Decorator]) -> None: """Check if function definition is compatible with base classes. - This may defer the method if a signature is not available in at least on base class. + This may defer the method if a signature is not available in at least one base class. """ # Check against definitions in base classes. for base in defn.info.mro[1:]: diff --git a/mypy/server/update.py b/mypy/server/update.py index 2b98683aa6f2..7943ffaee95b 100644 --- a/mypy/server/update.py +++ b/mypy/server/update.py @@ -122,7 +122,7 @@ BuildManager, State, BuildSource, BuildResult, Graph, load_graph, process_fresh_modules, DEBUG_FINE_GRAINED, ) -from mypy.checker import FineDeferredNode +from mypy.checker import FineGrainedDeferredNode from mypy.errors import CompileError from mypy.nodes import ( MypyFile, FuncDef, TypeInfo, SymbolNode, Decorator, @@ -780,7 +780,7 @@ def find_targets_recursive( graph: Graph, triggers: Set[str], deps: Dict[str, Set[str]], - up_to_date_modules: Set[str]) -> Tuple[Dict[str, Set[FineDeferredNode]], + up_to_date_modules: Set[str]) -> Tuple[Dict[str, Set[FineGrainedDeferredNode]], Set[str], Set[TypeInfo]]: """Find names of all targets that need to reprocessed, given some triggers. @@ -788,7 +788,7 @@ def find_targets_recursive( * Dictionary from module id to a set of stale targets. * A set of module ids for unparsed modules with stale targets. """ - result = {} # type: Dict[str, Set[FineDeferredNode]] + result = {} # type: Dict[str, Set[FineGrainedDeferredNode]] worklist = triggers processed = set() # type: Set[str] stale_protos = set() # type: Set[TypeInfo] @@ -834,7 +834,7 @@ def find_targets_recursive( def reprocess_nodes(manager: BuildManager, graph: Dict[str, State], module_id: str, - nodeset: Set[FineDeferredNode], + nodeset: Set[FineGrainedDeferredNode], deps: Dict[str, Set[str]]) -> Set[str]: """Reprocess a set of nodes within a single module. @@ -850,7 +850,7 @@ def reprocess_nodes(manager: BuildManager, old_symbols = {name: names.copy() for name, names in old_symbols.items()} old_symbols_snapshot = snapshot_symbol_table(file_node.fullname(), file_node.names) - def key(node: FineDeferredNode) -> int: + def key(node: FineGrainedDeferredNode) -> int: # Unlike modules which are sorted by name within SCC, # nodes within the same module are sorted by line number, because # this is how they are processed in normal mode. @@ -959,7 +959,7 @@ def find_symbol_tables_recursive(prefix: str, symbols: SymbolTable) -> Dict[str, def update_deps(module_id: str, - nodes: List[FineDeferredNode], + nodes: List[FineGrainedDeferredNode], graph: Dict[str, State], deps: Dict[str, Set[str]], options: Options) -> None: @@ -977,7 +977,7 @@ def update_deps(module_id: str, def lookup_target(manager: BuildManager, - target: str) -> Tuple[List[FineDeferredNode], Optional[TypeInfo]]: + target: str) -> Tuple[List[FineGrainedDeferredNode], Optional[TypeInfo]]: """Look up a target by fully-qualified name. The first item in the return tuple is a list of deferred nodes that @@ -1025,7 +1025,7 @@ def not_found() -> None: # a deserialized TypeInfo with missing attributes. not_found() return [], None - result = [FineDeferredNode(file, None, None)] + result = [FineGrainedDeferredNode(file, None, None)] stale_info = None # type: Optional[TypeInfo] if node.is_protocol: stale_info = node @@ -1050,7 +1050,7 @@ def not_found() -> None: # context will be wrong and it could be a partially initialized deserialized node. not_found() return [], None - return [FineDeferredNode(node, active_class_name, active_class)], None + return [FineGrainedDeferredNode(node, active_class_name, active_class)], None def is_verbose(manager: BuildManager) -> bool: diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index a0000ba12e84..f1e0f7c7c26e 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -5162,7 +5162,7 @@ class B: def meth(self, x: str) -> str: ... [out] -[case testCyclicOverloadDoubleDeferred] +[case testCyclicOverloadDeferred] import b [file a.py] import b @@ -5277,7 +5277,7 @@ class Sub(a.Base): def x(self) -> int: pass [file a.py] -import b +from b import Sub import c class Base: def __init__(self) -> None: @@ -5307,7 +5307,7 @@ class Other: self.x = f() def f() -> int: ... [file b.py] -import a +from a import A import c class B: @@ -5333,7 +5333,7 @@ class A(b.B): reveal_type(y) # E: Revealed type is 'Tuple[builtins.int*, builtins.int]' return 0 [file b.py] -import a +from a import A import c class B: @@ -5357,7 +5357,7 @@ class A(b.B): def meth(self) -> int: pass [file b.py] -import a +from a import A import c class B: @@ -5397,7 +5397,7 @@ class Other: self.x = f() def f() -> int: ... [file b.py] -import a +from a import A import c class B: