Fix crashes and fails in forward references

mypy/build.py

@@ -491,7 +491,7 @@ def __init__(self, data_dir: str,
         self.semantic_analyzer = SemanticAnalyzer(self.modules, self.missing_modules,
                                                   lib_path, self.errors, self.plugin)
         self.modules = self.semantic_analyzer.modules
-        self.semantic_analyzer_pass3 = ThirdPass(self.modules, self.errors)
+        self.semantic_analyzer_pass3 = ThirdPass(self.modules, self.errors, self.semantic_analyzer)
         self.all_types = {}  # type: Dict[Expression, Type]
         self.indirection_detector = TypeIndirectionVisitor()
         self.stale_modules = set()  # type: Set[str]
@@ -1722,10 +1722,13 @@ def semantic_analysis(self) -> None:
 
     def semantic_analysis_pass_three(self) -> None:
         assert self.tree is not None, "Internal error: method must be called on parsed file only"
+        patches = []  # type: List[Callable[[], None]]
         with self.wrap_context():
-            self.manager.semantic_analyzer_pass3.visit_file(self.tree, self.xpath, self.options)
+            self.manager.semantic_analyzer_pass3.visit_file(self.tree, self.xpath,
+                                                            self.options, patches)
             if self.options.dump_type_stats:
                 dump_type_stats(self.tree, self.xpath)
+        self.patches = patches + self.patches
 
     def semantic_analysis_apply_patches(self) -> None:
         for patch_func in self.patches:

mypy/indirection.py

@@ -101,3 +101,6 @@ def visit_ellipsis_type(self, t: types.EllipsisType) -> Set[str]:
 
     def visit_type_type(self, t: types.TypeType) -> Set[str]:
         return self._visit(t.item)
+
+    def visit_forwardref_type(self, t: types.ForwardRef) -> Set[str]:
+        return self._visit(t.link)

mypy/messages.py

@@ -20,7 +20,7 @@
 from mypy.types import (
     Type, CallableType, Instance, TypeVarType, TupleType, TypedDictType,
     UnionType, NoneTyp, AnyType, Overloaded, FunctionLike, DeletedType, TypeType,
-    UninhabitedType, TypeOfAny
+    UninhabitedType, TypeOfAny, ForwardRef, UnboundType
 )
 from mypy.nodes import (
     TypeInfo, Context, MypyFile, op_methods, FuncDef, reverse_type_aliases,
@@ -194,7 +194,7 @@ def quote_type_string(self, type_string: str) -> str:
         """Quotes a type representation for use in messages."""
         no_quote_regex = r'^<(tuple|union): \d+ items>$'
         if (type_string in ['Module', 'overloaded function', '<nothing>', '<deleted>']
-                or re.match(no_quote_regex, type_string) is not None):
+                or re.match(no_quote_regex, type_string) is not None or type_string.endswith('?')):
             # Messages are easier to read if these aren't quoted.  We use a
             # regex to match strings with variable contents.
             return type_string
@@ -309,6 +309,8 @@ def format_bare(self, typ: Type, verbosity: int = 0) -> str:
                 return '<nothing>'
         elif isinstance(typ, TypeType):
             return 'Type[{}]'.format(self.format_bare(typ.item, verbosity))
+        elif isinstance(typ, ForwardRef):  # may appear in semanal.py
+            return self.format_bare(typ.link, verbosity)
         elif isinstance(typ, FunctionLike):
             func = typ
             if func.is_type_obj():
@@ -350,6 +352,8 @@ def format_bare(self, typ: Type, verbosity: int = 0) -> str:
                 # function types may result in long and difficult-to-read
                 # error messages.
                 return 'overloaded function'
+        elif isinstance(typ, UnboundType):
+            return str(typ)
         elif typ is None:
             raise RuntimeError('Type is None')
         else:

mypy/nodes.py

@@ -1976,6 +1976,12 @@ class is generic then it will be a type constructor of higher kind.
     # Is this a newtype type?
     is_newtype = False
 
+    # If during analysis of ClassDef associated with this TypeInfo a syntethic
+    # type (NamedTuple or TypedDict) was generated, store the corresponding
+    # TypeInfo here. (This attribute does not need to be serialized, it is only
+    # needed during the semantic passes.)
+    replaced = None  # type: TypeInfo
+
     FLAGS = [
         'is_abstract', 'is_enum', 'fallback_to_any', 'is_named_tuple',
         'is_newtype', 'is_protocol', 'runtime_protocol'

mypy/server/deps.py

@@ -11,7 +11,7 @@
 from mypy.types import (
     Type, Instance, AnyType, NoneTyp, TypeVisitor, CallableType, DeletedType, PartialType,
     TupleType, TypeType, TypeVarType, TypedDictType, UnboundType, UninhabitedType, UnionType,
-    FunctionLike
+    FunctionLike, ForwardRef
 )
 from mypy.server.trigger import make_trigger
 
@@ -212,6 +212,9 @@ def visit_type_type(self, typ: TypeType) -> List[str]:
         # TODO: replace with actual implementation
         return []
 
+    def visit_forwardref_type(self, typ: ForwardRef) -> List[str]:
+        return get_type_dependencies(typ.link)
+
     def visit_type_var(self, typ: TypeVarType) -> List[str]:
         # TODO: replace with actual implementation
         return []

mypy/typeanal.py

@@ -1,7 +1,7 @@
 """Semantic analysis of types"""
 
 from collections import OrderedDict
-from typing import Callable, List, Optional, Set, Tuple, Iterator, TypeVar, Iterable
+from typing import Callable, List, Optional, Set, Tuple, Iterator, TypeVar, Iterable, Dict
 from itertools import chain
 
 from contextlib import contextmanager
@@ -14,13 +14,13 @@
     Type, UnboundType, TypeVarType, TupleType, TypedDictType, UnionType, Instance, AnyType,
     CallableType, NoneTyp, DeletedType, TypeList, TypeVarDef, TypeVisitor, SyntheticTypeVisitor,
     StarType, PartialType, EllipsisType, UninhabitedType, TypeType, get_typ_args, set_typ_args,
-    CallableArgument, get_type_vars, TypeQuery, union_items, TypeOfAny
+    CallableArgument, get_type_vars, TypeQuery, union_items, TypeOfAny, ForwardRef
 )
 
 from mypy.nodes import (
     TVAR, TYPE_ALIAS, UNBOUND_IMPORTED, TypeInfo, Context, SymbolTableNode, Var, Expression,
     IndexExpr, RefExpr, nongen_builtins, check_arg_names, check_arg_kinds, ARG_POS, ARG_NAMED,
-    ARG_OPT, ARG_NAMED_OPT, ARG_STAR, ARG_STAR2, TypeVarExpr, CallExpr, NameExpr
+    ARG_OPT, ARG_NAMED_OPT, ARG_STAR, ARG_STAR2, TypeVarExpr, FuncDef, CallExpr, NameExpr
 )
 from mypy.tvar_scope import TypeVarScope
 from mypy.sametypes import is_same_type
@@ -29,6 +29,9 @@
 from mypy.plugin import Plugin, AnalyzerPluginInterface, AnalyzeTypeContext
 from mypy import nodes, messages
 
+MYPY = False
+if MYPY:
+    from mypy.semanal import SemanticAnalyzer
 
 T = TypeVar('T')
 
@@ -59,7 +62,8 @@ def analyze_type_alias(node: Expression,
                        plugin: Plugin,
                        options: Options,
                        is_typeshed_stub: bool,
-                       allow_unnormalized: bool = False) -> Optional[Type]:
+                       allow_unnormalized: bool = False,
+                       in_dynamic_func: bool = False) -> Optional[Type]:
     """Return type if node is valid as a type alias rvalue.
 
     Return None otherwise. 'node' must have been semantically analyzed.
@@ -113,6 +117,7 @@ def analyze_type_alias(node: Expression,
         return None
     analyzer = TypeAnalyser(lookup_func, lookup_fqn_func, tvar_scope, fail_func, plugin, options,
                             is_typeshed_stub, aliasing=True, allow_unnormalized=allow_unnormalized)
+    analyzer.in_dynamic_func = in_dynamic_func
     return type.accept(analyzer)
 
 
@@ -130,6 +135,9 @@ class TypeAnalyser(SyntheticTypeVisitor[Type], AnalyzerPluginInterface):
     Converts unbound types into bound types.
     """
 
+    # Is this called from an untyped function definition
+    in_dynamic_func = False  # type: bool
+
     def __init__(self,
                  lookup_func: Callable[[str, Context], SymbolTableNode],
                  lookup_fqn_func: Callable[[str], SymbolTableNode],
@@ -140,7 +148,8 @@ def __init__(self,
                  is_typeshed_stub: bool, *,
                  aliasing: bool = False,
                  allow_tuple_literal: bool = False,
-                 allow_unnormalized: bool = False) -> None:
+                 allow_unnormalized: bool = False,
+                 third_pass: bool = False) -> None:
         self.lookup = lookup_func
         self.lookup_fqn_func = lookup_fqn_func
         self.fail_func = fail_func
@@ -153,14 +162,15 @@ def __init__(self,
         self.plugin = plugin
         self.options = options
         self.is_typeshed_stub = is_typeshed_stub
+        self.third_pass = third_pass
 
     def visit_unbound_type(self, t: UnboundType) -> Type:
         if t.optional:
             t.optional = False
             # We don't need to worry about double-wrapping Optionals or
             # wrapping Anys: Union simplification will take care of that.
             return make_optional_type(self.visit_unbound_type(t))
-        sym = self.lookup(t.name, t)
+        sym = self.lookup(t.name, t, suppress_errors=self.third_pass)  # type: ignore
         if sym is not None:
             if sym.node is None:
                 # UNBOUND_IMPORTED can happen if an unknown name was imported.
@@ -269,6 +279,10 @@ def visit_unbound_type(self, t: UnboundType) -> Type:
                 # Allow unbound type variables when defining an alias
                 if not (self.aliasing and sym.kind == TVAR and
                         self.tvar_scope.get_binding(sym) is None):
+                    if (not self.third_pass and not self.in_dynamic_func and
+                            not (isinstance(sym.node, FuncDef) or
+                                 isinstance(sym.node, Var) and sym.node.is_ready)):
+                        return ForwardRef(t)
                     self.fail('Invalid type "{}"'.format(name), t)
                 return t
             info = sym.node  # type: TypeInfo
@@ -304,6 +318,9 @@ def visit_unbound_type(self, t: UnboundType) -> Type:
                                             fallback=instance)
                 return instance
         else:
+            if self.third_pass:
+                self.fail('Invalid type {}'.format(t), t)
+                return AnyType(TypeOfAny.from_error)
             return AnyType(TypeOfAny.special_form)
 
     def visit_any(self, t: AnyType) -> Type:
@@ -387,6 +404,9 @@ def visit_ellipsis_type(self, t: EllipsisType) -> Type:
     def visit_type_type(self, t: TypeType) -> Type:
         return TypeType.make_normalized(self.anal_type(t.item), line=t.line)
 
+    def visit_forwardref_type(self, t: ForwardRef) -> Type:
+        return t
+
     def analyze_callable_type(self, t: UnboundType) -> Type:
         fallback = self.named_type('builtins.function')
         if len(t.args) == 0:
@@ -590,13 +610,18 @@ class TypeAnalyserPass3(TypeVisitor[None]):
     def __init__(self,
                  fail_func: Callable[[str, Context], None],
                  options: Options,
-                 is_typeshed_stub: bool) -> None:
+                 is_typeshed_stub: bool,
+                 sem: 'SemanticAnalyzer', indicator: Dict[str, bool]) -> None:
         self.fail = fail_func
         self.options = options
         self.is_typeshed_stub = is_typeshed_stub
+        self.sem = sem
+        self.indicator = indicator
 
     def visit_instance(self, t: Instance) -> None:
         info = t.type
+        if info.replaced or info.tuple_type:
+            self.indicator['synthetic'] = True
         # Check type argument count.
         if len(t.args) != len(info.type_vars):
             if len(t.args) == 0:
@@ -647,6 +672,8 @@ def visit_instance(self, t: Instance) -> None:
                     else:
                         arg_values = [arg]
                     self.check_type_var_values(info, arg_values, tvar.name, tvar.values, i + 1, t)
+                if isinstance(arg, ForwardRef):
+                    arg = arg.link
                 if not is_subtype(arg, tvar.upper_bound):
                     self.fail('Type argument "{}" of "{}" must be '
                               'a subtype of "{}"'.format(
@@ -712,13 +739,22 @@ def visit_type_list(self, t: TypeList) -> None:
         self.fail('Invalid type', t)
 
     def visit_type_var(self, t: TypeVarType) -> None:
-        pass
+        if t.upper_bound:
+            t.upper_bound.accept(self)
+        if t.values:
+            for v in t.values:
+                v.accept(self)
 
     def visit_partial_type(self, t: PartialType) -> None:
         pass
 
     def visit_type_type(self, t: TypeType) -> None:
-        pass
+        t.item.accept(self)
+
+    def visit_forwardref_type(self, t: ForwardRef) -> None:
+        self.indicator['forward'] = True
+        if isinstance(t.link, UnboundType):
+            t.link = self.sem.anal_type(t.link, third_pass=True)
 
 
 TypeVarList = List[Tuple[str, TypeVarExpr]]

mypy/types.py

@@ -1375,6 +1375,26 @@ def deserialize(cls, data: JsonDict) -> Type:
         return TypeType.make_normalized(deserialize_type(data['item']))
 
 
+class ForwardRef(Type):
+    """Class to wrap forward references to other types."""
+    link = None  # type: Type # the wrapped type
+
+    def __init__(self, link: Type) -> None:
+        self.link = link
+
+    def accept(self, visitor: 'TypeVisitor[T]') -> T:
+        return visitor.visit_forwardref_type(self)
+
+    def serialize(self):
+        if isinstance(self.link, UnboundType):
+            name = self.link.name
+        if isinstance(self.link, Instance):
+            name = self.link.type.name()
+        else:
+            name = self.link.__class__.__name__
+        assert False, "Internal error: Unresolved forward reference to {}".format(name)
+
+
 #
 # Visitor-related classes
 #
@@ -1450,6 +1470,9 @@ def visit_partial_type(self, t: PartialType) -> T:
     def visit_type_type(self, t: TypeType) -> T:
         pass
 
+    def visit_forwardref_type(self, t: ForwardRef) -> T:
+        raise RuntimeError('Internal error: unresolved forward reference')
+
 
 class SyntheticTypeVisitor(TypeVisitor[T]):
     """A TypeVisitor that also knows how to visit synthetic AST constructs.
@@ -1552,6 +1575,9 @@ def visit_overloaded(self, t: Overloaded) -> Type:
     def visit_type_type(self, t: TypeType) -> Type:
         return TypeType.make_normalized(t.item.accept(self), line=t.line, column=t.column)
 
+    def visit_forwardref_type(self, t: ForwardRef) -> Type:
+        return t
+
 
 class TypeStrVisitor(SyntheticTypeVisitor[str]):
     """Visitor for pretty-printing types into strings.
@@ -1707,6 +1733,9 @@ def visit_ellipsis_type(self, t: EllipsisType) -> str:
     def visit_type_type(self, t: TypeType) -> str:
         return 'Type[{}]'.format(t.item.accept(self))
 
+    def visit_forwardref_type(self, t: ForwardRef) -> str:
+        return '~{}'.format(t.link.accept(self))
+
     def list_str(self, a: List[Type]) -> str:
         """Convert items of an array to strings (pretty-print types)
         and join the results with commas.
@@ -1786,6 +1815,9 @@ def visit_overloaded(self, t: Overloaded) -> T:
     def visit_type_type(self, t: TypeType) -> T:
         return t.item.accept(self)
 
+    def visit_forwardref_type(self, t: ForwardRef) -> T:
+        return t.link.accept(self)
+
     def visit_ellipsis_type(self, t: EllipsisType) -> T:
         return self.strategy([])