From 4e8c6d6ca6fe6f13fcdaffc3723ce6912d0fcf1a Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 14 Nov 2022 12:07:12 -0800 Subject: [PATCH 1/5] Error for invalid TypedDict keys --- pyanalyze/implementation.py | 8 ++++---- pyanalyze/test_typeddict.py | 11 +++++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/pyanalyze/implementation.py b/pyanalyze/implementation.py index dfb11609..c8cd2f05 100644 --- a/pyanalyze/implementation.py +++ b/pyanalyze/implementation.py @@ -554,10 +554,10 @@ def inner(key: Value) -> Value: except Exception: # No error here; TypedDicts may have additional keys at runtime. pass - # TODO strictly we should throw an error for any non-Literal or unknown key: - # https://www.python.org/dev/peps/pep-0589/#supported-and-unsupported-operations - # Don't do that yet because it may cause too much disruption. - return AnyValue(AnySource.inference) + ctx.show_error( + f"Unknown TypedDict key {key}", ErrorCode.invalid_typeddict_key, arg="k" + ) + return AnyValue(AnySource.error) elif isinstance(self_value, DictIncompleteValue): val = self_value.get_value(key, ctx.visitor) if val is UNINITIALIZED_VALUE: diff --git a/pyanalyze/test_typeddict.py b/pyanalyze/test_typeddict.py index 4723580d..00e7d98e 100644 --- a/pyanalyze/test_typeddict.py +++ b/pyanalyze/test_typeddict.py @@ -35,3 +35,14 @@ def capybara(): {"x": (True, TypedValue(int)), "y": (False, TypedValue(str))} ), ) + + @assert_passes() + def test_unknown_key(self): + from typing_extensions import TypedDict, assert_type + + class Capybara(TypedDict): + x: int + + def user(c: Capybara): + assert_type(c["x"], int) + c["y"] # E: invalid_typeddict_key From e84d9b9fc313e8bf581170cb5f680f941af175b3 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 14 Nov 2022 14:20:17 -0800 Subject: [PATCH 2/5] Allow non-literal keys for now --- docs/changelog.md | 1 + pyanalyze/implementation.py | 16 ++++++++++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index f423e40f..97acc497 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,6 +2,7 @@ ## Unreleased +- Emit an error for unknown `TypedDict` keys (#567) - Fix crash on recursive type aliases. Recursive type aliases now fall back to `Any` (#565) - Support `in` on objects with only `__getitem__` (#564) - Add support for `except*` (PEP 654) (#562) diff --git a/pyanalyze/implementation.py b/pyanalyze/implementation.py index c8cd2f05..07d89fa1 100644 --- a/pyanalyze/implementation.py +++ b/pyanalyze/implementation.py @@ -552,12 +552,16 @@ def inner(key: Value) -> Value: # probably KeyError, but catch anything in case it's an # unhashable str subclass or something except Exception: - # No error here; TypedDicts may have additional keys at runtime. - pass - ctx.show_error( - f"Unknown TypedDict key {key}", ErrorCode.invalid_typeddict_key, arg="k" - ) - return AnyValue(AnySource.error) + ctx.show_error( + f"Unknown TypedDict key {key}", + ErrorCode.invalid_typeddict_key, + arg="k", + ) + return AnyValue(AnySource.error) + # TODO strictly we should throw an error for any non-Literal or unknown key: + # https://www.python.org/dev/peps/pep-0589/#supported-and-unsupported-operations + # Don't do that yet because it may cause too much disruption. + return AnyValue(AnySource.inference) elif isinstance(self_value, DictIncompleteValue): val = self_value.get_value(key, ctx.visitor) if val is UNINITIALIZED_VALUE: From 056f09524e906f2ffe2eab0e06f5e897b598c3ef Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 14 Nov 2022 15:48:31 -0800 Subject: [PATCH 3/5] move some tests --- pyanalyze/implementation.py | 29 ++++++++---- pyanalyze/test_name_check_visitor.py | 69 ---------------------------- pyanalyze/test_typeddict.py | 69 +++++++++++++++++++++++++++- 3 files changed, 87 insertions(+), 80 deletions(-) diff --git a/pyanalyze/implementation.py b/pyanalyze/implementation.py index 730c3095..a6f8bdef 100644 --- a/pyanalyze/implementation.py +++ b/pyanalyze/implementation.py @@ -566,10 +566,12 @@ def inner(key: Value) -> Value: return AnyValue(AnySource.error) if self_value.extra_keys is not None: return self_value.extra_keys - # TODO strictly we should throw an error for any non-Literal or unknown key: - # https://www.python.org/dev/peps/pep-0589/#supported-and-unsupported-operations - # Don't do that yet because it may cause too much disruption. - return AnyValue(AnySource.inference) + ctx.show_error( + f"TypedDict key must be a literal, not {key}", + ErrorCode.invalid_typeddict_key, + arg="k", + ) + return AnyValue(AnySource.error) elif isinstance(self_value, DictIncompleteValue): val = self_value.get_value(key, ctx.visitor) if val is UNINITIALIZED_VALUE: @@ -627,8 +629,13 @@ def inner(key: Value) -> Value: # probably KeyError, but catch anything in case it's an # unhashable str subclass or something except Exception: - # No error here; TypedDicts may have additional keys at runtime. - pass + if self_value.extra_keys is None: + ctx.show_error( + f"Unknown TypedDict key {key.val!r}", + ErrorCode.invalid_typeddict_key, + arg="k", + ) + return AnyValue(AnySource.error) else: if required: return value @@ -636,10 +643,12 @@ def inner(key: Value) -> Value: return value | default if self_value.extra_keys is not None: return self_value.extra_keys | default - # TODO strictly we should throw an error for any non-Literal or unknown key: - # https://www.python.org/dev/peps/pep-0589/#supported-and-unsupported-operations - # Don't do that yet because it may cause too much disruption. - return default + ctx.show_error( + f"TypedDict key must be a literal, not {key}", + ErrorCode.invalid_typeddict_key, + arg="k", + ) + return AnyValue(AnySource.error) elif isinstance(self_value, DictIncompleteValue): val = self_value.get_value(key, ctx.visitor) if val is UNINITIALIZED_VALUE: diff --git a/pyanalyze/test_name_check_visitor.py b/pyanalyze/test_name_check_visitor.py index 1a6223ac..519c8779 100644 --- a/pyanalyze/test_name_check_visitor.py +++ b/pyanalyze/test_name_check_visitor.py @@ -1748,75 +1748,6 @@ def capybara(): return f"{x}" # E: undefined_name -class TestTypedDict(TestNameCheckVisitorBase): - @assert_passes() - def test_basic(self): - from mypy_extensions import TypedDict as METypedDict - from typing_extensions import TypedDict as TETypedDict - - T = METypedDict("T", {"a": int, "b": str}) - T2 = TETypedDict("T2", {"a": int, "b": str}) - - def capybara(x: T, y: T2): - assert_is_value( - x, - TypedDictValue( - {"a": (True, TypedValue(int)), "b": (True, TypedValue(str))} - ), - ) - assert_is_value(x["a"], TypedValue(int)) - assert_is_value( - y, - TypedDictValue( - {"a": (True, TypedValue(int)), "b": (True, TypedValue(str))} - ), - ) - assert_is_value(y["a"], TypedValue(int)) - - @assert_passes() - def test_unknown_key_unresolved(self): - from mypy_extensions import TypedDict - - T = TypedDict("T", {"a": int, "b": str}) - - def capybara(x: T): - assert_is_value(x["not a key"], AnyValue(AnySource.inference)) - - @assert_passes() - def test_invalid_key(self): - from mypy_extensions import TypedDict - - T = TypedDict("T", {"a": int, "b": str}) - - def capybara(x: T): - x[0] # E: invalid_typeddict_key - - @assert_passes() - def test_total(self): - from typing_extensions import TypedDict - - class TD(TypedDict, total=False): - a: int - b: str - - class TD2(TD): - c: float - - def f(td: TD) -> None: - pass - - def g(td2: TD2) -> None: - pass - - def caller() -> None: - f({}) - f({"a": 1}) - f({"a": 1, "b": "c"}) - f({"a": "a"}) # E: incompatible_argument - g({"c": 1.0}) - g({}) # E: incompatible_argument - - _AnnotSettings = { ErrorCode.missing_parameter_annotation: True, ErrorCode.missing_return_annotation: True, diff --git a/pyanalyze/test_typeddict.py b/pyanalyze/test_typeddict.py index 103e0273..9cdc8851 100644 --- a/pyanalyze/test_typeddict.py +++ b/pyanalyze/test_typeddict.py @@ -2,7 +2,7 @@ from .implementation import assert_is_value from .test_name_check_visitor import TestNameCheckVisitorBase from .test_node_visitor import assert_passes -from .value import TypedDictValue, TypedValue +from .value import TypedDictValue, TypedValue, AnyValue, AnySource class TestExtraKeys(TestNameCheckVisitorBase): @@ -164,3 +164,70 @@ class Capybara(TypedDict): def user(c: Capybara): assert_type(c["x"], int) c["y"] # E: invalid_typeddict_key + + @assert_passes() + def test_basic(self): + from mypy_extensions import TypedDict as METypedDict + from typing_extensions import TypedDict as TETypedDict + + T = METypedDict("T", {"a": int, "b": str}) + T2 = TETypedDict("T2", {"a": int, "b": str}) + + def capybara(x: T, y: T2): + assert_is_value( + x, + TypedDictValue( + {"a": (True, TypedValue(int)), "b": (True, TypedValue(str))} + ), + ) + assert_is_value(x["a"], TypedValue(int)) + assert_is_value( + y, + TypedDictValue( + {"a": (True, TypedValue(int)), "b": (True, TypedValue(str))} + ), + ) + assert_is_value(y["a"], TypedValue(int)) + + @assert_passes() + def test_unknown_key_unresolved(self): + from mypy_extensions import TypedDict + + T = TypedDict("T", {"a": int, "b": str}) + + def capybara(x: T): + assert_is_value(x["not a key"], AnyValue(AnySource.inference)) + + @assert_passes() + def test_invalid_key(self): + from mypy_extensions import TypedDict + + T = TypedDict("T", {"a": int, "b": str}) + + def capybara(x: T): + x[0] # E: invalid_typeddict_key + + @assert_passes() + def test_total(self): + from typing_extensions import TypedDict + + class TD(TypedDict, total=False): + a: int + b: str + + class TD2(TD): + c: float + + def f(td: TD) -> None: + pass + + def g(td2: TD2) -> None: + pass + + def caller() -> None: + f({}) + f({"a": 1}) + f({"a": 1, "b": "c"}) + f({"a": "a"}) # E: incompatible_argument + g({"c": 1.0}) + g({}) # E: incompatible_argument From 4656553d75716471e5924dc3933176e314ab0152 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 14 Nov 2022 15:49:23 -0800 Subject: [PATCH 4/5] update test --- pyanalyze/test_typeddict.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyanalyze/test_typeddict.py b/pyanalyze/test_typeddict.py index 9cdc8851..6974809c 100644 --- a/pyanalyze/test_typeddict.py +++ b/pyanalyze/test_typeddict.py @@ -196,7 +196,8 @@ def test_unknown_key_unresolved(self): T = TypedDict("T", {"a": int, "b": str}) def capybara(x: T): - assert_is_value(x["not a key"], AnyValue(AnySource.inference)) + val = x["not a key"] # E: invalid_typeddict_key + assert_is_value(val, AnyValue(AnySource.error)) @assert_passes() def test_invalid_key(self): From f0ff29bfd85412fe7bb823259c0a6e11e471c61d Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 14 Nov 2022 15:57:16 -0800 Subject: [PATCH 5/5] key_type must be str after all --- pyanalyze/test_typeddict.py | 8 +++++--- pyanalyze/value.py | 6 ++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pyanalyze/test_typeddict.py b/pyanalyze/test_typeddict.py index 6974809c..547d0468 100644 --- a/pyanalyze/test_typeddict.py +++ b/pyanalyze/test_typeddict.py @@ -75,6 +75,7 @@ def capybara() -> None: def test_compatibility(self): from pyanalyze.extensions import has_extra_keys from typing_extensions import TypedDict + from typing import Any, Dict @has_extra_keys(int) class TD(TypedDict): @@ -91,10 +92,11 @@ class TD3(TypedDict): def want_td(td: TD) -> None: pass - def capybara(td: TD, td2: TD2, td3: TD3) -> None: + def capybara(td: TD, td2: TD2, td3: TD3, anydict: Dict[str, Any]) -> None: want_td(td) want_td(td2) want_td(td3) # E: incompatible_argument + want_td(anydict) @assert_passes() def test_iteration(self): @@ -111,13 +113,13 @@ class TD2(TypedDict): def capybara(td: TD, td2: TD2) -> None: for k, v in td.items(): - assert_type(k, Union[str, Literal["a"]]) + assert_type(k, str) assert_type(v, Union[int, str]) for k in td: assert_type(k, Union[str, Literal["a"]]) for k, v in td2.items(): - assert_type(k, Literal["a"]) + assert_type(k, str) assert_type(v, str) for k in td2: assert_type(k, Literal["a"]) diff --git a/pyanalyze/value.py b/pyanalyze/value.py index 7253c635..7bdc08a7 100644 --- a/pyanalyze/value.py +++ b/pyanalyze/value.py @@ -1154,19 +1154,17 @@ def __init__( self, items: Dict[str, Tuple[bool, Value]], extra_keys: Optional[Value] = None ) -> None: value_types = [] - key_types = [] if items: value_types += [val for _, val in items.values()] - key_types += [KnownValue(key) for key in items.keys()] if extra_keys is not None: value_types.append(extra_keys) - key_types.append(TypedValue(str)) value_type = ( unite_values(*value_types) if value_types else AnyValue(AnySource.unreachable) ) - key_type = unite_values(*key_types) if key_types else TypedValue(str) + # The key type must be str so dict[str, Any] is compatible with a TypedDict + key_type = TypedValue(str) super().__init__(dict, (key_type, value_type)) self.items = items self.extra_keys = extra_keys