From a3299ed6fd27134fc702c196a0116aef3284f154 Mon Sep 17 00:00:00 2001 From: valtron Date: Sat, 21 Dec 2019 14:14:48 -0700 Subject: [PATCH 01/12] Pickling of generic annotations/types in 3.5+ --- CHANGES.md | 4 ++ cloudpickle/cloudpickle.py | 73 ++++++++++++++++++++--- tests/cloudpickle_test.py | 116 ++++++++++--------------------------- 3 files changed, 98 insertions(+), 95 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 6fb058b8a..d29fcd6b7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,6 +15,10 @@ instances on Python 3.7+ ([PR #351](https://github.com/cloudpipe/cloudpickle/pull/351)) +- Add support for pickling dynamic classes subclassing `typing.Generic` + instances on Python 3.5+ + ([PR #318](https://github.com/cloudpipe/cloudpickle/pull/318)) + 1.3.0 ===== diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 7fb6ee559..08f0292df 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -611,11 +611,6 @@ def save_dynamic_class(self, obj): if isinstance(__dict__, property): type_kwargs['__dict__'] = __dict__ - if sys.version_info < (3, 7): - # Although annotations were added in Python 3.4, It is not possible - # to properly pickle them until Python 3.7. (See #193) - clsdict.pop('__annotations__', None) - save = self.save write = self.write @@ -715,9 +710,7 @@ def save_function_tuple(self, func): 'doc': func.__doc__, '_cloudpickle_submodules': submodules } - if hasattr(func, '__annotations__') and sys.version_info >= (3, 7): - # Although annotations were added in Python3.4, It is not possible - # to properly pickle them until Python3.7. (See #193) + if hasattr(func, '__annotations__'): state['annotations'] = func.__annotations__ if hasattr(func, '__qualname__'): state['qualname'] = func.__qualname__ @@ -800,6 +793,12 @@ def save_global(self, obj, name=None, pack=struct.pack): elif obj in _BUILTIN_TYPE_NAMES: return self.save_reduce( _builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj) + + decomposed_generic = _try_decompose_generic(obj) + if decomposed_generic is not None: + class_tracker_id = _get_or_create_tracker_id(obj) + reduce_args = (*decomposed_generic, class_tracker_id) + return self.save_reduce(_make_generic, reduce_args, obj=obj) elif name is not None: Pickler.save_global(self, obj, name=name) elif not _is_importable_by_name(obj, name=name): @@ -1284,3 +1283,61 @@ def _get_bases(typ): # For regular class objects bases_attr = '__bases__' return getattr(typ, bases_attr) + + +def _make_generic(origin, args, class_tracker_id): + return _lookup_class_or_track(class_tracker_id, origin[args]) + + +if sys.version_info >= (3, 7): + def _try_decompose_generic(obj): + return None +else: + def _try_decompose_generic(obj): + origin = getattr(obj, '__origin__', None) + if origin is None: + return _try_decompose_special_generic(obj) + args = obj.__args__ + if origin is typing.Callable and args[0] is not Ellipsis: + args = (list(args[:-1]), args[-1]) + return origin, args + + Type_ClassVar = type(typing.ClassVar) + + try: + import typing_extensions + except ImportError: + def _try_decompose_special_generic(obj): + if type(obj) is Type_ClassVar: + t = getattr(obj, '__type__', None) + if t is None: + return None + return typing.ClassVar, t + + return None + else: + Literal = typing_extensions.Literal + Type_Literal = type(typing_extensions.Literal) + Final = typing_extensions.Final + Type_Final = type(typing_extensions.Final) + + def _try_decompose_special_generic(obj): + if type(obj) is Type_ClassVar: + t = getattr(obj, '__type__', None) + if t is None: + return None + return typing.ClassVar, t + + if type(obj) is Type_Literal: + v = getattr(obj, '__values__', None) + if v is None: + return None + return Literal, v + + if type(obj) is Type_Final: + t = getattr(obj, '__type__', None) + if t is None: + return None + return Final, t + + return None diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index c1a55134d..83a183617 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -1787,9 +1787,6 @@ def g(): self.assertEqual(f2.__doc__, f.__doc__) - @unittest.skipIf(sys.version_info < (3, 7), - "Pickling type annotations isn't supported for py36 and " - "below.") def test_wraps_preserves_function_annotations(self): def f(x): pass @@ -1804,79 +1801,7 @@ def g(x): self.assertEqual(f2.__annotations__, f.__annotations__) - @unittest.skipIf(sys.version_info >= (3, 7), - "pickling annotations is supported starting Python 3.7") - def test_function_annotations_silent_dropping(self): - # Because of limitations of typing module, cloudpickle does not pickle - # the type annotations of a dynamic function or class for Python < 3.7 - - class UnpicklableAnnotation: - # Mock Annotation metaclass that errors out loudly if we try to - # pickle one of its instances - def __reduce__(self): - raise Exception("not picklable") - - unpickleable_annotation = UnpicklableAnnotation() - - def f(a: unpickleable_annotation): - return a - - with pytest.raises(Exception): - cloudpickle.dumps(f.__annotations__) - - depickled_f = pickle_depickle(f, protocol=self.protocol) - assert depickled_f.__annotations__ == {} - - @unittest.skipIf(sys.version_info >= (3, 7) or sys.version_info < (3, 6), - "pickling annotations is supported starting Python 3.7") - def test_class_annotations_silent_dropping(self): - # Because of limitations of typing module, cloudpickle does not pickle - # the type annotations of a dynamic function or class for Python < 3.7 - - # Pickling and unpickling must be done in different processes when - # testing dynamic classes (see #313) - - code = '''if 1: - import cloudpickle - import sys - - class UnpicklableAnnotation: - # Mock Annotation metaclass that errors out loudly if we try to - # pickle one of its instances - def __reduce__(self): - raise Exception("not picklable") - - unpickleable_annotation = UnpicklableAnnotation() - - class A: - a: unpickleable_annotation - - try: - cloudpickle.dumps(A.__annotations__) - except Exception: - pass - else: - raise AssertionError - - sys.stdout.buffer.write(cloudpickle.dumps(A, protocol={protocol})) - ''' - cmd = [sys.executable, '-c', code.format(protocol=self.protocol)] - proc = subprocess.Popen( - cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - proc.wait() - out, err = proc.communicate() - assert proc.returncode == 0, err - - depickled_a = pickle.loads(out) - assert not hasattr(depickled_a, "__annotations__") - - @unittest.skipIf(sys.version_info < (3, 7), - "Pickling type hints isn't supported for py36" - " and below.") def test_type_hint(self): - # Try to pickle compound typing constructs. This would typically fail - # on Python < 3.7 (See #193) t = typing.Union[list, int] assert pickle_depickle(t) == t @@ -2142,8 +2067,6 @@ def test_pickle_importable_typevar(self): from typing import AnyStr assert AnyStr is pickle_depickle(AnyStr, protocol=self.protocol) - @unittest.skipIf(sys.version_info < (3, 7), - "Pickling generics not supported below py37") def test_generic_type(self): T = typing.TypeVar('T') @@ -2170,22 +2093,13 @@ def check_generic(generic, origin, type_value): assert check_generic(C[int], C, int) == "ok" assert worker.run(check_generic, C[int], C, int) == "ok" - @unittest.skipIf(sys.version_info < (3, 7), - "Pickling type hints not supported below py37") def test_locally_defined_class_with_type_hints(self): with subprocess_worker(protocol=self.protocol) as worker: for type_ in _all_types_to_test(): - # The type annotation syntax causes a SyntaxError on Python 3.5 - code = textwrap.dedent("""\ class MyClass: - attribute: type_ - def method(self, arg: type_) -> type_: return arg - """) - ns = {"type_": type_} - exec(code, ns) - MyClass = ns["MyClass"] + MyClass.__annotations__ = {'attribute': type_} def check_annotations(obj, expected_type): assert obj.__annotations__["attribute"] is expected_type @@ -2197,6 +2111,34 @@ def check_annotations(obj, expected_type): assert check_annotations(obj, type_) == "ok" assert worker.run(check_annotations, obj, type_) == "ok" + def test_generic_extensions(self): + typing_extensions = pytest.importorskip('typing_extensions') + + objs = [ + typing_extensions.Literal, + typing_extensions.Final, + typing_extensions.Literal['a'], + typing_extensions.Final[int], + ] + + for obj in objs: + _ = pickle_depickle(obj, protocol=self.protocol) + + def test_class_annotations(self): + class C: + pass + C.__annotations__ = {'a': int} + + C1 = pickle_depickle(C, protocol=self.protocol) + assert C1.__annotations__ == C.__annotations__ + + def test_function_annotations(self): + def f(a: int) -> str: + pass + + f1 = pickle_depickle(f, protocol=self.protocol) + assert f1.__annotations__ == f.__annotations__ + class Protocol2CloudPickleTest(CloudPickleTest): From 73ab6dff1a6cdf0a7a65a8b03b759cdbc090bf93 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Mon, 16 Mar 2020 09:15:39 +0100 Subject: [PATCH 02/12] Simplify top-level module structure --- cloudpickle/cloudpickle.py | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 08f0292df..cb258919c 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -117,7 +117,18 @@ def _whichmodule(obj, name): - Errors arising during module introspection are ignored, as those errors are considered unwanted side effects. """ - module_name = _get_module_attr(obj) + if sys.version_info[:2] < (3, 7) and isinstance(obj, typing.TypeVar): + # Workaround bug in old Python versions: prior to Python 3.7, + # T.__module__ would always be set to "typing" even when the TypeVar T + # would be defined in a different module. + # + # For such older Python versions, we ignore the __module__ attribute of + # TypeVar instances and instead exhaustively lookup those instances in + # all currently imported modules. + module_name = None + else: + module_name = getattr(obj, '__module__', None) + if module_name is not None: return module_name # Protect the iteration by using a copy of sys.modules against dynamic @@ -140,23 +151,6 @@ def _whichmodule(obj, name): return None -if sys.version_info[:2] < (3, 7): # pragma: no branch - # Workaround bug in old Python versions: prior to Python 3.7, T.__module__ - # would always be set to "typing" even when the TypeVar T would be defined - # in a different module. - # - # For such older Python versions, we ignore the __module__ attribute of - # TypeVar instances and instead exhaustively lookup those instances in all - # currently imported modules via the _whichmodule function. - def _get_module_attr(obj): - if isinstance(obj, typing.TypeVar): - return None - return getattr(obj, '__module__', None) -else: - def _get_module_attr(obj): - return getattr(obj, '__module__', None) - - def _is_importable_by_name(obj, name=None): """Determine if obj can be pickled as attribute of a file-backed module""" return _lookup_module_and_qualname(obj, name=name) is not None From bc0c368b7e0309a6b6b1da9b6c41b9b4408bf78c Mon Sep 17 00:00:00 2001 From: valtron Date: Sat, 4 Apr 2020 19:47:27 -0600 Subject: [PATCH 03/12] Handle generic special forms in `dispatch` --- cloudpickle/cloudpickle.py | 76 ++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 41 deletions(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index cb258919c..49e5e95a6 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -66,6 +66,11 @@ from io import BytesIO from importlib._bootstrap import _find_spec +try: + import typing_extensions as _typing_extensions +except ImportError: + _typing_extensions = None + # cloudpickle is meant for inter process communication: we expect all # communicating processes to run the same Python version hence we favor @@ -934,6 +939,35 @@ def inject_addons(self): """Plug in system. Register additional pickling functions if modules already loaded""" pass + if sys.version_info < (3, 7): + def _save_special_form(self, obj, base, arg_attr): + t = getattr(obj, arg_attr, None) + if t is None: + # Special forms (ClassVar, Final, Literal) don't store + # their `__name__` in an easy-to-get place. + name = typing._trim_name(typing._qualname(type(obj))) + self.save_global(obj, name=name) + else: + class_tracker_id = _get_or_create_tracker_id(obj) + reduce_args = (base, t, class_tracker_id) + self.save_reduce(_make_generic, reduce_args, obj=obj) + + def save_classvar(self, obj): + self._save_special_form(obj, typing.ClassVar, '__type__') + + dispatch[type(typing.ClassVar)] = save_classvar + + if _typing_extensions is not None: + def save_literal(self, obj): + self._save_special_form(obj, _typing_extensions.Literal, '__values__') + + dispatch[type(_typing_extensions.Literal)] = save_literal + + def save_final(self, obj): + self._save_special_form(obj, _typing_extensions.Final, '__type__') + + dispatch[type(_typing_extensions.Final)] = save_final + # Tornado support @@ -1290,48 +1324,8 @@ def _try_decompose_generic(obj): def _try_decompose_generic(obj): origin = getattr(obj, '__origin__', None) if origin is None: - return _try_decompose_special_generic(obj) + return None args = obj.__args__ if origin is typing.Callable and args[0] is not Ellipsis: args = (list(args[:-1]), args[-1]) return origin, args - - Type_ClassVar = type(typing.ClassVar) - - try: - import typing_extensions - except ImportError: - def _try_decompose_special_generic(obj): - if type(obj) is Type_ClassVar: - t = getattr(obj, '__type__', None) - if t is None: - return None - return typing.ClassVar, t - - return None - else: - Literal = typing_extensions.Literal - Type_Literal = type(typing_extensions.Literal) - Final = typing_extensions.Final - Type_Final = type(typing_extensions.Final) - - def _try_decompose_special_generic(obj): - if type(obj) is Type_ClassVar: - t = getattr(obj, '__type__', None) - if t is None: - return None - return typing.ClassVar, t - - if type(obj) is Type_Literal: - v = getattr(obj, '__values__', None) - if v is None: - return None - return Literal, v - - if type(obj) is Type_Final: - t = getattr(obj, '__type__', None) - if t is None: - return None - return Final, t - - return None From 55489f4516e2f641389f2567af66a9fb476a4598 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Tue, 14 Apr 2020 17:41:37 +0200 Subject: [PATCH 04/12] refactor a bit the type-saving logic --- cloudpickle/cloudpickle.py | 98 ++++++++++++++++++-------------------- tests/cloudpickle_test.py | 4 +- 2 files changed, 49 insertions(+), 53 deletions(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 49e5e95a6..d15eb3f91 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -61,6 +61,7 @@ import typing from enum import Enum +from typing import Generic, Union, Tuple, Callable, ClassVar from pickle import _Pickler as Pickler from pickle import _getattribute from io import BytesIO @@ -68,8 +69,9 @@ try: import typing_extensions as _typing_extensions + from typing_extensions import Literal, Final except ImportError: - _typing_extensions = None + _typing_extensions = Literal = Final = None # cloudpickle is meant for inter process communication: we expect all @@ -422,6 +424,18 @@ def _extract_class_dict(cls): return clsdict +if sys.version_info[:2] < (3, 7): # pramga: no branch + def _is_parametrized_type_hint(obj): + # This is very cheap but might generate false positives. + origin = getattr(obj, '__origin__', None) # typing Constructs + values = getattr(obj, '__values__', None) # typing_extensions.Literal + type_ = getattr(obj, '__type__', None) # typing_extensions.Final + return origin is not None or values is not None or type_ is not None + + def _create_parametrized_type_hint(origin, args): + return origin[args] + + class CloudPickler(Pickler): dispatch = Pickler.dispatch.copy() @@ -793,11 +807,13 @@ def save_global(self, obj, name=None, pack=struct.pack): return self.save_reduce( _builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj) - decomposed_generic = _try_decompose_generic(obj) - if decomposed_generic is not None: - class_tracker_id = _get_or_create_tracker_id(obj) - reduce_args = (*decomposed_generic, class_tracker_id) - return self.save_reduce(_make_generic, reduce_args, obj=obj) + if sys.version_info[:2] < (3, 7) and _is_parametrized_type_hint(obj): # noqa # pragma: no branch + # Parametrized typing constructs in Python < 3.7 are not compatible + # with type checks and ``isinstance`` semantics. For this reason, + # it is easier to detect them using a duck-typing-based check + # (``_is_parametrized_type_hint``) than to populate the Pickler's + # dispatch with type-specific savers. + self._save_parametrized_type_hint(obj) elif name is not None: Pickler.save_global(self, obj, name=name) elif not _is_importable_by_name(obj, name=name): @@ -939,34 +955,30 @@ def inject_addons(self): """Plug in system. Register additional pickling functions if modules already loaded""" pass - if sys.version_info < (3, 7): - def _save_special_form(self, obj, base, arg_attr): - t = getattr(obj, arg_attr, None) - if t is None: - # Special forms (ClassVar, Final, Literal) don't store - # their `__name__` in an easy-to-get place. - name = typing._trim_name(typing._qualname(type(obj))) - self.save_global(obj, name=name) - else: - class_tracker_id = _get_or_create_tracker_id(obj) - reduce_args = (base, t, class_tracker_id) - self.save_reduce(_make_generic, reduce_args, obj=obj) - - def save_classvar(self, obj): - self._save_special_form(obj, typing.ClassVar, '__type__') - - dispatch[type(typing.ClassVar)] = save_classvar - - if _typing_extensions is not None: - def save_literal(self, obj): - self._save_special_form(obj, _typing_extensions.Literal, '__values__') - - dispatch[type(_typing_extensions.Literal)] = save_literal - - def save_final(self, obj): - self._save_special_form(obj, _typing_extensions.Final, '__type__') - - dispatch[type(_typing_extensions.Final)] = save_final + if sys.version_info < (3, 7): # pragma: no branch + def _save_parametrized_type_hint(self, obj): + # The distorted type check sematic for typing construct becomes: + # ``type(obj) is type(TypeHint)``, which means "obj is a + # parametrized TypeHint" + if type(obj) is type(Literal): + initargs = (Literal, obj.__values__) + elif type(obj) is type(Final): + initargs = (Final, obj.__type__) + elif type(obj) is type(ClassVar): + initargs = (ClassVar, obj.__type__) + elif type(obj) in [type(Union), type(Tuple), type(Generic)]: + initargs = (obj.__origin__, obj.__args__) + elif type(obj) is type(Callable): + args = obj.__args__ + if args[0] is Ellipsis: + initargs = (obj.__origin__, args) + else: + initargs = (obj.__origin__, (list(args[:-1]), args[-1])) + else: # pramga: no cover + raise pickle.PicklingError( + "Cloudpickle Error: Unknown type {}".format(type(obj)) + ) + self.save_reduce(_create_parametrized_type_hint, initargs, obj=obj) # Tornado support @@ -1311,21 +1323,3 @@ def _get_bases(typ): # For regular class objects bases_attr = '__bases__' return getattr(typ, bases_attr) - - -def _make_generic(origin, args, class_tracker_id): - return _lookup_class_or_track(class_tracker_id, origin[args]) - - -if sys.version_info >= (3, 7): - def _try_decompose_generic(obj): - return None -else: - def _try_decompose_generic(obj): - origin = getattr(obj, '__origin__', None) - if origin is None: - return None - args = obj.__args__ - if origin is typing.Callable and args[0] is not Ellipsis: - args = (list(args[:-1]), args[-1]) - return origin, args diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index 83a183617..7f08466f4 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -2074,7 +2074,9 @@ class C(typing.Generic[T]): pass assert pickle_depickle(C, protocol=self.protocol) is C - assert pickle_depickle(C[int], protocol=self.protocol) is C[int] + + # Identity is not part of the typing contract. + assert pickle_depickle(C[int], protocol=self.protocol) == C[int] with subprocess_worker(protocol=self.protocol) as worker: From ef4a263203b8809ff1517e97360646f61469dea3 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 15 Apr 2020 11:22:44 +0200 Subject: [PATCH 05/12] test pickling typing-extensions constructs in CI --- .github/workflows/testing.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 16dad3b58..486011b7c 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -61,6 +61,10 @@ jobs: python -m pip install -r dev-requirements.txt python ci/install_coverage_subprocess_pth.py export + - name: Install optional typing_extensions in Python 3.6 + shell: bash + run: python -m pip install typing-extensions + if: matrix.python_version == "3.6" - name: Display Python version shell: bash run: python -c "import sys; print(sys.version)" From 333d6b0f5849b7bfd21c68fe0e2c197d4a212a5b Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 15 Apr 2020 11:26:29 +0200 Subject: [PATCH 06/12] CI use single quotes (?) --- .github/workflows/testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 486011b7c..d7cc99f6e 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -64,7 +64,7 @@ jobs: - name: Install optional typing_extensions in Python 3.6 shell: bash run: python -m pip install typing-extensions - if: matrix.python_version == "3.6" + if: matrix.python_version == '3.6' - name: Display Python version shell: bash run: python -c "import sys; print(sys.version)" From 26e06f584717d76b150a3a0044ce6a437b0101c7 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 15 Apr 2020 11:33:33 +0200 Subject: [PATCH 07/12] correct typos in coverage pragmas --- cloudpickle/cloudpickle.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index d15eb3f91..2b2de7223 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -424,7 +424,7 @@ def _extract_class_dict(cls): return clsdict -if sys.version_info[:2] < (3, 7): # pramga: no branch +if sys.version_info[:2] < (3, 7): # pragma: no branch def _is_parametrized_type_hint(obj): # This is very cheap but might generate false positives. origin = getattr(obj, '__origin__', None) # typing Constructs @@ -974,7 +974,7 @@ def _save_parametrized_type_hint(self, obj): initargs = (obj.__origin__, args) else: initargs = (obj.__origin__, (list(args[:-1]), args[-1])) - else: # pramga: no cover + else: # pragma: no cover raise pickle.PicklingError( "Cloudpickle Error: Unknown type {}".format(type(obj)) ) From eb1cebdef211c27bcfbe92a67654bfcb6e8f2267 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 15 Apr 2020 12:51:19 +0200 Subject: [PATCH 08/12] add a few more pytest pragmas --- cloudpickle/cloudpickle.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 2b2de7223..73ca30cc7 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -67,7 +67,7 @@ from io import BytesIO from importlib._bootstrap import _find_spec -try: +try: # pragma: no branch import typing_extensions as _typing_extensions from typing_extensions import Literal, Final except ImportError: @@ -124,7 +124,7 @@ def _whichmodule(obj, name): - Errors arising during module introspection are ignored, as those errors are considered unwanted side effects. """ - if sys.version_info[:2] < (3, 7) and isinstance(obj, typing.TypeVar): + if sys.version_info[:2] < (3, 7) and isinstance(obj, typing.TypeVar): # pragma: no branch # noqa # Workaround bug in old Python versions: prior to Python 3.7, # T.__module__ would always be set to "typing" even when the TypeVar T # would be defined in a different module. @@ -960,9 +960,9 @@ def _save_parametrized_type_hint(self, obj): # The distorted type check sematic for typing construct becomes: # ``type(obj) is type(TypeHint)``, which means "obj is a # parametrized TypeHint" - if type(obj) is type(Literal): + if type(obj) is type(Literal): # pragma: no branch initargs = (Literal, obj.__values__) - elif type(obj) is type(Final): + elif type(obj) is type(Final): # pragma: no branch initargs = (Final, obj.__type__) elif type(obj) is type(ClassVar): initargs = (ClassVar, obj.__type__) From 63bc55116e7b331f93728ae9870fecb20f55abc4 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 22 Apr 2020 09:50:04 +0200 Subject: [PATCH 09/12] comment rephrasings Co-Authored-By: Olivier Grisel --- tests/cloudpickle_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index 7f08466f4..cfc64ab80 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -2075,7 +2075,8 @@ class C(typing.Generic[T]): assert pickle_depickle(C, protocol=self.protocol) is C - # Identity is not part of the typing contract. + # Identity is not part of the typing contract: only test for + # equality instead. assert pickle_depickle(C[int], protocol=self.protocol) == C[int] with subprocess_worker(protocol=self.protocol) as worker: From f889731e9c880c7c6b6232bc030269e555a27827 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 22 Apr 2020 09:59:05 +0200 Subject: [PATCH 10/12] CI trigger From 236b339dde164586e3c6a147e4e690661bb89d95 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 22 Apr 2020 10:24:44 +0200 Subject: [PATCH 11/12] rephrase CHANGES.md --- CHANGES.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index d29fcd6b7..99d1bac80 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -3,6 +3,10 @@ **This version requires Python 3.5 or later** +- cloudpickle can now all pickle all constructs from the ``typing`` module + and the ``typing_extensions`` library in Python 3.5+ + ([PR #318](https://github.com/cloudpipe/cloudpickle/pull/318)) + - Stop pickling the annotations of a dynamic class for Python < 3.6 (follow up on #276) ([issue #347](https://github.com/cloudpipe/cloudpickle/issues/347)) @@ -15,10 +19,6 @@ instances on Python 3.7+ ([PR #351](https://github.com/cloudpipe/cloudpickle/pull/351)) -- Add support for pickling dynamic classes subclassing `typing.Generic` - instances on Python 3.5+ - ([PR #318](https://github.com/cloudpipe/cloudpickle/pull/318)) - 1.3.0 ===== From 554a4c636082b9b5bed660d273126c3c8b6bb3fc Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 22 Apr 2020 10:26:12 +0200 Subject: [PATCH 12/12] check equality of depickled type hints --- tests/cloudpickle_test.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index cfc64ab80..126bb310a 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -2105,9 +2105,11 @@ def method(self, arg: type_) -> type_: MyClass.__annotations__ = {'attribute': type_} def check_annotations(obj, expected_type): - assert obj.__annotations__["attribute"] is expected_type - assert obj.method.__annotations__["arg"] is expected_type - assert obj.method.__annotations__["return"] is expected_type + assert obj.__annotations__["attribute"] == expected_type + assert obj.method.__annotations__["arg"] == expected_type + assert ( + obj.method.__annotations__["return"] == expected_type + ) return "ok" obj = MyClass() @@ -2125,7 +2127,8 @@ def test_generic_extensions(self): ] for obj in objs: - _ = pickle_depickle(obj, protocol=self.protocol) + depickled_obj = pickle_depickle(obj, protocol=self.protocol) + assert depickled_obj == obj def test_class_annotations(self): class C: