From 279e4560112aeb1d0253ea69cc60655ba0795b23 Mon Sep 17 00:00:00 2001 From: Marius van Niekerk Date: Mon, 14 Aug 2017 11:38:31 -0400 Subject: [PATCH 1/2] Dynamic modules don't necessarily have a __package__ attribute When trying to serialize some of the dynamically generated modules that pytest generates, errors like these are encountered. ``` File "/Users/mvanniekerk/miniconda3/envs/py36/lib/python3.6/pickle.py", line 476, in save f(self, obj) # Call unbound method with explicit self File "/Users/mvanniekerk/src/pytest-dask/.tox/py36/lib/python3.6/site-packages/cloudpickle/cloudpickle.py", line 366, in save_function self.save_function_tuple(obj) File "/Users/mvanniekerk/src/pytest-dask/.tox/py36/lib/python3.6/site-packages/cloudpickle/cloudpickle.py", line 494, in save_function_tuple itertools.chain(f_globals.values(), closure_values or ()), File "/Users/mvanniekerk/src/pytest-dask/.tox/py36/lib/python3.6/site-packages/cloudpickle/cloudpickle.py", line 388, in _save_subimports if isinstance(x, types.ModuleType) and x.__package__: File "/Users/mvanniekerk/src/pytest-dask/.tox/py36/lib/python3.6/site-packages/py/_apipkg.py", line 123, in __makeattr raise AttributeError(name) AttributeError: __package__ ``` After applying this change the resulting serialized objects work correctly --- cloudpickle/cloudpickle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 17a1f9529..d6c140871 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -385,7 +385,7 @@ def _save_subimports(self, code, top_level_dependencies): """ # check if any known dependency is an imported package for x in top_level_dependencies: - if isinstance(x, types.ModuleType) and x.__package__: + if isinstance(x, types.ModuleType) and hasattr(x, '__package__') and x.__package__: # check if the package has any currently loaded sub-imports prefix = x.__name__ + '.' for name, module in sys.modules.items(): From d1b1cf1fbb59aedfad4c6ad5b52ba10c7d40d01d Mon Sep 17 00:00:00 2001 From: Marius van Niekerk Date: Mon, 14 Aug 2017 13:21:08 -0400 Subject: [PATCH 2/2] Added simple test case for the issue --- tests/cloudpickle_test.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index aa33ce4bf..3963e49e3 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -680,6 +680,22 @@ def foo(): finally: sys.modules.pop("_fake_module", None) + def test_dynamic_pytest_module(self): + # Test case for pull request https://github.com/cloudpipe/cloudpickle/pull/116 + import py + + def f(): + s = py.builtin.set([1]) + return s.pop() + + # some setup is required to allow pytest apimodules to be correctly serializable. + from cloudpickle import CloudPickler + from py._apipkg import ApiModule + CloudPickler.dispatch[ApiModule] = CloudPickler.save_module + g = cloudpickle.loads(cloudpickle.dumps(f)) + + result = g() + self.assertEqual(1, result) if __name__ == '__main__': unittest.main()