From a6bf9a2f4412a06e7a18747e625266fb1ba2c589 Mon Sep 17 00:00:00 2001 From: Anirudh Vegesana Date: Thu, 9 Jun 2022 11:50:32 -0700 Subject: [PATCH 01/13] Create a soft_def shim to combine move_to and Getattr (#473) * Create a soft_def shim to combine move_to and Getattr * Rename `soft_def` to `register_shim` --- dill/_shims.py | 44 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/dill/_shims.py b/dill/_shims.py index 6e170437..2da6f5be 100644 --- a/dill/_shims.py +++ b/dill/_shims.py @@ -150,11 +150,43 @@ def decorator(func): return func return decorator +def register_shim(name, default): + """ + A easier to understand and more compact way of "softly" defining a function. + These two pieces of code are equivalent: + + if _dill.OLD3X: + def _create_class(): + ... + _create_class = register_shim('_create_class', types.new_class) + + if _dill.OLD3X: + @move_to(_dill) + def _create_class(): + ... + _create_class = Getattr(_dill, '_create_class', types.new_class) + + Intuitively, it creates a function or object in the versions of dill/python + that require special reimplementations, and use a core library or default + implementation if that function or object does not exist. + """ + func = globals().get(name) + if func is not None: + _dill.__dict__[name] = func + func.__module__ = _dill.__name__ + + if default is Getattr.NO_DEFAULT: + reduction = (getattr, (_dill, name)) + else: + reduction = (getattr, (_dill, name, default)) + + return Reduce(*reduction, is_callable=callable(default)) + ###################### ## Compatibility Shims are defined below ###################### -_CELL_EMPTY = Getattr(_dill, '_CELL_EMPTY', None) +_CELL_EMPTY = register_shim('_CELL_EMPTY', None) if _dill.OLD37: if _dill.HAS_CTYPES and hasattr(_dill.ctypes, 'pythonapi') and hasattr(_dill.ctypes.pythonapi, 'PyCell_Set'): @@ -163,7 +195,6 @@ def decorator(func): _PyCell_Set = ctypes.pythonapi.PyCell_Set - @move_to(_dill) def _setattr(object, name, value): if type(object) is _dill.CellType and name == 'cell_contents': _PyCell_Set.argtypes = (ctypes.py_object, ctypes.py_object) @@ -171,7 +202,6 @@ def _setattr(object, name, value): else: setattr(object, name, value) - @move_to(_dill) def _delattr(object, name): if type(object) is _dill.CellType and name == 'cell_contents': _PyCell_Set.argtypes = (ctypes.py_object, ctypes.c_void_p) @@ -195,7 +225,6 @@ def cell_setter(value): func(value) else: setattr(cell, name, value)''' % __nonlocal) - move_to(_dill)(_setattr) exec('''def _delattr(cell, name): if type(cell) is _dill.CellType and name == 'cell_contents': @@ -210,7 +239,6 @@ def cell_deleter(): func() else: delattr(cell, name)''' % __nonlocal) - move_to(_dill)(_delattr) else: # Likely PyPy 2.7. Simulate the nonlocal keyword with bytecode @@ -221,7 +249,6 @@ def cell_deleter(): # Copyright (c) 2012, Regents of the University of California. # Copyright (c) 2009 `PiCloud, Inc. `_. # License: https://github.com/cloudpipe/cloudpickle/blob/master/LICENSE - @move_to(_dill) def _setattr(cell, name, value): if type(cell) is _dill.CellType and name == 'cell_contents': _cell_set = _dill.FunctionType( @@ -255,12 +282,11 @@ def _cell_set_factory(value): del co - @move_to(_dill) def _delattr(cell, name): if type(cell) is _dill.CellType and name == 'cell_contents': pass else: delattr(cell, name) -_setattr = Getattr(_dill, '_setattr', setattr) -_delattr = Getattr(_dill, '_delattr', delattr) +_setattr = register_shim('_setattr', setattr) +_delattr = register_shim('_delattr', delattr) From 0ce3baf15c9ede24bcece1c4dd6e99d48bc54b18 Mon Sep 17 00:00:00 2001 From: Anirudh Vegesana Date: Thu, 9 Jun 2022 17:26:15 -0700 Subject: [PATCH 02/13] Incidental implementation specific types (#472) * Incidental implementation specific types * Add incedental types to objects.py * Remove types for objects that are automatically by Python * Rename `SymtableStentryType` to `SymtableEntryType` --- dill/_dill.py | 79 +++++++++++++++++++++++++++++++++++++++++------- dill/_objects.py | 49 ++++++++++++++++++++++++------ 2 files changed, 108 insertions(+), 20 deletions(-) diff --git a/dill/_dill.py b/dill/_dill.py index df3f69fa..415e2b06 100644 --- a/dill/_dill.py +++ b/dill/_dill.py @@ -42,6 +42,7 @@ def _trace(boolean): OLDER = (PY3 and sys.hexversion < 0x3040000) or (sys.hexversion < 0x2070ab1) OLD33 = (sys.hexversion < 0x3030000) OLD37 = (sys.hexversion < 0x3070000) +OLD38 = (sys.hexversion < 0x3080000) OLD39 = (sys.hexversion < 0x3090000) OLD310 = (sys.hexversion < 0x30a0000) PY34 = (0x3040000 <= sys.hexversion < 0x3050000) @@ -87,6 +88,7 @@ def _trace(boolean): import gc # import zlib from weakref import ReferenceType, ProxyType, CallableProxyType +from collections import OrderedDict from functools import partial from operator import itemgetter, attrgetter # new in python3.3 @@ -274,8 +276,6 @@ def get_file_type(*args, **kwargs): except NameError: ExitType = None singletontypes = [] -from collections import OrderedDict - import inspect ### Shims for different versions of Python and dill @@ -572,7 +572,6 @@ def __init__(self, *args, **kwds): self._strictio = False #_strictio self._fmode = settings['fmode'] if _fmode is None else _fmode self._recurse = settings['recurse'] if _recurse is None else _recurse - from collections import OrderedDict self._postproc = OrderedDict() def dump(self, obj): #NOTE: if settings change, need to update attributes @@ -723,6 +722,15 @@ def _create_typemap(): 'SuperType': SuperType, 'ItemGetterType': ItemGetterType, 'AttrGetterType': AttrGetterType, +}) + +# "Incidental" implementation specific types. Unpickling these types in another +# implementation of Python (PyPy -> CPython) is not gauranteed to work + +# This dictionary should contain all types that appear in Python implementations +# but are not defined in https://docs.python.org/3/library/types.html#standard-interpreter-types +x=OrderedDict() +_incedental_reverse_typemap = { 'FileType': FileType, 'BufferedRandomType': BufferedRandomType, 'BufferedReaderType': BufferedReaderType, @@ -732,18 +740,62 @@ def _create_typemap(): 'PyBufferedReaderType': PyBufferedReaderType, 'PyBufferedWriterType': PyBufferedWriterType, 'PyTextWrapperType': PyTextWrapperType, -}) +} + +if PY3: + _incedental_reverse_typemap.update({ + "DictKeysType": type({}.keys()), + "DictValuesType": type({}.values()), + "DictItemsType": type({}.items()), + + "OdictKeysType": type(x.keys()), + "OdictValuesType": type(x.values()), + "OdictItemsType": type(x.items()), + }) +else: + _incedental_reverse_typemap.update({ + "DictKeysType": type({}.viewkeys()), + "DictValuesType": type({}.viewvalues()), + "DictItemsType": type({}.viewitems()), + }) + if ExitType: - _reverse_typemap['ExitType'] = ExitType + _incedental_reverse_typemap['ExitType'] = ExitType if InputType: - _reverse_typemap['InputType'] = InputType - _reverse_typemap['OutputType'] = OutputType + _incedental_reverse_typemap['InputType'] = InputType + _incedental_reverse_typemap['OutputType'] = OutputType if not IS_PYPY: - _reverse_typemap['WrapperDescriptorType'] = WrapperDescriptorType - _reverse_typemap['MethodDescriptorType'] = MethodDescriptorType - _reverse_typemap['ClassMethodDescriptorType'] = ClassMethodDescriptorType + _incedental_reverse_typemap['WrapperDescriptorType'] = WrapperDescriptorType + _incedental_reverse_typemap['MethodDescriptorType'] = MethodDescriptorType + _incedental_reverse_typemap['ClassMethodDescriptorType'] = ClassMethodDescriptorType else: - _reverse_typemap['MemberDescriptorType'] = MemberDescriptorType + _incedental_reverse_typemap['MemberDescriptorType'] = MemberDescriptorType + +try: + import symtable + _incedental_reverse_typemap["SymtableStentryType"] = type(symtable.symtable("", "string", "exec")._table) +except: + pass + +if sys.hexversion >= 0x30a00a0: + _incedental_reverse_typemap['LineIteratorType'] = type(compile('3', '', 'eval').co_lines()) + +if sys.hexversion >= 0x30b00b0: + from types import GenericAlias + _incedental_reverse_typemap["GenericAliasIteratorType"] = type(iter(GenericAlias(list, (int,)))) + _incedental_reverse_typemap['PositionsIteratorType'] = type(compile('3', '', 'eval').co_positions()) + +try: + import winreg + _incedental_reverse_typemap["HKEYType"] = winreg.HKEYType +except: + pass + +_reverse_typemap.update(_incedental_reverse_typemap) +_incedental_types = set(_incedental_reverse_typemap.values()) + +del x + if PY3: _typemap = dict((v, k) for k, v in _reverse_typemap.items()) else: @@ -1988,6 +2040,8 @@ def save_module(pickler, obj): def save_type(pickler, obj, postproc_list=None): if obj in _typemap: log.info("T1: %s" % obj) + # if obj in _incedental_types: + # warnings.warn('Type %r may only exist on this implementation of Python and cannot be unpickled in other implementations.' % (obj,), PicklingWarning) pickler.save_reduce(_load_type, (_typemap[obj],), obj=obj) log.info("# T1") elif obj.__bases__ == (tuple,) and all([hasattr(obj, attr) for attr in ('_fields','_asdict','_make','_replace')]): @@ -2255,6 +2309,9 @@ def save_capsule(pickler, obj): destructor = _PyCapsule_GetDestructor(obj) pickler.save_reduce(_create_capsule, (pointer, name, context, destructor), obj=obj) log.info("# Cap") + _incedental_reverse_typemap['PyCapsuleType'] = PyCapsuleType + _reverse_typemap['PyCapsuleType'] = PyCapsuleType + _incedental_types.add(PyCapsuleType) else: _testcapsule = None diff --git a/dill/_objects.py b/dill/_objects.py index 31823636..f8680ad3 100644 --- a/dill/_objects.py +++ b/dill/_objects.py @@ -417,9 +417,49 @@ class _Struct(ctypes.Structure): a['FileType'] = open(os.devnull, 'rb', buffering=0) # same 'wb','wb+','rb+' # FIXME: FileType fails >= 3.1 # built-in functions (CH 2) +# Iterators: a['ListIteratorType'] = iter(_list) # empty vs non-empty FIXME: fail < 3.2 +x['SetIteratorType'] = iter(_set) #XXX: empty vs non-empty a['TupleIteratorType']= iter(_tuple) # empty vs non-empty FIXME: fail < 3.2 a['XRangeIteratorType'] = iter(_xrange) # empty vs non-empty FIXME: fail < 3.2 +a["BytesIteratorType"] = iter(b'') +a["BytearrayIteratorType"] = iter(bytearray(b'')) +a["CallableIteratorType"] = iter(iter, None) +a["MemoryIteratorType"] = iter(memoryview(b'')) +a["ListReverseiteratorType"] = reversed([]) +X = a['OrderedDictType'] +a["OdictKeysType"] = X.keys() +a["OdictValuesType"] = X.values() +a["OdictItemsType"] = X.items() +a["OdictIteratorType"] = iter(X.keys()) +del X +if PY3: + x['DictionaryItemIteratorType'] = iter(type.__dict__.items()) + x['DictionaryKeyIteratorType'] = iter(type.__dict__.keys()) + x['DictionaryValueIteratorType'] = iter(type.__dict__.values()) +else: + x['DictionaryItemIteratorType'] = type.__dict__.iteritems() + x['DictionaryKeyIteratorType'] = type.__dict__.iterkeys() + x['DictionaryValueIteratorType'] = type.__dict__.itervalues() +if sys.hexversion >= 0x30800a0: + a["DictReversekeyiteratorType"] = reversed({}.keys()) + a["DictReversevalueiteratorType"] = reversed({}.values()) + a["DictReverseitemiteratorType"] = reversed({}.items()) + +try: + import symtable + a["SymtableEntryType"] = symtable.symtable("", "string", "exec")._table +except: + pass + +if sys.hexversion >= 0x30a00a0: + a['LineIteratorType'] = compile('3', '', 'eval').co_lines() + +if sys.hexversion >= 0x30b00b0: + from types import GenericAlias + a["GenericAliasIteratorType"] = iter(GenericAlias(list, (int,))) + a['PositionsIteratorType'] = compile('3', '', 'eval').co_positions() + # data types (CH 8) a['PrettyPrinterType'] = pprint.PrettyPrinter() #FIXME: fail >= 3.2 and == 2.5 # numeric and mathematical types (CH 9) @@ -452,16 +492,7 @@ class _Struct(ctypes.Structure): # other (concrete) object types # (also: Capsule / CObject ?) # built-in functions (CH 2) -x['SetIteratorType'] = iter(_set) #XXX: empty vs non-empty # built-in types (CH 5) -if PY3: - x['DictionaryItemIteratorType'] = iter(type.__dict__.items()) - x['DictionaryKeyIteratorType'] = iter(type.__dict__.keys()) - x['DictionaryValueIteratorType'] = iter(type.__dict__.values()) -else: - x['DictionaryItemIteratorType'] = type.__dict__.iteritems() - x['DictionaryKeyIteratorType'] = type.__dict__.iterkeys() - x['DictionaryValueIteratorType'] = type.__dict__.itervalues() # string services (CH 7) x['StructType'] = struct.Struct('c') x['CallableIteratorType'] = _srepattern.finditer('') From dc9a1b932036be8592291e092d3deefa6033de29 Mon Sep 17 00:00:00 2001 From: Anirudh Vegesana Date: Thu, 9 Jun 2022 17:34:34 -0700 Subject: [PATCH 03/13] Fix bug in pickling MappingProxyType in PyPy 3.7+ (#506) --- dill/_dill.py | 31 +++++++------------------------ tests/test_dictviews.py | 6 +++++- 2 files changed, 12 insertions(+), 25 deletions(-) diff --git a/dill/_dill.py b/dill/_dill.py index 415e2b06..89f2c464 100644 --- a/dill/_dill.py +++ b/dill/_dill.py @@ -1878,30 +1878,13 @@ def save_dictproxy(pickler, obj): pickler.save_reduce(DictProxyType, (mapping,), obj=obj) log.info("# Mp") return -elif not IS_PYPY: - if not OLD33: - @register(DictProxyType) - def save_dictproxy(pickler, obj): - log.info("Mp: %s" % obj) - pickler.save_reduce(DictProxyType, (obj.copy(),), obj=obj) - log.info("# Mp") - return - else: - # The following function is based on 'saveDictProxy' from spickle - # Copyright (c) 2011 by science+computing ag - # License: http://www.apache.org/licenses/LICENSE-2.0 - @register(DictProxyType) - def save_dictproxy(pickler, obj): - log.info("Dp: %s" % obj) - attr = obj.get('__dict__') - #pickler.save_reduce(_create_dictproxy, (attr,'nested'), obj=obj) - if type(attr) == GetSetDescriptorType and attr.__name__ == "__dict__" \ - and getattr(attr.__objclass__, "__dict__", None) == obj: - pickler.save_reduce(getattr, (attr.__objclass__,"__dict__"),obj=obj) - log.info("# Dp") - return - # all bad below... so throw ReferenceError or TypeError - raise ReferenceError("%s does not reference a class __dict__" % obj) +else: + @register(DictProxyType) + def save_dictproxy(pickler, obj): + log.info("Mp: %s" % obj) + pickler.save_reduce(DictProxyType, (obj.copy(),), obj=obj) + log.info("# Mp") + return @register(SliceType) def save_slice(pickler, obj): diff --git a/tests/test_dictviews.py b/tests/test_dictviews.py index 213e7ab2..87d0a9eb 100644 --- a/tests/test_dictviews.py +++ b/tests/test_dictviews.py @@ -7,7 +7,10 @@ # - https://github.com/uqfoundation/dill/blob/master/LICENSE import dill -from dill._dill import OLD310, MAPPING_PROXY_TRICK +from dill._dill import OLD310, MAPPING_PROXY_TRICK, DictProxyType + +def test_dictproxy(): + assert dill.copy(DictProxyType({'a': 2})) def test_dictviews(): x = {'a': 1} @@ -31,5 +34,6 @@ def test_dictproxy_trick(): assert dict(seperate_views[1]) == new_x if __name__ == '__main__': + test_dictproxy() test_dictviews() test_dictproxy_trick() From 408b8de9c4d8246721230615cc43d6b2f20d5c39 Mon Sep 17 00:00:00 2001 From: Anirudh Vegesana Date: Thu, 16 Jun 2022 02:52:32 -0700 Subject: [PATCH 04/13] Bring back old method pickling function (#511) This is a very strange use case of functions and methods that Python's pickle package doesn't handle correctly (#510). This case used to work in pre-0.3.5 dill, so let's bring back the old dill implementation. The comments are irrelevant in Python 3. --- dill/_dill.py | 64 ++++++++++++++++++++++++++++------------- tests/test_functions.py | 16 +++++++++++ 2 files changed, 60 insertions(+), 20 deletions(-) diff --git a/dill/_dill.py b/dill/_dill.py index 89f2c464..becd1b6a 100644 --- a/dill/_dill.py +++ b/dill/_dill.py @@ -1767,15 +1767,29 @@ def save_builtin_method(pickler, obj): log.info("# B2") return - @register(MethodType) #FIXME: fails for 'hidden' or 'name-mangled' classes - def save_instancemethod0(pickler, obj):# example: cStringIO.StringI - log.info("Me: %s" % obj) #XXX: obj.__dict__ handled elsewhere? - if PY3: - pickler.save_reduce(MethodType, (obj.__func__, obj.__self__), obj=obj) - else: - pickler.save_reduce(MethodType, (obj.im_func, obj.im_self, - obj.im_class), obj=obj) - log.info("# Me") +if IS_PYPY: + @register(MethodType) + def save_instancemethod0(pickler, obj): + code = getattr(obj.__func__, '__code__', None) + if code is not None and type(code) is not CodeType \ + and getattr(obj.__self__, obj.__name__) == obj: + # Some PyPy builtin functions have no module name + log.info("Me2: %s" % obj) + # TODO: verify that this works for all PyPy builtin methods + pickler.save_reduce(getattr, (obj.__self__, obj.__name__), obj=obj) + log.info("# Me2") + return + + log.info("Me1: %s" % obj) + pickler.save_reduce(MethodType, (obj.__func__, obj.__self__), obj=obj) + log.info("# Me1") + return +else: + @register(MethodType) + def save_instancemethod0(pickler, obj): + log.info("Me1: %s" % obj) + pickler.save_reduce(MethodType, (obj.__func__, obj.__self__), obj=obj) + log.info("# Me1") return if sys.hexversion >= 0x20500f0: @@ -1801,17 +1815,6 @@ def save_wrapper_descriptor(pickler, obj): log.info("# Wr") return - @register(MethodWrapperType) - def save_instancemethod(pickler, obj): - log.info("Mw: %s" % obj) - if IS_PYPY2 and obj.__self__ is None and obj.im_class: - # Can be a class method in PYPY2 if __self__ is none - pickler.save_reduce(getattr, (obj.im_class, obj.__name__), obj=obj) - return - pickler.save_reduce(getattr, (obj.__self__, obj.__name__), obj=obj) - log.info("# Mw") - return - elif not IS_PYPY: @register(MethodDescriptorType) @register(WrapperDescriptorType) @@ -2139,6 +2142,27 @@ def save_classmethod(pickler, obj): @register(FunctionType) def save_function(pickler, obj): if not _locate_function(obj, pickler): + if type(obj.__code__) is not CodeType: + # Some PyPy builtin functions have no module name, and thus are not + # able to be located + module_name = getattr(obj, '__module__', None) + if module_name is None: + module_name = __builtin__.__name__ + module = _import_module(module_name, safe=True) + _pypy_builtin = False + try: + found, _ = _getattribute(module, obj.__qualname__) + if getattr(found, '__func__', None) is obj: + _pypy_builtin = True + except: + pass + + if _pypy_builtin: + log.info("F3: %s" % obj) + pickler.save_reduce(getattr, (found, '__func__'), obj=obj) + log.info("# F3") + return + log.info("F1: %s" % obj) _recurse = getattr(pickler, '_recurse', None) _postproc = getattr(pickler, '_postproc', None) diff --git a/tests/test_functions.py b/tests/test_functions.py index ec9670e2..a86cbde0 100644 --- a/tests/test_functions.py +++ b/tests/test_functions.py @@ -54,6 +54,21 @@ def function_with_unassigned_variable(): return (lambda: value) +def test_issue_510(): + # A very bizzare use of functions and methods that pickle doesn't get + # correctly for odd reasons. + class Foo: + def __init__(self): + def f2(self): + return self + self.f2 = f2.__get__(self) + + import dill, pickletools + f = Foo() + f1 = dill.copy(f) + assert f1.f2() is f1 + + def test_functions(): dumped_func_a = dill.dumps(function_a) assert dill.loads(dumped_func_a)(0) == 0 @@ -104,3 +119,4 @@ def test_functions(): if __name__ == '__main__': test_functions() + test_issue_510() From 13e3f80f8ab790293cae81afcf757b6a0b63b1d5 Mon Sep 17 00:00:00 2001 From: mmckerns Date: Thu, 16 Jun 2022 21:12:27 -0400 Subject: [PATCH 05/13] protect against sys.modules contents change --- dill/__diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dill/__diff.py b/dill/__diff.py index df2589eb..3ff65763 100644 --- a/dill/__diff.py +++ b/dill/__diff.py @@ -235,6 +235,6 @@ def _imp(*args, **kwds): # memorise all already imported modules. This implies that this must be # imported first for any changes to be recorded -for mod in sys.modules.values(): +for mod in list(sys.modules.values()): memorise(mod) release_gone() From 60911fcce8e457f626de15ff98969da100c4f89c Mon Sep 17 00:00:00 2001 From: mmckerns Date: Sat, 25 Jun 2022 18:17:58 -0400 Subject: [PATCH 06/13] correct failures and succeeds dicts --- dill/_objects.py | 93 ++++++++++++++++++++++++------------------------ 1 file changed, 46 insertions(+), 47 deletions(-) diff --git a/dill/_objects.py b/dill/_objects.py index f8680ad3..3b9d920d 100644 --- a/dill/_objects.py +++ b/dill/_objects.py @@ -214,7 +214,7 @@ class _Struct(ctypes.Structure): a['CFloatType'] = ctypes.c_float() a['CDoubleType'] = ctypes.c_double() a['CSizeTType'] = ctypes.c_size_t() - a['CLibraryLoaderType'] = ctypes.cdll + x['CLibraryLoaderType'] = ctypes.cdll a['StructureType'] = _Struct # if not IS_PYPY: # a['BigEndianStructureType'] = ctypes.BigEndianStructure() @@ -340,35 +340,35 @@ class _Struct(ctypes.Structure): else: _fileW = _tmpf # data persistence (CH 11) if HAS_ALL: - a['ConnectionType'] = _conn = sqlite3.connect(':memory:') - a['CursorType'] = _conn.cursor() + x['ConnectionType'] = _conn = sqlite3.connect(':memory:') + x['CursorType'] = _conn.cursor() a['ShelveType'] = shelve.Shelf({}) # data compression and archiving (CH 12) if HAS_ALL: if (hex(sys.hexversion) < '0x2070ef0') or PY3: - a['BZ2FileType'] = bz2.BZ2File(os.devnull) #FIXME: fail >= 3.3, 2.7.14 - a['BZ2CompressorType'] = bz2.BZ2Compressor() - a['BZ2DecompressorType'] = bz2.BZ2Decompressor() -#a['ZipFileType'] = _zip = zipfile.ZipFile(os.devnull,'w') #FIXME: fail >= 3.2 + x['BZ2FileType'] = bz2.BZ2File(os.devnull) + x['BZ2CompressorType'] = bz2.BZ2Compressor() + x['BZ2DecompressorType'] = bz2.BZ2Decompressor() +#x['ZipFileType'] = _zip = zipfile.ZipFile(os.devnull,'w') #_zip.write(_tempfile,'x') [causes annoying warning/error printed on import] #a['ZipInfoType'] = _zip.getinfo('x') a['TarFileType'] = tarfile.open(fileobj=_fileW,mode='w') # file formats (CH 13) -a['DialectType'] = csv.get_dialect('excel') +x['DialectType'] = csv.get_dialect('excel') a['PackerType'] = xdrlib.Packer() # optional operating system services (CH 16) a['LockType'] = threading.Lock() a['RLockType'] = threading.RLock() # generic operating system services (CH 15) # also closed/open and r/w/etc... -a['NamedLoggerType'] = _logger = logging.getLogger(__name__) #FIXME: fail >= 3.2 and <= 2.6 +a['NamedLoggerType'] = _logger = logging.getLogger(__name__) #a['FrozenModuleType'] = __hello__ #FIXME: prints "Hello world..." # interprocess communication (CH 17) if PY3: - a['SocketType'] = _socket = socket.socket() #FIXME: fail >= 3.3 - a['SocketPairType'] = socket.socketpair()[0] #FIXME: fail >= 3.3 + x['SocketType'] = _socket = socket.socket() + x['SocketPairType'] = socket.socketpair()[0] else: - a['SocketType'] = _socket = socket.socket() - a['SocketPairType'] = _socket._sock + x['SocketType'] = _socket = socket.socket() + x['SocketPairType'] = _socket._sock # python runtime services (CH 27) if PY3: a['GeneratorContextManagerType'] = contextlib.contextmanager(max)([1]) @@ -394,7 +394,7 @@ class _Struct(ctypes.Structure): # numeric and mathematical types (CH 9) a['ProductType'] = itertools.product('0','1') # generic operating system services (CH 15) - a['FileHandlerType'] = logging.FileHandler(os.devnull) #FIXME: fail >= 3.2 and <= 2.6 + a['FileHandlerType'] = logging.FileHandler(os.devnull) a['RotatingFileHandlerType'] = logging.handlers.RotatingFileHandler(os.devnull) a['SocketHandlerType'] = logging.handlers.SocketHandler('localhost',514) a['MemoryHandlerType'] = logging.handlers.MemoryHandler(1) @@ -415,17 +415,16 @@ class _Struct(ctypes.Structure): # -- dill fails in some versions below here --------------------------------- # types module (part of CH 8) a['FileType'] = open(os.devnull, 'rb', buffering=0) # same 'wb','wb+','rb+' -# FIXME: FileType fails >= 3.1 # built-in functions (CH 2) # Iterators: -a['ListIteratorType'] = iter(_list) # empty vs non-empty FIXME: fail < 3.2 -x['SetIteratorType'] = iter(_set) #XXX: empty vs non-empty -a['TupleIteratorType']= iter(_tuple) # empty vs non-empty FIXME: fail < 3.2 -a['XRangeIteratorType'] = iter(_xrange) # empty vs non-empty FIXME: fail < 3.2 +a['ListIteratorType'] = iter(_list) # empty vs non-empty +a['SetIteratorType'] = iter(_set) #XXX: empty vs non-empty +a['TupleIteratorType']= iter(_tuple) # empty vs non-empty +a['XRangeIteratorType'] = iter(_xrange) # empty vs non-empty a["BytesIteratorType"] = iter(b'') a["BytearrayIteratorType"] = iter(bytearray(b'')) a["CallableIteratorType"] = iter(iter, None) -a["MemoryIteratorType"] = iter(memoryview(b'')) +x["MemoryIteratorType"] = iter(memoryview(b'')) a["ListReverseiteratorType"] = reversed([]) X = a['OrderedDictType'] a["OdictKeysType"] = X.keys() @@ -434,13 +433,13 @@ class _Struct(ctypes.Structure): a["OdictIteratorType"] = iter(X.keys()) del X if PY3: - x['DictionaryItemIteratorType'] = iter(type.__dict__.items()) - x['DictionaryKeyIteratorType'] = iter(type.__dict__.keys()) - x['DictionaryValueIteratorType'] = iter(type.__dict__.values()) + a['DictionaryItemIteratorType'] = iter(type.__dict__.items()) + a['DictionaryKeyIteratorType'] = iter(type.__dict__.keys()) + a['DictionaryValueIteratorType'] = iter(type.__dict__.values()) else: - x['DictionaryItemIteratorType'] = type.__dict__.iteritems() - x['DictionaryKeyIteratorType'] = type.__dict__.iterkeys() - x['DictionaryValueIteratorType'] = type.__dict__.itervalues() + a['DictionaryItemIteratorType'] = type.__dict__.iteritems() + a['DictionaryKeyIteratorType'] = type.__dict__.iterkeys() + a['DictionaryValueIteratorType'] = type.__dict__.itervalues() if sys.hexversion >= 0x30800a0: a["DictReversekeyiteratorType"] = reversed({}.keys()) a["DictReversevalueiteratorType"] = reversed({}.values()) @@ -448,7 +447,7 @@ class _Struct(ctypes.Structure): try: import symtable - a["SymtableEntryType"] = symtable.symtable("", "string", "exec")._table + x["SymtableEntryType"] = symtable.symtable("", "string", "exec")._table except: pass @@ -461,25 +460,25 @@ class _Struct(ctypes.Structure): a['PositionsIteratorType'] = compile('3', '', 'eval').co_positions() # data types (CH 8) -a['PrettyPrinterType'] = pprint.PrettyPrinter() #FIXME: fail >= 3.2 and == 2.5 +a['PrettyPrinterType'] = pprint.PrettyPrinter() # numeric and mathematical types (CH 9) -a['CycleType'] = itertools.cycle('0') #FIXME: fail < 3.2 +a['CycleType'] = itertools.cycle('0') # file and directory access (CH 10) -a['TemporaryFileType'] = _tmpf #FIXME: fail >= 3.2 and == 2.5 +a['TemporaryFileType'] = _tmpf # data compression and archiving (CH 12) -a['GzipFileType'] = gzip.GzipFile(fileobj=_fileW) #FIXME: fail > 3.2 and <= 2.6 +x['GzipFileType'] = gzip.GzipFile(fileobj=_fileW) # generic operating system services (CH 15) -a['StreamHandlerType'] = logging.StreamHandler() #FIXME: fail >= 3.2 and == 2.5 +a['StreamHandlerType'] = logging.StreamHandler() try: # python 2.6 # numeric and mathematical types (CH 9) - a['PermutationsType'] = itertools.permutations('0') #FIXME: fail < 3.2 - a['CombinationsType'] = itertools.combinations('0',1) #FIXME: fail < 3.2 + a['PermutationsType'] = itertools.permutations('0') + a['CombinationsType'] = itertools.combinations('0',1) except AttributeError: pass try: # python 2.7 # numeric and mathematical types (CH 9) - a['RepeatType'] = itertools.repeat(0) #FIXME: fail < 3.2 - a['CompressType'] = itertools.compress('0',[1]) #FIXME: fail < 3.2 + a['RepeatType'] = itertools.repeat(0) + a['CompressType'] = itertools.compress('0',[1]) #XXX: ...and etc except AttributeError: pass @@ -548,7 +547,7 @@ class _Struct(ctypes.Structure): x['CFunctionType'] = _cfunc(str) try: # python 2.6 # numeric and mathematical types (CH 9) - x['MethodCallerType'] = operator.methodcaller('mro') # 2.6 + a['MethodCallerType'] = operator.methodcaller('mro') # 2.6 except AttributeError: pass try: # python 2.7 @@ -556,17 +555,17 @@ class _Struct(ctypes.Structure): x['MemoryType'] = memoryview(_in) # 2.7 x['MemoryType2'] = memoryview(bytearray(_in)) # 2.7 if PY3: - x['DictItemsType'] = _dict.items() # 2.7 - x['DictKeysType'] = _dict.keys() # 2.7 - x['DictValuesType'] = _dict.values() # 2.7 + a['DictItemsType'] = _dict.items() # 2.7 + a['DictKeysType'] = _dict.keys() # 2.7 + a['DictValuesType'] = _dict.values() # 2.7 else: - x['DictItemsType'] = _dict.viewitems() # 2.7 - x['DictKeysType'] = _dict.viewkeys() # 2.7 - x['DictValuesType'] = _dict.viewvalues() # 2.7 + a['DictItemsType'] = _dict.viewitems() # 2.7 + a['DictKeysType'] = _dict.viewkeys() # 2.7 + a['DictValuesType'] = _dict.viewvalues() # 2.7 # generic operating system services (CH 15) - x['RawTextHelpFormatterType'] = argparse.RawTextHelpFormatter('PROG') - x['RawDescriptionHelpFormatterType'] = argparse.RawDescriptionHelpFormatter('PROG') - x['ArgDefaultsHelpFormatterType'] = argparse.ArgumentDefaultsHelpFormatter('PROG') + a['RawTextHelpFormatterType'] = argparse.RawTextHelpFormatter('PROG') + a['RawDescriptionHelpFormatterType'] = argparse.RawDescriptionHelpFormatter('PROG') + a['ArgDefaultsHelpFormatterType'] = argparse.ArgumentDefaultsHelpFormatter('PROG') except NameError: pass try: # python 2.7 (and not 3.1) @@ -581,7 +580,7 @@ class _Struct(ctypes.Structure): from dill._dill import _testcapsule if _testcapsule is not None: - x['PyCapsuleType'] = _testcapsule + a['PyCapsuleType'] = _testcapsule del _testcapsule # -- cleanup ---------------------------------------------------------------- From 178e7c05e037b4fcd158a0fa276651321dd9492c Mon Sep 17 00:00:00 2001 From: mmckerns Date: Sat, 25 Jun 2022 20:26:04 -0400 Subject: [PATCH 07/13] correct failures for registered --- dill/_dill.py | 38 +++++++--------------------------- dill/_objects.py | 53 ++++++++++++++++++++++++------------------------ 2 files changed, 33 insertions(+), 58 deletions(-) diff --git a/dill/_dill.py b/dill/_dill.py index becd1b6a..197660e8 100644 --- a/dill/_dill.py +++ b/dill/_dill.py @@ -194,33 +194,16 @@ def ndarraysubclassinstance(obj): return False def numpyufunc(obj): return False def numpydtype(obj): return False +from types import GetSetDescriptorType, ClassMethodDescriptorType, \ + WrapperDescriptorType, MethodDescriptorType, MemberDescriptorType, \ + MethodWrapperType #XXX: unused + # make sure to add these 'hand-built' types to _typemap if PY3: CellType = type((lambda x: lambda y: x)(0).__closure__[0]) else: CellType = type((lambda x: lambda y: x)(0).func_closure[0]) -# new in python2.5 -if sys.hexversion >= 0x20500f0: - from types import GetSetDescriptorType - if not IS_PYPY: - from types import MemberDescriptorType - else: - # oddly, MemberDescriptorType is GetSetDescriptorType - # while, member_descriptor does exist otherwise... is this a pypy bug? - class _member(object): - __slots__ = ['descriptor'] - MemberDescriptorType = type(_member.descriptor) -if IS_PYPY: - WrapperDescriptorType = MethodType - MethodDescriptorType = FunctionType - ClassMethodDescriptorType = FunctionType -else: - WrapperDescriptorType = type(type.__repr__) - MethodDescriptorType = type(type.__dict__['mro']) - ClassMethodDescriptorType = type(type.__dict__['__prepare__' if PY3 else 'mro']) - -MethodWrapperType = type([].__repr__) -PartialType = type(partial(int,base=2)) +PartialType = type(partial(int, base=2)) SuperType = type(super(Exception, TypeError())) ItemGetterType = type(itemgetter(0)) AttrGetterType = type(attrgetter('__repr__')) @@ -717,7 +700,6 @@ def _create_typemap(): _reverse_typemap = dict(_create_typemap()) _reverse_typemap.update({ 'CellType': CellType, - 'MethodWrapperType': MethodWrapperType, 'PartialType': PartialType, 'SuperType': SuperType, 'ItemGetterType': ItemGetterType, @@ -725,7 +707,7 @@ def _create_typemap(): }) # "Incidental" implementation specific types. Unpickling these types in another -# implementation of Python (PyPy -> CPython) is not gauranteed to work +# implementation of Python (PyPy -> CPython) is not guaranteed to work # This dictionary should contain all types that appear in Python implementations # but are not defined in https://docs.python.org/3/library/types.html#standard-interpreter-types @@ -764,16 +746,10 @@ def _create_typemap(): if InputType: _incedental_reverse_typemap['InputType'] = InputType _incedental_reverse_typemap['OutputType'] = OutputType -if not IS_PYPY: - _incedental_reverse_typemap['WrapperDescriptorType'] = WrapperDescriptorType - _incedental_reverse_typemap['MethodDescriptorType'] = MethodDescriptorType - _incedental_reverse_typemap['ClassMethodDescriptorType'] = ClassMethodDescriptorType -else: - _incedental_reverse_typemap['MemberDescriptorType'] = MemberDescriptorType try: import symtable - _incedental_reverse_typemap["SymtableStentryType"] = type(symtable.symtable("", "string", "exec")._table) + _incedental_reverse_typemap["SymtableEntryType"] = type(symtable.symtable("", "string", "exec")._table) except: pass diff --git a/dill/_objects.py b/dill/_objects.py index 3b9d920d..67da9b5f 100644 --- a/dill/_objects.py +++ b/dill/_objects.py @@ -275,16 +275,16 @@ class _Struct(ctypes.Structure): a['NotImplementedType'] = NotImplemented a['SliceType'] = slice(1) a['UnboundMethodType'] = _class._method #XXX: works when not imported! -a['TextWrapperType'] = open(os.devnull, 'r') # same as mode='w','w+','r+' -a['BufferedRandomType'] = open(os.devnull, 'r+b') # same as mode='w+b' -a['BufferedReaderType'] = open(os.devnull, 'rb') # (default: buffering=-1) -a['BufferedWriterType'] = open(os.devnull, 'wb') +d['TextWrapperType'] = open(os.devnull, 'r') # same as mode='w','w+','r+' +d['BufferedRandomType'] = open(os.devnull, 'r+b') # same as mode='w+b' +d['BufferedReaderType'] = open(os.devnull, 'rb') # (default: buffering=-1) +d['BufferedWriterType'] = open(os.devnull, 'wb') try: # oddities: deprecated from _pyio import open as _open - a['PyTextWrapperType'] = _open(os.devnull, 'r', buffering=-1) - a['PyBufferedRandomType'] = _open(os.devnull, 'r+b', buffering=-1) - a['PyBufferedReaderType'] = _open(os.devnull, 'rb', buffering=-1) - a['PyBufferedWriterType'] = _open(os.devnull, 'wb', buffering=-1) + d['PyTextWrapperType'] = _open(os.devnull, 'r', buffering=-1) + d['PyBufferedRandomType'] = _open(os.devnull, 'r+b', buffering=-1) + d['PyBufferedReaderType'] = _open(os.devnull, 'rb', buffering=-1) + d['PyBufferedWriterType'] = _open(os.devnull, 'wb', buffering=-1) except ImportError: pass # other (concrete) object types @@ -294,17 +294,16 @@ class _Struct(ctypes.Structure): else: d['CellType'] = (_lambda)(0).func_closure[0] a['XRangeType'] = _xrange = xrange(1) -if not IS_PYPY: - d['MethodDescriptorType'] = type.__dict__['mro'] - d['WrapperDescriptorType'] = type.__repr__ - a['WrapperDescriptorType2'] = type.__dict__['__module__'] - d['ClassMethodDescriptorType'] = type.__dict__['__prepare__' if PY3 else 'mro'] +a['MethodDescriptorType'] = type.__dict__['mro'] +a['WrapperDescriptorType'] = type.__repr__ +#a['WrapperDescriptorType2'] = type.__dict__['__module__']#XXX: GetSetDescriptor +a['ClassMethodDescriptorType'] = type.__dict__['__prepare__' if PY3 else 'mro'] # built-in functions (CH 2) if PY3 or IS_PYPY: _methodwrap = (1).__lt__ else: _methodwrap = (1).__cmp__ -d['MethodWrapperType'] = _methodwrap +a['MethodWrapperType'] = _methodwrap a['StaticMethodType'] = staticmethod(_method) a['ClassMethodType'] = classmethod(_method) a['PropertyType'] = property() @@ -414,7 +413,7 @@ class _Struct(ctypes.Structure): # -- dill fails in some versions below here --------------------------------- # types module (part of CH 8) -a['FileType'] = open(os.devnull, 'rb', buffering=0) # same 'wb','wb+','rb+' +d['FileType'] = open(os.devnull, 'rb', buffering=0) # same 'wb','wb+','rb+' # built-in functions (CH 2) # Iterators: a['ListIteratorType'] = iter(_list) # empty vs non-empty @@ -427,9 +426,9 @@ class _Struct(ctypes.Structure): x["MemoryIteratorType"] = iter(memoryview(b'')) a["ListReverseiteratorType"] = reversed([]) X = a['OrderedDictType'] -a["OdictKeysType"] = X.keys() -a["OdictValuesType"] = X.values() -a["OdictItemsType"] = X.items() +d["OdictKeysType"] = X.keys() +d["OdictValuesType"] = X.values() +d["OdictItemsType"] = X.items() a["OdictIteratorType"] = iter(X.keys()) del X if PY3: @@ -447,8 +446,8 @@ class _Struct(ctypes.Structure): try: import symtable - x["SymtableEntryType"] = symtable.symtable("", "string", "exec")._table -except: + d["SymtableEntryType"] = symtable.symtable("", "string", "exec")._table +except: #FIXME: fails; should not be registered (in _reverse__typemap)? pass if sys.hexversion >= 0x30a00a0: @@ -555,13 +554,13 @@ class _Struct(ctypes.Structure): x['MemoryType'] = memoryview(_in) # 2.7 x['MemoryType2'] = memoryview(bytearray(_in)) # 2.7 if PY3: - a['DictItemsType'] = _dict.items() # 2.7 - a['DictKeysType'] = _dict.keys() # 2.7 - a['DictValuesType'] = _dict.values() # 2.7 + d['DictItemsType'] = _dict.items() # 2.7 + d['DictKeysType'] = _dict.keys() # 2.7 + d['DictValuesType'] = _dict.values() # 2.7 else: - a['DictItemsType'] = _dict.viewitems() # 2.7 - a['DictKeysType'] = _dict.viewkeys() # 2.7 - a['DictValuesType'] = _dict.viewvalues() # 2.7 + d['DictItemsType'] = _dict.viewitems() # 2.7 + d['DictKeysType'] = _dict.viewkeys() # 2.7 + d['DictValuesType'] = _dict.viewvalues() # 2.7 # generic operating system services (CH 15) a['RawTextHelpFormatterType'] = argparse.RawTextHelpFormatter('PROG') a['RawDescriptionHelpFormatterType'] = argparse.RawDescriptionHelpFormatter('PROG') @@ -580,7 +579,7 @@ class _Struct(ctypes.Structure): from dill._dill import _testcapsule if _testcapsule is not None: - a['PyCapsuleType'] = _testcapsule + d['PyCapsuleType'] = _testcapsule del _testcapsule # -- cleanup ---------------------------------------------------------------- From 73d9f85d8706875849a683c25a9a71951e57e6b3 Mon Sep 17 00:00:00 2001 From: mmckerns Date: Sat, 25 Jun 2022 20:42:01 -0400 Subject: [PATCH 08/13] unregister SymtableEntryType --- dill/_dill.py | 4 +++- dill/_objects.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/dill/_dill.py b/dill/_dill.py index 197660e8..1d7aee50 100644 --- a/dill/_dill.py +++ b/dill/_dill.py @@ -747,11 +747,13 @@ def _create_typemap(): _incedental_reverse_typemap['InputType'] = InputType _incedental_reverse_typemap['OutputType'] = OutputType +''' try: import symtable _incedental_reverse_typemap["SymtableEntryType"] = type(symtable.symtable("", "string", "exec")._table) -except: +except: #FIXME: fails to pickle pass +''' if sys.hexversion >= 0x30a00a0: _incedental_reverse_typemap['LineIteratorType'] = type(compile('3', '', 'eval').co_lines()) diff --git a/dill/_objects.py b/dill/_objects.py index 67da9b5f..31797770 100644 --- a/dill/_objects.py +++ b/dill/_objects.py @@ -446,8 +446,8 @@ class _Struct(ctypes.Structure): try: import symtable - d["SymtableEntryType"] = symtable.symtable("", "string", "exec")._table -except: #FIXME: fails; should not be registered (in _reverse__typemap)? + x["SymtableEntryType"] = symtable.symtable("", "string", "exec")._table +except: #FIXME: fails to pickle pass if sys.hexversion >= 0x30a00a0: From ecf60d0f71f8cff708fc92af09ad2b93e3da515a Mon Sep 17 00:00:00 2001 From: mmckerns Date: Sun, 26 Jun 2022 08:34:16 -0400 Subject: [PATCH 09/13] help silence some test noise --- dill/_dill.py | 5 ++++- dill/_objects.py | 6 +++--- tests/test_check.py | 20 ++++++++++---------- tests/test_objects.py | 2 ++ tests/test_selected.py | 7 +++++++ 5 files changed, 26 insertions(+), 14 deletions(-) diff --git a/dill/_dill.py b/dill/_dill.py index 1d7aee50..01eb79d2 100644 --- a/dill/_dill.py +++ b/dill/_dill.py @@ -2373,7 +2373,10 @@ def check(obj, *args, **kwds): # unpickle = "dill.loads(%s, ignore=%s)"%(repr(_obj), repr(ignore)) # cmd = [python, "-c", "import dill; print(%s)"%unpickle] # msg = "SUCCESS" if not subprocess.call(cmd) else "LOAD FAILED" - msg = "%s -c import dill; print(dill.loads(%s))" % (python, repr(_obj)) + if verbose is None: + msg = "%s -c import dill; dill.loads(%s)" % (python, repr(_obj)) + else: + msg = "%s -c import dill; print(dill.loads(%s))" % (python, repr(_obj)) msg = "SUCCESS" if not subprocess.call(msg.split(None,2)) else "LOAD FAILED" if verbose: print(msg) diff --git a/dill/_objects.py b/dill/_objects.py index 31797770..954a3c04 100644 --- a/dill/_objects.py +++ b/dill/_objects.py @@ -417,7 +417,7 @@ class _Struct(ctypes.Structure): # built-in functions (CH 2) # Iterators: a['ListIteratorType'] = iter(_list) # empty vs non-empty -a['SetIteratorType'] = iter(_set) #XXX: empty vs non-empty +a['SetIteratorType'] = iter(_set) #XXX: empty vs non-empty #FIXME: list_iterator a['TupleIteratorType']= iter(_tuple) # empty vs non-empty a['XRangeIteratorType'] = iter(_xrange) # empty vs non-empty a["BytesIteratorType"] = iter(b'') @@ -429,9 +429,9 @@ class _Struct(ctypes.Structure): d["OdictKeysType"] = X.keys() d["OdictValuesType"] = X.values() d["OdictItemsType"] = X.items() -a["OdictIteratorType"] = iter(X.keys()) +a["OdictIteratorType"] = iter(X.keys()) #FIXME: list_iterator del X -if PY3: +if PY3: #FIXME: list_iterator a['DictionaryItemIteratorType'] = iter(type.__dict__.items()) a['DictionaryKeyIteratorType'] = iter(type.__dict__.keys()) a['DictionaryValueIteratorType'] = iter(type.__dict__.values()) diff --git a/tests/test_check.py b/tests/test_check.py index 134b8b0c..3b3fc590 100644 --- a/tests/test_check.py +++ b/tests/test_check.py @@ -30,24 +30,24 @@ def raise_check(func, **kwds): f = lambda x:x**2 -def test_simple(): - raise_check(f) +def test_simple(verbose=None): + raise_check(f, verbose=verbose) -def test_recurse(): - raise_check(f, recurse=True) +def test_recurse(verbose=None): + raise_check(f, recurse=True, verbose=verbose) -def test_byref(): - raise_check(f, byref=True) +def test_byref(verbose=None): + raise_check(f, byref=True, verbose=verbose) -def test_protocol(): - raise_check(f, protocol=True) +def test_protocol(verbose=None): + raise_check(f, protocol=True, verbose=verbose) -def test_python(): - raise_check(f, python=None) +def test_python(verbose=None): + raise_check(f, python=None, verbose=verbose) #TODO: test incompatible versions diff --git a/tests/test_objects.py b/tests/test_objects.py index c83060c3..e69a2293 100644 --- a/tests/test_objects.py +++ b/tests/test_objects.py @@ -58,4 +58,6 @@ def test_objects(): pickles(member, exact=False) if __name__ == '__main__': + import warnings + warnings.simplefilter('ignore') test_objects() diff --git a/tests/test_selected.py b/tests/test_selected.py index bd790835..9eef8d2d 100644 --- a/tests/test_selected.py +++ b/tests/test_selected.py @@ -42,6 +42,13 @@ def _method(self): from dill import load_types load_types(pickleable=True,unpickleable=False) _newclass = objects['ClassObjectType'] +# some clean-up #FIXME: should happen internal to dill +objects['TemporaryFileType'].close() +objects['TextWrapperType'].close() +objects['BufferedRandomType'].close() +objects['BufferedReaderType'].close() +objects['BufferedWriterType'].close() +objects['FileType'].close() del objects # getset_descriptor for new-style classes (fails on '_method', if not __main__) From 38d2a9508ce285a358e3d4e144dc5a9a3c7f2f10 Mon Sep 17 00:00:00 2001 From: mmckerns Date: Sun, 26 Jun 2022 08:35:32 -0400 Subject: [PATCH 10/13] better python handling in test main --- tests/__main__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/__main__.py b/tests/__main__.py index 312e5dae..7f7be536 100644 --- a/tests/__main__.py +++ b/tests/__main__.py @@ -5,17 +5,17 @@ # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/dill/blob/master/LICENSE -from __future__ import print_function import glob import os +import sys +import subprocess as sp +python = sys.executable try: import pox - python = pox.which_python(version=True, fullpath=False) or 'python' + python = pox.which_python(version=True) or python except ImportError: - python = 'python' -import subprocess as sp -from sys import platform -shell = platform[:3] == 'win' + pass +shell = sys.platform[:3] == 'win' suite = os.path.dirname(__file__) or os.path.curdir tests = glob.glob(suite + os.path.sep + 'test_*.py') From 9000ab949fdd0d8d9e0c5a4a1b3d3959151d2260 Mon Sep 17 00:00:00 2001 From: Leonardo Gama Date: Fri, 1 Jul 2022 18:02:16 -0300 Subject: [PATCH 11/13] Add detailed trace mode showing saved object size and visual depth level (#471) * Add detailed trace mode showing dumped object size and visual depeth level * Tests: logging code coverage * logger: just show size after writing object * Box drawing formatting * fallback for non-UTF-8 terminals * review: corrections and improvements * review: adjustments * split module imports into separated lines * deal with non-standardized UTF-8 encoding name * update sample trace in module docs * use trace() as a context manager * pypy special cases * accept file handle in trace context manager * fix dict (PyPy) and mappingproxy (CPython) trace Co-authored-by: anivegesana --- README.md | 36 +++--- dill/__init__.py | 36 +++--- dill/_dill.py | 285 +++++++++++++++++++++---------------------- dill/detect.py | 2 +- dill/logger.py | 284 ++++++++++++++++++++++++++++++++++++++++++ tests/test_logger.py | 70 +++++++++++ 6 files changed, 537 insertions(+), 176 deletions(-) create mode 100644 dill/logger.py create mode 100644 tests/test_logger.py diff --git a/README.md b/README.md index 2b3688e6..d94e1b4a 100644 --- a/README.md +++ b/README.md @@ -170,26 +170,30 @@ To aid in debugging pickling issues, use *dill.detect* which provides tools like pickle tracing:: >>> import dill.detect - >>> dill.detect.trace(True) - >>> f = dumps(squared) - F1: at 0x108899e18> - F2: - # F2 - Co: at 0x10866a270, file "", line 1> - F2: - # F2 - # Co - D1: - # D1 - D2: - # D2 - # F1 - >>> dill.detect.trace(False) + >>> with dill.detect.trace(): + >>> dumps(squared) + ┬ F1: at 0x7fe074f8c280> + ├┬ F2: + │└ # F2 [34 B] + ├┬ Co: at 0x7fe07501eb30, file "", line 1> + │├┬ F2: + ││└ # F2 [19 B] + │└ # Co [87 B] + ├┬ D1: + │└ # D1 [22 B] + ├┬ D2: + │└ # D2 [2 B] + ├┬ D2: + │├┬ D2: + ││└ # D2 [2 B] + │└ # D2 [23 B] + └ # F1 [180 B] With trace, we see how ``dill`` stored the lambda (``F1``) by first storing ``_create_function``, the underlying code object (``Co``) and ``_create_code`` (which is used to handle code objects), then we handle the reference to -the global dict (``D2``). A ``#`` marks when the object is actually stored. +the global dict (``D2``) plus other dictionaries (``D1`` and ``D2``) that +save the lambda object's state. A ``#`` marks when the object is actually stored. More Information diff --git a/dill/__init__.py b/dill/__init__.py index ac93ff6a..39ac4df2 100644 --- a/dill/__init__.py +++ b/dill/__init__.py @@ -185,26 +185,30 @@ tools like pickle tracing:: >>> import dill.detect - >>> dill.detect.trace(True) - >>> f = dumps(squared) - F1: at 0x108899e18> - F2: - # F2 - Co: at 0x10866a270, file "", line 1> - F2: - # F2 - # Co - D1: - # D1 - D2: - # D2 - # F1 - >>> dill.detect.trace(False) + >>> with dill.detect.trace(): + >>> dumps(squared) + ┬ F1: at 0x7fe074f8c280> + ├┬ F2: + │└ # F2 [34 B] + ├┬ Co: at 0x7fe07501eb30, file "", line 1> + │├┬ F2: + ││└ # F2 [19 B] + │└ # Co [87 B] + ├┬ D1: + │└ # D1 [22 B] + ├┬ D2: + │└ # D2 [2 B] + ├┬ D2: + │├┬ D2: + ││└ # D2 [2 B] + │└ # D2 [23 B] + └ # F1 [180 B] With trace, we see how ``dill`` stored the lambda (``F1``) by first storing ``_create_function``, the underlying code object (``Co``) and ``_create_code`` (which is used to handle code objects), then we handle the reference to -the global dict (``D2``). A ``#`` marks when the object is actually stored. +the global dict (``D2``) plus other dictionaries (``D1`` and ``D2``) that +save the lambda object's state. A ``#`` marks when the object is actually stored. More Information diff --git a/dill/_dill.py b/dill/_dill.py index 01eb79d2..7660cb07 100644 --- a/dill/_dill.py +++ b/dill/_dill.py @@ -23,15 +23,9 @@ __module__ = 'dill' -import logging -log = logging.getLogger("dill") -log.addHandler(logging.StreamHandler()) -def _trace(boolean): - """print a trace through the stack when pickling; useful for debugging""" - if boolean: log.setLevel(logging.INFO) - else: log.setLevel(logging.WARN) - return import warnings +from .logger import adapter as logger +from .logger import trace as _trace import os import sys @@ -476,6 +470,7 @@ def dump_session(filename='/tmp/session.pkl', main=None, byref=False, **kwds): f = open(filename, 'wb') try: pickler = Pickler(f, protocol, **kwds) + pickler._file = f pickler._original_main = main if byref: main = _stash_modules(main) @@ -542,13 +537,13 @@ class Pickler(StockPickler): _session = False from .settings import settings - def __init__(self, *args, **kwds): + def __init__(self, file, *args, **kwds): settings = Pickler.settings _byref = kwds.pop('byref', None) #_strictio = kwds.pop('strictio', None) _fmode = kwds.pop('fmode', None) _recurse = kwds.pop('recurse', None) - StockPickler.__init__(self, *args, **kwds) + StockPickler.__init__(self, file, *args, **kwds) self._main = _main_module self._diff_cache = {} self._byref = settings['byref'] if _byref is None else _byref @@ -556,6 +551,7 @@ def __init__(self, *args, **kwds): self._fmode = settings['fmode'] if _fmode is None else _fmode self._recurse = settings['recurse'] if _recurse is None else _recurse self._postproc = OrderedDict() + self._file = file def dump(self, obj): #NOTE: if settings change, need to update attributes # register if the object is a numpy ufunc @@ -563,10 +559,10 @@ def dump(self, obj): #NOTE: if settings change, need to update attributes if NumpyUfuncType and numpyufunc(obj): @register(type(obj)) def save_numpy_ufunc(pickler, obj): - log.info("Nu: %s" % obj) + logger.trace(pickler, "Nu: %s", obj) name = getattr(obj, '__qualname__', getattr(obj, '__name__', None)) StockPickler.save_global(pickler, obj, name=name) - log.info("# Nu") + logger.trace(pickler, "# Nu") return # NOTE: the above 'save' performs like: # import copy_reg @@ -577,9 +573,9 @@ def save_numpy_ufunc(pickler, obj): if NumpyDType and numpydtype(obj): @register(type(obj)) def save_numpy_dtype(pickler, obj): - log.info("Dt: %s" % obj) + logger.trace(pickler, "Dt: %s", obj) pickler.save_reduce(_create_dtypemeta, (obj.type,), obj=obj) - log.info("# Dt") + logger.trace(pickler, "# Dt") return # NOTE: the above 'save' performs like: # import copy_reg @@ -590,21 +586,20 @@ def save_numpy_dtype(pickler, obj): if NumpyArrayType and ndarraysubclassinstance(obj): @register(type(obj)) def save_numpy_array(pickler, obj): - log.info("Nu: (%s, %s)" % (obj.shape,obj.dtype)) + logger.trace(pickler, "Nu: (%s, %s)", obj.shape, obj.dtype) npdict = getattr(obj, '__dict__', None) f, args, state = obj.__reduce__() pickler.save_reduce(_create_array, (f,args,state,npdict), obj=obj) - log.info("# Nu") + logger.trace(pickler, "# Nu") return # end hack if GENERATOR_FAIL and type(obj) == GeneratorType: msg = "Can't pickle %s: attribute lookup builtins.generator failed" % GeneratorType raise PicklingError(msg) - else: - StockPickler.dump(self, obj) - return + logger.trace_setup(self) + StockPickler.dump(self, obj) + dump.__doc__ = StockPickler.dump.__doc__ - pass class Unpickler(StockUnpickler): """python's Unpickler extended to interpreter sessions and more types""" @@ -1382,9 +1377,9 @@ def _save_with_postproc(pickler, reduction, is_pickler_dill=None, obj=Getattr.NO #@register(CodeType) #def save_code(pickler, obj): -# log.info("Co: %s" % obj) +# logger.trace(pickler, "Co: %s", obj) # pickler.save_reduce(_unmarshal, (marshal.dumps(obj),), obj=obj) -# log.info("# Co") +# logger.trace(pickler, "# Co") # return # The following function is based on 'save_codeobject' from 'cloudpickle' @@ -1393,7 +1388,7 @@ def _save_with_postproc(pickler, reduction, is_pickler_dill=None, obj=Getattr.NO # License: https://github.com/cloudpipe/cloudpickle/blob/master/LICENSE @register(CodeType) def save_code(pickler, obj): - log.info("Co: %s" % obj) + logger.trace(pickler, "Co: %s", obj) if PY3: if hasattr(obj, "co_endlinetable"): # python 3.11a (20 args) args = ( @@ -1452,42 +1447,46 @@ def save_code(pickler, obj): ) pickler.save_reduce(_create_code, args, obj=obj) - log.info("# Co") + logger.trace(pickler, "# Co") return +def _repr_dict(obj): + """make a short string representation of a dictionary""" + return "<%s object at %#012x>" % (type(obj).__name__, id(obj)) + @register(dict) def save_module_dict(pickler, obj): if is_dill(pickler, child=False) and obj == pickler._main.__dict__ and \ not (pickler._session and pickler._first_pass): - log.info("D1: " % (obj,)) + logger.trace(pickler, "Dkvi: <%s>", obj) mapping = obj.mapping | _dictproxy_helper_instance pickler.save_reduce(func, (mapping,), obj=obj) - log.info("# Dkvi") + logger.trace(pickler, "# Dkvi") return _save_dict_view return [ (funcname, save_dict_view_for_function(getattr(dicttype, funcname))) @@ -1512,21 +1511,21 @@ def _save_dict_view(pickler, obj): # License: https://github.com/cloudpipe/cloudpickle/blob/master/LICENSE def save_dict_view(dicttype): def save_dict_keys(pickler, obj): - log.info("Dk: <%s>" % (obj,)) + logger.trace(pickler, "Dk: <%s>", obj) dict_constructor = _shims.Reduce(dicttype.fromkeys, (list(obj),)) pickler.save_reduce(dicttype.keys, (dict_constructor,), obj=obj) - log.info("# Dk") + logger.trace(pickler, "# Dk") def save_dict_values(pickler, obj): - log.info("Dv: <%s>" % (obj,)) + logger.trace(pickler, "Dv: <%s>", obj) dict_constructor = _shims.Reduce(dicttype, (enumerate(obj),)) pickler.save_reduce(dicttype.values, (dict_constructor,), obj=obj) - log.info("# Dv") + logger.trace(pickler, "# Dv") def save_dict_items(pickler, obj): - log.info("Di: <%s>" % (obj,)) + logger.trace(pickler, "Di: <%s>", obj) pickler.save_reduce(dicttype.items, (dicttype(obj),), obj=obj) - log.info("# Di") + logger.trace(pickler, "# Di") return ( ('keys', save_dict_keys), @@ -1549,61 +1548,61 @@ def save_dict_items(pickler, obj): @register(ClassType) def save_classobj(pickler, obj): #FIXME: enable pickler._byref if not _locate_function(obj, pickler): - log.info("C1: %s" % obj) + logger.trace(pickler, "C1: %s", obj) pickler.save_reduce(ClassType, (obj.__name__, obj.__bases__, obj.__dict__), obj=obj) #XXX: or obj.__dict__.copy()), obj=obj) ? - log.info("# C1") + logger.trace(pickler, "# C1") else: - log.info("C2: %s" % obj) + logger.trace(pickler, "C2: %s", obj) name = getattr(obj, '__qualname__', getattr(obj, '__name__', None)) StockPickler.save_global(pickler, obj, name=name) - log.info("# C2") + logger.trace(pickler, "# C2") return @register(LockType) def save_lock(pickler, obj): - log.info("Lo: %s" % obj) + logger.trace(pickler, "Lo: %s", obj) pickler.save_reduce(_create_lock, (obj.locked(),), obj=obj) - log.info("# Lo") + logger.trace(pickler, "# Lo") return @register(RLockType) def save_rlock(pickler, obj): - log.info("RL: %s" % obj) + logger.trace(pickler, "RL: %s", obj) r = obj.__repr__() # don't use _release_save as it unlocks the lock count = int(r.split('count=')[1].split()[0].rstrip('>')) owner = int(r.split('owner=')[1].split()[0]) if PY3 else getattr(obj, '_RLock__owner') pickler.save_reduce(_create_rlock, (count,owner,), obj=obj) - log.info("# RL") + logger.trace(pickler, "# RL") return if not IS_PYPY2: #@register(SocketType) #FIXME: causes multiprocess test_pickling FAIL def save_socket(pickler, obj): - log.info("So: %s" % obj) + logger.trace(pickler, "So: %s", obj) pickler.save_reduce(*reduce_socket(obj)) - log.info("# So") + logger.trace(pickler, "# So") return if sys.hexversion <= 0x3050000: @register(ItemGetterType) def save_itemgetter(pickler, obj): - log.info("Ig: %s" % obj) + logger.trace(pickler, "Ig: %s", obj) helper = _itemgetter_helper() obj(helper) pickler.save_reduce(type(obj), tuple(helper.items), obj=obj) - log.info("# Ig") + logger.trace(pickler, "# Ig") return @register(AttrGetterType) def save_attrgetter(pickler, obj): - log.info("Ag: %s" % obj) + logger.trace(pickler, "Ag: %s", obj) attrs = [] helper = _attrgetter_helper(attrs) obj(helper) pickler.save_reduce(type(obj), tuple(attrs), obj=obj) - log.info("# Ag") + logger.trace(pickler, "# Ag") return def _save_file(pickler, obj, open_): @@ -1639,9 +1638,9 @@ def _save_file(pickler, obj, open_): @register(BufferedWriterType) @register(TextWrapperType) def save_file(pickler, obj): - log.info("Fi: %s" % obj) + logger.trace(pickler, "Fi: %s", obj) f = _save_file(pickler, obj, open) - log.info("# Fi") + logger.trace(pickler, "# Fi") return f if PyTextWrapperType: @@ -1650,9 +1649,9 @@ def save_file(pickler, obj): @register(PyBufferedWriterType) @register(PyTextWrapperType) def save_file(pickler, obj): - log.info("Fi: %s" % obj) + logger.trace(pickler, "Fi: %s", obj) f = _save_file(pickler, obj, _open) - log.info("# Fi") + logger.trace(pickler, "# Fi") return f # The following two functions are based on 'saveCStringIoInput' @@ -1662,42 +1661,42 @@ def save_file(pickler, obj): if InputType: @register(InputType) def save_stringi(pickler, obj): - log.info("Io: %s" % obj) + logger.trace(pickler, "Io: %s", obj) if obj.closed: value = ''; position = 0 else: value = obj.getvalue(); position = obj.tell() pickler.save_reduce(_create_stringi, (value, position, \ obj.closed), obj=obj) - log.info("# Io") + logger.trace(pickler, "# Io") return @register(OutputType) def save_stringo(pickler, obj): - log.info("Io: %s" % obj) + logger.trace(pickler, "Io: %s", obj) if obj.closed: value = ''; position = 0 else: value = obj.getvalue(); position = obj.tell() pickler.save_reduce(_create_stringo, (value, position, \ obj.closed), obj=obj) - log.info("# Io") + logger.trace(pickler, "# Io") return if 0x2050000 <= sys.hexversion < 0x3010000: @register(PartialType) def save_functor(pickler, obj): - log.info("Fu: %s" % obj) + logger.trace(pickler, "Fu: %s", obj) pickler.save_reduce(_create_ftype, (type(obj), obj.func, obj.args, obj.keywords), obj=obj) - log.info("# Fu") + logger.trace(pickler, "# Fu") return if LRUCacheType is not None: from functools import lru_cache @register(LRUCacheType) def save_lru_cache(pickler, obj): - log.info("LRU: %s" % obj) + logger.trace(pickler, "LRU: %s", obj) if OLD39: kwargs = obj.cache_info() args = (kwargs.maxsize,) @@ -1709,14 +1708,14 @@ def save_lru_cache(pickler, obj): else: wrapper = lru_cache pickler.save_reduce(wrapper, (obj.__wrapped__,), obj=obj) - log.info("# LRU") + logger.trace(pickler, "# LRU") return @register(SuperType) def save_super(pickler, obj): - log.info("Su: %s" % obj) + logger.trace(pickler, "Su: %s", obj) pickler.save_reduce(super, (obj.__thisclass__, obj.__self__), obj=obj) - log.info("# Su") + logger.trace(pickler, "# Su") return if OLDER or not PY3: @@ -1726,23 +1725,23 @@ def save_builtin_method(pickler, obj): if obj.__self__ is __builtin__: module = 'builtins' if PY3 else '__builtin__' _t = "B1" - log.info("%s: %s" % (_t, obj)) + logger.trace(pickler, "%s: %s", _t, obj) else: module = obj.__self__ _t = "B3" - log.info("%s: %s" % (_t, obj)) + logger.trace(pickler, "%s: %s", _t, obj) if is_dill(pickler, child=True): _recurse = pickler._recurse pickler._recurse = False pickler.save_reduce(_get_attr, (module, obj.__name__), obj=obj) if is_dill(pickler, child=True): pickler._recurse = _recurse - log.info("# %s" % _t) + logger.trace(pickler, "# %s", _t) else: - log.info("B2: %s" % obj) + logger.trace(pickler, "B2: %s", obj) name = getattr(obj, '__qualname__', getattr(obj, '__name__', None)) StockPickler.save_global(pickler, obj, name=name) - log.info("# B2") + logger.trace(pickler, "# B2") return if IS_PYPY: @@ -1752,22 +1751,22 @@ def save_instancemethod0(pickler, obj): if code is not None and type(code) is not CodeType \ and getattr(obj.__self__, obj.__name__) == obj: # Some PyPy builtin functions have no module name - log.info("Me2: %s" % obj) + logger.trace(pickler, "Me2: %s" % obj) # TODO: verify that this works for all PyPy builtin methods pickler.save_reduce(getattr, (obj.__self__, obj.__name__), obj=obj) - log.info("# Me2") + logger.trace(pickler, "# Me2") return - log.info("Me1: %s" % obj) + logger.trace(pickler, "Me1: %s" % obj) pickler.save_reduce(MethodType, (obj.__func__, obj.__self__), obj=obj) - log.info("# Me1") + logger.trace(pickler, "# Me1") return else: @register(MethodType) def save_instancemethod0(pickler, obj): - log.info("Me1: %s" % obj) + logger.trace(pickler, "Me1: %s" % obj) pickler.save_reduce(MethodType, (obj.__func__, obj.__self__), obj=obj) - log.info("# Me1") + logger.trace(pickler, "# Me1") return if sys.hexversion >= 0x20500f0: @@ -1778,29 +1777,29 @@ def save_instancemethod0(pickler, obj): @register(WrapperDescriptorType) @register(ClassMethodDescriptorType) def save_wrapper_descriptor(pickler, obj): - log.info("Wr: %s" % obj) + logger.trace(pickler, "Wr: %s", obj) pickler.save_reduce(_getattr, (obj.__objclass__, obj.__name__, obj.__repr__()), obj=obj) - log.info("# Wr") + logger.trace(pickler, "# Wr") return else: @register(MemberDescriptorType) @register(GetSetDescriptorType) def save_wrapper_descriptor(pickler, obj): - log.info("Wr: %s" % obj) + logger.trace(pickler, "Wr: %s", obj) pickler.save_reduce(_getattr, (obj.__objclass__, obj.__name__, obj.__repr__()), obj=obj) - log.info("# Wr") + logger.trace(pickler, "# Wr") return elif not IS_PYPY: @register(MethodDescriptorType) @register(WrapperDescriptorType) def save_wrapper_descriptor(pickler, obj): - log.info("Wr: %s" % obj) + logger.trace(pickler, "Wr: %s", obj) pickler.save_reduce(_getattr, (obj.__objclass__, obj.__name__, obj.__repr__()), obj=obj) - log.info("# Wr") + logger.trace(pickler, "# Wr") return @register(CellType) @@ -1808,7 +1807,7 @@ def save_cell(pickler, obj): try: f = obj.cell_contents except: - log.info("Ce3: %s" % obj) + logger.trace(pickler, "Ce3: %s", obj) # _shims._CELL_EMPTY is defined in _shims.py to support PyPy 2.7. # It unpickles to a sentinel object _dill._CELL_EMPTY, also created in # _shims.py. This object is not present in Python 3 because the cell's @@ -1827,7 +1826,7 @@ def save_cell(pickler, obj): pickler.write(bytes('0', 'UTF-8')) else: pickler.write('0') - log.info("# Ce3") + logger.trace(pickler, "# Ce3") return if is_dill(pickler, child=True): if id(f) in pickler._postproc: @@ -1838,49 +1837,49 @@ def save_cell(pickler, obj): # value as late as possible to prevent cycle. postproc = next(iter(pickler._postproc.values()), None) if postproc is not None: - log.info("Ce2: %s" % obj) + logger.trace(pickler, "Ce2: %s", obj) # _CELL_REF is defined in _shims.py to support older versions of # dill. When breaking changes are made to dill, (_CELL_REF,) can # be replaced by () pickler.save_reduce(_create_cell, (_CELL_REF,), obj=obj) postproc.append((_shims._setattr, (obj, 'cell_contents', f))) - log.info("# Ce2") + logger.trace(pickler, "# Ce2") return - log.info("Ce1: %s" % obj) + logger.trace(pickler, "Ce1: %s", obj) pickler.save_reduce(_create_cell, (f,), obj=obj) - log.info("# Ce1") + logger.trace(pickler, "# Ce1") return if MAPPING_PROXY_TRICK: @register(DictProxyType) def save_dictproxy(pickler, obj): - log.info("Mp: %s" % obj) + logger.trace(pickler, "Mp: %s", _repr_dict(obj)) # obj mapping = obj | _dictproxy_helper_instance pickler.save_reduce(DictProxyType, (mapping,), obj=obj) - log.info("# Mp") + logger.trace(pickler, "# Mp") return else: @register(DictProxyType) def save_dictproxy(pickler, obj): - log.info("Mp: %s" % obj) + logger.trace(pickler, "Mp: %s", _repr_dict(obj)) # obj pickler.save_reduce(DictProxyType, (obj.copy(),), obj=obj) - log.info("# Mp") + logger.trace(pickler, "# Mp") return @register(SliceType) def save_slice(pickler, obj): - log.info("Sl: %s" % obj) + logger.trace(pickler, "Sl: %s", obj) pickler.save_reduce(slice, (obj.start, obj.stop, obj.step), obj=obj) - log.info("# Sl") + logger.trace(pickler, "# Sl") return @register(XRangeType) @register(EllipsisType) @register(NotImplementedType) def save_singleton(pickler, obj): - log.info("Si: %s" % obj) + logger.trace(pickler, "Si: %s", obj) pickler.save_reduce(_eval_repr, (obj.__repr__(),), obj=obj) - log.info("# Si") + logger.trace(pickler, "# Si") return def _proxy_helper(obj): # a dead proxy returns a reference to None @@ -1926,10 +1925,10 @@ def _locate_object(address, module=None): @register(ReferenceType) def save_weakref(pickler, obj): refobj = obj() - log.info("R1: %s" % obj) + logger.trace(pickler, "R1: %s", obj) #refobj = ctypes.pythonapi.PyWeakref_GetObject(obj) # dead returns "None" pickler.save_reduce(_create_weakref, (refobj,), obj=obj) - log.info("# R1") + logger.trace(pickler, "# R1") return @register(ProxyType) @@ -1938,15 +1937,15 @@ def save_weakproxy(pickler, obj): refobj = _locate_object(_proxy_helper(obj)) try: _t = "R2" - log.info("%s: %s" % (_t, obj)) + logger.trace(pickler, "%s: %s", _t, obj) except ReferenceError: _t = "R3" - log.info("%s: %s" % (_t, sys.exc_info()[1])) + logger.trace(pickler, "%s: %s", _t, sys.exc_info()[1]) #callable = bool(getattr(refobj, '__call__', None)) if type(obj) is CallableProxyType: callable = True else: callable = False pickler.save_reduce(_create_weakproxy, (refobj, callable), obj=obj) - log.info("# %s" % _t) + logger.trace(pickler, "# %s", _t) return def _is_builtin_module(module): @@ -1968,74 +1967,74 @@ def save_module(pickler, obj): except RuntimeError: # not memorised module, probably part of dill pass else: - log.info("M2: %s with diff" % obj) - log.info("Diff: %s", changed.keys()) + logger.trace(pickler, "M2: %s with diff", obj) + logger.trace(pickler, "Diff: %s", changed.keys()) pickler.save_reduce(_import_module, (obj.__name__,), obj=obj, state=changed) - log.info("# M2") + logger.trace(pickler, "# M2") return - log.info("M1: %s" % obj) + logger.trace(pickler, "M1: %s", obj) pickler.save_reduce(_import_module, (obj.__name__,), obj=obj) - log.info("# M1") + logger.trace(pickler, "# M1") else: builtin_mod = _is_builtin_module(obj) if obj.__name__ not in ("builtins", "dill", "dill._dill") and not builtin_mod or \ is_dill(pickler, child=True) and obj is pickler._main: - log.info("M1: %s" % obj) + logger.trace(pickler, "M1: %s", obj) _main_dict = obj.__dict__.copy() #XXX: better no copy? option to copy? [_main_dict.pop(item, None) for item in singletontypes + ["__builtins__", "__loader__"]] pickler.save_reduce(_import_module, (obj.__name__,), obj=obj, state=_main_dict) - log.info("# M1") + logger.trace(pickler, "# M1") elif PY3 and obj.__name__ == "dill._dill": - log.info("M2: %s" % obj) + logger.trace(pickler, "M2: %s", obj) pickler.save_global(obj, name="_dill") - log.info("# M2") + logger.trace(pickler, "# M2") else: - log.info("M2: %s" % obj) + logger.trace(pickler, "M2: %s", obj) pickler.save_reduce(_import_module, (obj.__name__,), obj=obj) - log.info("# M2") + logger.trace(pickler, "# M2") return return @register(TypeType) def save_type(pickler, obj, postproc_list=None): if obj in _typemap: - log.info("T1: %s" % obj) + logger.trace(pickler, "T1: %s" % obj) # if obj in _incedental_types: # warnings.warn('Type %r may only exist on this implementation of Python and cannot be unpickled in other implementations.' % (obj,), PicklingWarning) pickler.save_reduce(_load_type, (_typemap[obj],), obj=obj) - log.info("# T1") + logger.trace(pickler, "# T1") elif obj.__bases__ == (tuple,) and all([hasattr(obj, attr) for attr in ('_fields','_asdict','_make','_replace')]): # special case: namedtuples - log.info("T6: %s" % obj) + logger.trace(pickler, "T6: %s", obj) if OLD37 or (not obj._field_defaults): pickler.save_reduce(_create_namedtuple, (obj.__name__, obj._fields, obj.__module__), obj=obj) else: defaults = [obj._field_defaults[field] for field in obj._fields if field in obj._field_defaults] pickler.save_reduce(_create_namedtuple, (obj.__name__, obj._fields, obj.__module__, defaults), obj=obj) - log.info("# T6") + logger.trace(pickler, "# T6") return # special cases: NoneType, NotImplementedType, EllipsisType elif obj is type(None): - log.info("T7: %s" % obj) + logger.trace(pickler, "T7: %s", obj) #XXX: pickler.save_reduce(type, (None,), obj=obj) if PY3: pickler.write(bytes('c__builtin__\nNoneType\n', 'UTF-8')) else: pickler.write('c__builtin__\nNoneType\n') - log.info("# T7") + logger.trace(pickler, "# T7") elif obj is NotImplementedType: - log.info("T7: %s" % obj) + logger.trace(pickler, "T7: %s", obj) pickler.save_reduce(type, (NotImplemented,), obj=obj) - log.info("# T7") + logger.trace(pickler, "# T7") elif obj is EllipsisType: - log.info("T7: %s" % obj) + logger.trace(pickler, "T7: %s", obj) pickler.save_reduce(type, (Ellipsis,), obj=obj) - log.info("# T7") + logger.trace(pickler, "# T7") else: obj_name = getattr(obj, '__qualname__', getattr(obj, '__name__', None)) @@ -2046,11 +2045,11 @@ def save_type(pickler, obj, postproc_list=None): if issubclass(type(obj), type): # thanks to Tom Stepleton pointing out pickler._session unneeded _t = 'T2' - log.info("%s: %s" % (_t, obj)) + logger.trace(pickler, "%s: %s", _t, obj) _dict = _dict_from_dictproxy(obj.__dict__) else: _t = 'T3' - log.info("%s: %s" % (_t, obj)) + logger.trace(pickler, "%s: %s", _t, obj) _dict = obj.__dict__ #print (_dict) #print ("%s\n%s" % (type(obj), obj.__name__)) @@ -2064,9 +2063,9 @@ def save_type(pickler, obj, postproc_list=None): _save_with_postproc(pickler, (_create_type, ( type(obj), obj.__name__, obj.__bases__, _dict )), obj=obj, postproc_list=postproc_list) - log.info("# %s" % _t) + logger.trace(pickler, "# %s", _t) else: - log.info("T4: %s" % obj) + logger.trace(pickler, "T4: %s", obj) if incorrectly_named: warnings.warn('Cannot locate reference to %r.' % (obj,), PicklingWarning) if obj_recursive: @@ -2075,7 +2074,7 @@ def save_type(pickler, obj, postproc_list=None): #print ("%s\n%s" % (type(obj), obj.__name__)) #print ("%s\n%s" % (obj.__bases__, obj.__dict__)) StockPickler.save_global(pickler, obj, name=obj_name) - log.info("# T4") + logger.trace(pickler, "# T4") return # Error in PyPy 2.7 when adding ABC support @@ -2086,15 +2085,15 @@ def save_frame(pickler, obj): @register(property) def save_property(pickler, obj): - log.info("Pr: %s" % obj) + logger.trace(pickler, "Pr: %s", obj) pickler.save_reduce(property, (obj.fget, obj.fset, obj.fdel, obj.__doc__), obj=obj) - log.info("# Pr") + logger.trace(pickler, "# Pr") @register(staticmethod) @register(classmethod) def save_classmethod(pickler, obj): - log.info("Cm: %s" % obj) + logger.trace(pickler, "Cm: %s", obj) im_func = '__func__' if PY3 else 'im_func' try: orig_func = getattr(obj, im_func) @@ -2115,7 +2114,7 @@ def save_classmethod(pickler, obj): # state = None pickler.save_reduce(type(obj), (orig_func,), obj=obj) - log.info("# Cm") + logger.trace(pickler, "# Cm") @register(FunctionType) def save_function(pickler, obj): @@ -2136,12 +2135,12 @@ def save_function(pickler, obj): pass if _pypy_builtin: - log.info("F3: %s" % obj) + logger.trace(pickler, "F3: %s" % obj) pickler.save_reduce(getattr, (found, '__func__'), obj=obj) - log.info("# F3") + logger.trace(pickler, "# F3") return - log.info("F1: %s" % obj) + logger.trace(pickler, "F1: %s" % obj) _recurse = getattr(pickler, '_recurse', None) _postproc = getattr(pickler, '_postproc', None) _main_modified = getattr(pickler, '_main_modified', None) @@ -2243,12 +2242,12 @@ def save_function(pickler, obj): else: pickler.write('0') - log.info("# F1") + logger.trace(pickler, "# F1") else: - log.info("F2: %s" % obj) + logger.trace(pickler, "F2: %s", obj) name = getattr(obj, '__qualname__', getattr(obj, '__name__', None)) StockPickler.save_global(pickler, obj, name=name) - log.info("# F2") + logger.trace(pickler, "# F2") return if HAS_CTYPES and hasattr(ctypes, 'pythonapi'): @@ -2286,14 +2285,14 @@ def save_function(pickler, obj): PyCapsuleType = type(_testcapsule) @register(PyCapsuleType) def save_capsule(pickler, obj): - log.info("Cap: %s", obj) + logger.trace(pickler, "Cap: %s", obj) name = _PyCapsule_GetName(obj) warnings.warn('Pickling a PyCapsule (%s) does not pickle any C data structures and could cause segmentation faults or other memory errors when unpickling.' % (name,), PicklingWarning) pointer = _PyCapsule_GetPointer(obj, name) context = _PyCapsule_GetContext(obj) destructor = _PyCapsule_GetDestructor(obj) pickler.save_reduce(_create_capsule, (pointer, name, context, destructor), obj=obj) - log.info("# Cap") + logger.trace(pickler, "# Cap") _incedental_reverse_typemap['PyCapsuleType'] = PyCapsuleType _reverse_typemap['PyCapsuleType'] = PyCapsuleType _incedental_types.add(PyCapsuleType) @@ -2396,7 +2395,7 @@ def _extend(): try: StockPickler.dispatch[t] = func except: #TypeError, PicklingError, UnpicklingError - log.info("skip: %s" % t) + logger.trace(pickler, "skip: %s", t) else: pass return diff --git a/dill/detect.py b/dill/detect.py index 41575205..26d5b219 100644 --- a/dill/detect.py +++ b/dill/detect.py @@ -13,8 +13,8 @@ from inspect import ismethod, isfunction, istraceback, isframe, iscode from .pointers import parent, reference, at, parents, children -from ._dill import _trace as trace from ._dill import PY3 +from .logger import trace __all__ = ['baditems','badobjects','badtypes','code','errors','freevars', 'getmodule','globalvars','nestedcode','nestedglobals','outermost', diff --git a/dill/logger.py b/dill/logger.py new file mode 100644 index 00000000..7fa2856c --- /dev/null +++ b/dill/logger.py @@ -0,0 +1,284 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Author: Leonardo Gama (@leogama) +# Copyright (c) 2022 The Uncertainty Quantification Foundation. +# License: 3-clause BSD. The full license text is available at: +# - https://github.com/uqfoundation/dill/blob/master/LICENSE +""" +Logging utilities for dill. + +The 'logger' object is dill's top-level logger. + +The 'adapter' object wraps the logger and implements a 'trace()' method that +generates a detailed tree-style trace for the pickling call at log level INFO. + +The 'trace()' function sets and resets dill's logger log level, enabling and +disabling the pickling trace. + +The trace shows a tree structure depicting the depth of each object serialized +*with dill save functions*, but not the ones that use save functions from +'pickle._Pickler.dispatch'. If the information is available, it also displays +the size in bytes that the object contributed to the pickle stream (including +its child objects). Sample trace output: + + >>> import dill, dill.tests + >>> dill.detect.trace(True) + >>> dill.dump_session(main=dill.tests) + ┬ M1: + ├┬ F2: + │└ # F2 [32 B] + ├┬ D2: + │├┬ T4: + ││└ # T4 [35 B] + │├┬ D2: + ││├┬ T4: + │││└ # T4 [50 B] + ││├┬ D2: + │││└ # D2 [84 B] + ││└ # D2 [413 B] + │└ # D2 [763 B] + └ # M1 [813 B] +""" + +__all__ = ['adapter', 'logger', 'trace'] + +import codecs +import contextlib +import locale +import logging +import math +import os +from functools import partial +from typing import NoReturn, TextIO, Union + +import dill + +# Tree drawing characters: Unicode to ASCII map. +ASCII_MAP = str.maketrans({"│": "|", "├": "|", "┬": "+", "└": "`"}) + +## Notes about the design choices ## + +# Here is some domumentation of the Standard Library's logging internals that +# can't be found completely in the official documentation. dill's logger is +# obtained by calling logging.getLogger('dill') and therefore is an instance of +# logging.getLoggerClass() at the call time. As this is controlled by the user, +# in order to add some functionality to it it's necessary to use a LoggerAdapter +# to wrap it, overriding some of the adapter's methods and creating new ones. +# +# Basic calling sequence +# ====================== +# +# Python's logging functionality can be conceptually divided into five steps: +# 0. Check logging level -> abort if call level is greater than logger level +# 1. Gather information -> construct a LogRecord from passed arguments and context +# 2. Filter (optional) -> discard message if the record matches a filter +# 3. Format -> format message with args, then format output string with message plus record +# 4. Handle -> write the formatted string to output as defined in the handler +# +# dill.logging.logger.log -> # or logger.info, etc. +# Logger.log -> \ +# Logger._log -> }- accept 'extra' parameter for custom record entries +# Logger.makeRecord -> / +# LogRecord.__init__ +# Logger.handle -> +# Logger.callHandlers -> +# Handler.handle -> +# Filterer.filter -> +# Filter.filter +# StreamHandler.emit -> +# Handler.format -> +# Formatter.format -> +# LogRecord.getMessage # does: record.message = msg % args +# Formatter.formatMessage -> +# PercentStyle.format # does: self._fmt % vars(record) +# +# NOTE: All methods from the second line on are from logging.__init__.py + +class TraceAdapter(logging.LoggerAdapter): + """ + Tracks object tree depth and calculates pickled object size. + + A single instance of this wraps the module's logger, as the logging API + doesn't allow setting it directly with a custom Logger subclass. The added + 'trace()' method receives a pickle instance as the first argument and + creates extra values to be added in the LogRecord from it, then calls + 'info()'. + + Usage of logger with 'trace()' method: + + >>> from .logger import adapter as logger # instead of 'from .logger import logger' + >>> ... + >>> def save_atype(pickler, obj): + >>> logger.trace(pickler, "Message with %s and %r etc. placeholders", 'text', obj) + >>> ... + """ + def __init__(self, logger): + self.logger = logger + def addHandler(self, handler): + formatter = TraceFormatter("%(prefix)s%(message)s%(suffix)s", handler=handler) + handler.setFormatter(formatter) + self.logger.addHandler(handler) + def removeHandler(self, handler): + self.logger.removeHandler(handler) + def process(self, msg, kwargs): + # A no-op override, as we don't have self.extra. + return msg, kwargs + def trace_setup(self, pickler): + # Called by Pickler.dump(). + if not dill._dill.is_dill(pickler, child=False): + return + if self.isEnabledFor(logging.INFO): + pickler._trace_depth = 1 + pickler._size_stack = [] + else: + pickler._trace_depth = None + def trace(self, pickler, msg, *args, **kwargs): + if not hasattr(pickler, '_trace_depth'): + logger.info(msg, *args, **kwargs) + return + if pickler._trace_depth is None: + return + extra = kwargs.get('extra', {}) + pushed_obj = msg.startswith('#') + size = None + try: + # Streams are not required to be tellable. + size = pickler._file.tell() + frame = pickler.framer.current_frame + try: + size += frame.tell() + except AttributeError: + # PyPy may use a BytesBuilder as frame + size += len(frame) + except (AttributeError, TypeError): + pass + if size is not None: + if not pushed_obj: + pickler._size_stack.append(size) + else: + size -= pickler._size_stack.pop() + extra['size'] = size + if pushed_obj: + pickler._trace_depth -= 1 + extra['depth'] = pickler._trace_depth + kwargs['extra'] = extra + self.info(msg, *args, **kwargs) + if not pushed_obj: + pickler._trace_depth += 1 + +class TraceFormatter(logging.Formatter): + """ + Generates message prefix and suffix from record. + + This Formatter adds prefix and suffix strings to the log message in trace + mode (an also provides empty string defaults for normal logs). + """ + def __init__(self, *args, handler=None, **kwargs): + super().__init__(*args, **kwargs) + try: + encoding = handler.stream.encoding + if encoding is None: + raise AttributeError + except AttributeError: + encoding = locale.getpreferredencoding() + try: + encoding = codecs.lookup(encoding).name + except LookupError: + self.is_utf8 = False + else: + self.is_utf8 = (encoding == codecs.lookup('utf-8').name) + def format(self, record): + fields = {'prefix': "", 'suffix': ""} + if getattr(record, 'depth', 0) > 0: + if record.msg.startswith("#"): + prefix = (record.depth - 1)*"│" + "└" + elif record.depth == 1: + prefix = "┬" + else: + prefix = (record.depth - 2)*"│" + "├┬" + if not self.is_utf8: + prefix = prefix.translate(ASCII_MAP) + "-" + fields['prefix'] = prefix + " " + if hasattr(record, 'size'): + # Show object size in human-redable form. + power = int(math.log(record.size, 2)) // 10 + size = record.size >> power*10 + fields['suffix'] = " [%d %sB]" % (size, "KMGTP"[power] + "i" if power else "") + vars(record).update(fields) + return super().format(record) + +logger = logging.getLogger('dill') +adapter = TraceAdapter(logger) +stderr_handler = logging.StreamHandler() +adapter.addHandler(stderr_handler) + +def trace(arg: Union[bool, TextIO, str, os.PathLike] = None, *, mode: str = 'a') -> NoReturn: + """print a trace through the stack when pickling; useful for debugging + + With a single boolean argument, enable or disable the tracing. + + Example usage: + + >>> import dill + >>> dill.detect.trace(True) + >>> dill.dump_session() + + Alternatively, ``trace()`` can be used as a context manager. With no + arguments, it just takes care of restoring the tracing state on exit. + Either a file handle, or a file name and (optionally) a file mode may be + specitfied to redirect the tracing output in the ``with`` block context. A + log function is yielded by the manager so the user can write extra + information to the file. + + Example usage: + + >>> from dill import detect + >>> D = {'a': 42, 'b': {'x': None}} + >>> with detect.trace(): + >>> dumps(D) + ┬ D2: + ├┬ D2: + │└ # D2 [8 B] + └ # D2 [22 B] + >>> squared = lambda x: x**2 + >>> with detect.trace('output.txt', mode='w') as log: + >>> log("> D = %r", D) + >>> dumps(D) + >>> log("> squared = %r", squared) + >>> dumps(squared) + + Arguments: + arg: a boolean value, or an optional file-like or path-like object for the context manager + mode: mode string for ``open()`` if a file name is passed as the first argument + """ + if not isinstance(arg, bool): + return TraceManager(file=arg, mode=mode) + logger.setLevel(logging.INFO if arg else logging.WARNING) + +class TraceManager(contextlib.AbstractContextManager): + """context manager version of trace(); can redirect the trace to a file""" + def __init__(self, file, mode): + self.file = file + self.mode = mode + self.redirect = file is not None + self.file_is_stream = hasattr(file, 'write') + def __enter__(self): + if self.redirect: + stderr_handler.flush() + if self.file_is_stream: + self.handler = logging.StreamHandler(self.file) + else: + self.handler = logging.FileHandler(self.file, self.mode) + adapter.removeHandler(stderr_handler) + adapter.addHandler(self.handler) + self.old_level = adapter.getEffectiveLevel() + adapter.setLevel(logging.INFO) + return adapter.info + def __exit__(self, *exc_info): + adapter.setLevel(self.old_level) + if self.redirect: + adapter.removeHandler(self.handler) + adapter.addHandler(stderr_handler) + if not self.file_is_stream: + self.handler.close() diff --git a/tests/test_logger.py b/tests/test_logger.py new file mode 100644 index 00000000..b4e4881a --- /dev/null +++ b/tests/test_logger.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python + +# Author: Leonardo Gama (@leogama) +# Copyright (c) 2022 The Uncertainty Quantification Foundation. +# License: 3-clause BSD. The full license text is available at: +# - https://github.com/uqfoundation/dill/blob/master/LICENSE + +import logging +import re +import tempfile + +import dill +from dill import detect +from dill.logger import stderr_handler, adapter as logger + +try: + from StringIO import StringIO +except ImportError: + from io import StringIO + +test_obj = {'a': (1, 2), 'b': object(), 'f': lambda x: x**2, 'big': list(range(10))} + +def test_logging(should_trace): + buffer = StringIO() + handler = logging.StreamHandler(buffer) + logger.addHandler(handler) + try: + dill.dumps(test_obj) + if should_trace: + regex = re.compile(r'(\S*┬ \w.*[^)]' # begin pickling object + r'|│*└ # \w.* \[\d+ (\wi)?B])' # object written (with size) + ) + for line in buffer.getvalue().splitlines(): + assert regex.fullmatch(line) + return buffer.getvalue() + else: + assert buffer.getvalue() == "" + finally: + logger.removeHandler(handler) + buffer.close() + +def test_trace_to_file(stream_trace): + file = tempfile.NamedTemporaryFile(mode='r') + with detect.trace(file.name, mode='w'): + dill.dumps(test_obj) + file_trace = file.read() + file.close() + # Apparently, objects can change location in memory... + reghex = re.compile(r'0x[0-9A-Za-z]+') + file_trace, stream_trace = reghex.sub('0x', file_trace), reghex.sub('0x', stream_trace) + # PyPy prints dictionary contents with repr(dict)... + regdict = re.compile(r'(dict\.__repr__ of ).*') + file_trace, stream_trace = regdict.sub(r'\1{}>', file_trace), regdict.sub(r'\1{}>', stream_trace) + assert file_trace == stream_trace + +if __name__ == '__main__': + logger.removeHandler(stderr_handler) + test_logging(should_trace=False) + detect.trace(True) + test_logging(should_trace=True) + detect.trace(False) + test_logging(should_trace=False) + + loglevel = logging.ERROR + logger.setLevel(loglevel) + with detect.trace(): + stream_trace = test_logging(should_trace=True) + test_logging(should_trace=False) + assert logger.getEffectiveLevel() == loglevel + test_trace_to_file(stream_trace) From 4220929eca4dc0942f823f8dcde892e3c4309dbb Mon Sep 17 00:00:00 2001 From: Leonardo Gama Date: Fri, 1 Jul 2022 20:06:00 -0300 Subject: [PATCH 12/13] Kickstart support drop for Python < 3.7 (#499) * Kickstart support drop for Python < 3.7 Step 1 of code clean-up: mechanically remove if-else branches that would never run in Python/PyPy >= 3.7. There's still some unused or redundant code, left for the next step to facilitate review. Note: all the remaining `sys.hexversion` tests were standardized to use numerical comparison and hexadecimal integer literals (e.g. `0x30b00a7` for version 3.11.0a7) as most of them already were. * convert single function defined via exec() due to Python2/3 differences * drop 'from __future__ import print_function' statements * remove conditional imports due to Python 2 and old Python 3 * substitute bare 'except' clauses by 'except Exception' or more specific where it's obvious * split module imports into separated lines * idem, but for test files * review: adjustments * tests: get a default Python 3 executable * update module docs * update README * review: corrections * remove unicode prefix from strings in docs/source/conf.py --- README.md | 19 +- dill/__diff.py | 12 +- dill/__init__.py | 40 +-- dill/_dill.py | 663 +++++++++++-------------------------- dill/_objects.py | 279 ++++++---------- dill/_shims.py | 103 +----- dill/detect.py | 109 ++---- dill/logger.py | 4 +- dill/settings.py | 5 +- dill/source.py | 62 +--- dill/temp.py | 23 +- docs/source/conf.py | 14 +- tests/__main__.py | 2 +- tests/test_check.py | 1 - tests/test_classdef.py | 77 ++--- tests/test_detect.py | 6 +- tests/test_diff.py | 21 +- tests/test_extendpickle.py | 7 +- tests/test_file.py | 2 - tests/test_functions.py | 39 +-- tests/test_module.py | 3 +- tests/test_nested.py | 2 +- tests/test_recursive.py | 10 +- tests/test_selected.py | 6 +- tests/test_session.py | 11 +- tests/test_source.py | 21 +- tests/test_weakref.py | 2 +- 27 files changed, 461 insertions(+), 1082 deletions(-) diff --git a/README.md b/README.md index d94e1b4a..de42db1b 100644 --- a/README.md +++ b/README.md @@ -28,26 +28,27 @@ a trustworthy source. ``dill`` is part of ``pathos``, a python framework for heterogeneous computing. ``dill`` is in active development, so any user feedback, bug reports, comments, -or suggestions are highly appreciated. A list of issues is located at https://github.com/uqfoundation/dill/issues, with a legacy list maintained at https://uqfoundation.github.io/project/pathos/query. +or suggestions are highly appreciated. A list of issues is located at +https://github.com/uqfoundation/dill/issues, with a legacy list maintained at +https://uqfoundation.github.io/project/pathos/query. Major Features -------------- ``dill`` can pickle the following standard types: -* none, type, bool, int, long, float, complex, str, unicode, +* none, type, bool, int, float, complex, bytes, str, * tuple, list, dict, file, buffer, builtin, -* both old and new style classes, -* instances of old and new style classes, +* python classes, namedtuples, dataclasses, metaclasses, +* instances of classes, * set, frozenset, array, functions, exceptions ``dill`` can also pickle more 'exotic' standard types: -* functions with yields, nested functions, lambdas +* functions with yields, nested functions, lambdas, * cell, method, unboundmethod, module, code, methodwrapper, -* dictproxy, methoddescriptor, getsetdescriptor, memberdescriptor, -* wrapperdescriptor, xrange, slice, -* notimplemented, ellipsis, quit +* methoddescriptor, getsetdescriptor, memberdescriptor, wrapperdescriptor, +* dictproxy, slice, notimplemented, ellipsis, quit ``dill`` cannot yet pickle these standard types: @@ -133,7 +134,7 @@ There are a number of options to control serialization which are provided as keyword arguments to several ``dill`` functions: * with *protocol*, the pickle protocol level can be set. This uses the - same value as the ``pickle`` module, *HIGHEST_PROTOCOL* or *DEFAULT_PROTOCOL*. + same value as the ``pickle`` module, *DEFAULT_PROTOCOL*. * with *byref=True*, ``dill`` to behave a lot more like pickle with certain objects (like modules) pickled by reference as opposed to attempting to pickle the object itself. diff --git a/dill/__diff.py b/dill/__diff.py index 3ff65763..60268a6d 100644 --- a/dill/__diff.py +++ b/dill/__diff.py @@ -10,18 +10,15 @@ Module to show if an object has changed since it was memorised """ +import builtins import os import sys import types try: import numpy HAS_NUMPY = True -except: - HAS_NUMPY = False -try: - import builtins except ImportError: - import __builtin__ as builtins + HAS_NUMPY = False # pypy doesn't use reference counting getrefcount = getattr(sys, 'getrefcount', lambda x:0) @@ -44,10 +41,7 @@ def get_attrs(obj): if type(obj) in builtins_types \ or type(obj) is type and obj in builtins_types: return - try: - return obj.__dict__ - except: - return + return getattr(obj, '__dict__', None) def get_seq(obj, cache={str: False, frozenset: False, list: True, set: True, diff --git a/dill/__init__.py b/dill/__init__.py index 39ac4df2..80281811 100644 --- a/dill/__init__.py +++ b/dill/__init__.py @@ -26,7 +26,7 @@ ``dill`` provides the user the same interface as the ``pickle`` module, and also includes some additional features. In addition to pickling python objects, ``dill`` provides the ability to save the state of an interpreter -session in a single command. Hence, it would be feasable to save an +session in a single command. Hence, it would be feasible to save an interpreter session, close the interpreter, ship the pickled file to another computer, open a new interpreter, unpickle the session and thus continue from the 'saved' state of the original interpreter @@ -42,7 +42,9 @@ ``dill`` is part of ``pathos``, a python framework for heterogeneous computing. ``dill`` is in active development, so any user feedback, bug reports, comments, -or suggestions are highly appreciated. A list of issues is located at https://github.com/uqfoundation/dill/issues, with a legacy list maintained at https://uqfoundation.github.io/project/pathos/query. +or suggestions are highly appreciated. A list of issues is located at +https://github.com/uqfoundation/dill/issues, with a legacy list maintained at +https://uqfoundation.github.io/project/pathos/query. Major Features @@ -50,19 +52,18 @@ ``dill`` can pickle the following standard types: - - none, type, bool, int, long, float, complex, str, unicode, + - none, type, bool, int, float, complex, bytes, str, - tuple, list, dict, file, buffer, builtin, - - both old and new style classes, - - instances of old and new style classes, + - python classes, namedtuples, dataclasses, metaclasses, + - instances of classes, - set, frozenset, array, functions, exceptions ``dill`` can also pickle more 'exotic' standard types: - functions with yields, nested functions, lambdas, - cell, method, unboundmethod, module, code, methodwrapper, - - dictproxy, methoddescriptor, getsetdescriptor, memberdescriptor, - - wrapperdescriptor, xrange, slice, - - notimplemented, ellipsis, quit + - methoddescriptor, getsetdescriptor, memberdescriptor, wrapperdescriptor, + - dictproxy, slice, notimplemented, ellipsis, quit ``dill`` cannot yet pickle these standard types: @@ -148,7 +149,7 @@ as keyword arguments to several ``dill`` functions: * with *protocol*, the pickle protocol level can be set. This uses the - same value as the ``pickle`` module, *HIGHEST_PROTOCOL* or *DEFAULT_PROTOCOL*. + same value as the ``pickle`` module, *DEFAULT_PROTOCOL*. * with *byref=True*, ``dill`` to behave a lot more like pickle with certain objects (like modules) pickled by reference as opposed to attempting to pickle the object itself. @@ -300,23 +301,9 @@ # make sure "trace" is turned off detect.trace(False) -try: - from importlib import reload -except ImportError: - try: - from imp import reload - except ImportError: - pass - -# put the objects in order, if possible -try: - from collections import OrderedDict as odict -except ImportError: - try: - from ordereddict import OrderedDict as odict - except ImportError: - odict = dict -objects = odict() +from importlib import reload + +objects = {} # local import of dill._objects #from . import _objects #objects.update(_objects.succeeds) @@ -377,7 +364,6 @@ def extend(use_dill=True): return extend() -del odict def license(): diff --git a/dill/_dill.py b/dill/_dill.py index 7660cb07..c718c4f2 100644 --- a/dill/_dill.py +++ b/dill/_dill.py @@ -31,52 +31,27 @@ import sys diff = None _use_diff = False -PY3 = (sys.hexversion >= 0x3000000) -# OLDER: 3.0 <= x < 3.4 *OR* x < 2.7.10 #NOTE: guessing relevant versions -OLDER = (PY3 and sys.hexversion < 0x3040000) or (sys.hexversion < 0x2070ab1) -OLD33 = (sys.hexversion < 0x3030000) -OLD37 = (sys.hexversion < 0x3070000) OLD38 = (sys.hexversion < 0x3080000) OLD39 = (sys.hexversion < 0x3090000) OLD310 = (sys.hexversion < 0x30a0000) -PY34 = (0x3040000 <= sys.hexversion < 0x3050000) -if PY3: #XXX: get types from .objtypes ? - import builtins as __builtin__ - from pickle import _Pickler as StockPickler, Unpickler as StockUnpickler - from _thread import LockType - if (sys.hexversion >= 0x30200f0): - from _thread import RLock as RLockType - else: - from threading import _RLock as RLockType - #from io import IOBase - from types import CodeType, FunctionType, MethodType, GeneratorType, \ - TracebackType, FrameType, ModuleType, BuiltinMethodType - BufferType = memoryview #XXX: unregistered - ClassType = type # no 'old-style' classes - EllipsisType = type(Ellipsis) - #FileType = IOBase - NotImplementedType = type(NotImplemented) - SliceType = slice - TypeType = type # 'new-style' classes #XXX: unregistered - XRangeType = range - if OLD33: - DictProxyType = type(object.__dict__) - else: - from types import MappingProxyType as DictProxyType -else: - import __builtin__ - from pickle import Pickler as StockPickler, Unpickler as StockUnpickler - from thread import LockType - from threading import _RLock as RLockType - from types import CodeType, FunctionType, ClassType, MethodType, \ - GeneratorType, DictProxyType, XRangeType, SliceType, TracebackType, \ - NotImplementedType, EllipsisType, FrameType, ModuleType, \ - BufferType, BuiltinMethodType, TypeType -from pickle import HIGHEST_PROTOCOL, PickleError, PicklingError, UnpicklingError -try: - from pickle import DEFAULT_PROTOCOL -except ImportError: - DEFAULT_PROTOCOL = HIGHEST_PROTOCOL +#XXX: get types from .objtypes ? +import builtins as __builtin__ +from pickle import _Pickler as StockPickler, Unpickler as StockUnpickler +from _thread import LockType +from _thread import RLock as RLockType +#from io import IOBase +from types import CodeType, FunctionType, MethodType, GeneratorType, \ + TracebackType, FrameType, ModuleType, BuiltinMethodType +BufferType = memoryview #XXX: unregistered +ClassType = type # no 'old-style' classes +EllipsisType = type(Ellipsis) +#FileType = IOBase +NotImplementedType = type(NotImplemented) +SliceType = slice +TypeType = type # 'new-style' classes #XXX: unregistered +XRangeType = range +from types import MappingProxyType as DictProxyType +from pickle import DEFAULT_PROTOCOL, HIGHEST_PROTOCOL, PickleError, PicklingError, UnpicklingError import __main__ as _main_module import marshal import gc @@ -85,20 +60,9 @@ from collections import OrderedDict from functools import partial from operator import itemgetter, attrgetter -# new in python3.3 -if sys.hexversion < 0x03030000: - FileNotFoundError = IOError -if PY3 and sys.hexversion < 0x03040000: - GENERATOR_FAIL = True -else: GENERATOR_FAIL = False -if PY3: - import importlib.machinery - EXTENSION_SUFFIXES = tuple(importlib.machinery.EXTENSION_SUFFIXES) -else: - import imp - EXTENSION_SUFFIXES = tuple(suffix - for (suffix, _, s_type) in imp.get_suffixes() - if s_type == imp.C_EXTENSION) +GENERATOR_FAIL = False +import importlib.machinery +EXTENSION_SUFFIXES = tuple(importlib.machinery.EXTENSION_SUFFIXES) try: import ctypes HAS_CTYPES = True @@ -107,28 +71,15 @@ except ImportError: HAS_CTYPES = False IS_PYPY = False -IS_PYPY2 = IS_PYPY and not PY3 NumpyUfuncType = None NumpyDType = None NumpyArrayType = None try: - if OLDER: - raise AttributeError('find_spec not found') - import importlib if not importlib.machinery.PathFinder().find_spec('numpy'): raise ImportError("No module named 'numpy'") NumpyUfuncType = True NumpyDType = True NumpyArrayType = True -except AttributeError: - try: - import imp - imp.find_module('numpy') - NumpyUfuncType = True - NumpyDType = True - NumpyArrayType = True - except ImportError: - pass except ImportError: pass def __hook__(): @@ -193,10 +144,7 @@ def numpydtype(obj): return False MethodWrapperType #XXX: unused # make sure to add these 'hand-built' types to _typemap -if PY3: - CellType = type((lambda x: lambda y: x)(0).__closure__[0]) -else: - CellType = type((lambda x: lambda y: x)(0).func_closure[0]) +CellType = type((lambda x: lambda y: x)(0).__closure__[0]) PartialType = type(partial(int, base=2)) SuperType = type(super(Exception, TypeError())) ItemGetterType = type(itemgetter(0)) @@ -204,7 +152,7 @@ def numpydtype(obj): return False try: from functools import _lru_cache_wrapper as LRUCacheType -except: +except ImportError: LRUCacheType = None if not isinstance(LRUCacheType, type): @@ -230,20 +178,11 @@ def get_file_type(*args, **kwargs): PyBufferedWriterType = get_file_type('wb', buffering=-1, open=_open) except ImportError: PyTextWrapperType = PyBufferedRandomType = PyBufferedReaderType = PyBufferedWriterType = None -try: - from cStringIO import StringIO, InputType, OutputType -except ImportError: - if PY3: - from io import BytesIO as StringIO - else: - from StringIO import StringIO - InputType = OutputType = None -if not IS_PYPY2: - from socket import socket as SocketType - try: #FIXME: additionally calls ForkingPickler.register several times - from multiprocessing.reduction import _reduce_socket as reduce_socket - except ImportError: - from multiprocessing.reduction import reduce_socket +from io import BytesIO as StringIO +InputType = OutputType = None +from socket import socket as SocketType +#FIXME: additionally calls ForkingPickler.register several times +from multiprocessing.reduction import _reduce_socket as reduce_socket try: __IPYTHON__ is True # is ipython ExitType = None # IPython.core.autocall.ExitAutocall @@ -384,8 +323,7 @@ def _module_map(): from collections import defaultdict, namedtuple modmap = namedtuple('Modmap', ['by_name', 'by_id', 'top_level']) modmap = modmap(defaultdict(list), defaultdict(list), {}) - items = 'items' if PY3 else 'iteritems' - for modname, module in getattr(sys.modules, items)(): + for modname, module in sys.modules.items(): if not isinstance(module, ModuleType): continue if '.' not in modname: @@ -414,8 +352,7 @@ def _stash_modules(main_module): imported_as = [] imported_top_level = [] # keep separeted for backwards compatibility original = {} - items = 'items' if PY3 else 'iteritems' - for name, obj in getattr(main_module.__dict__, items)(): + for name, obj in main_module.__dict__.items(): if obj is main_module: original[name] = newmod # self-reference continue @@ -674,22 +611,17 @@ def use_diff(on=True): if _use_diff and diff is None: try: from . import diff as d - except: + except ImportError: import diff as d diff = d def _create_typemap(): import types - if PY3: - d = dict(list(__builtin__.__dict__.items()) + \ - list(types.__dict__.items())).items() - builtin = 'builtins' - else: - d = types.__dict__.iteritems() - builtin = '__builtin__' + d = dict(list(__builtin__.__dict__.items()) + \ + list(types.__dict__.items())).items() for key, value in d: - if getattr(value, '__module__', None) == builtin \ - and type(value) is type: + if getattr(value, '__module__', None) == 'builtins' \ + and type(value) is type: yield key, value return _reverse_typemap = dict(_create_typemap()) @@ -719,22 +651,15 @@ def _create_typemap(): 'PyTextWrapperType': PyTextWrapperType, } -if PY3: - _incedental_reverse_typemap.update({ - "DictKeysType": type({}.keys()), - "DictValuesType": type({}.values()), - "DictItemsType": type({}.items()), +_incedental_reverse_typemap.update({ + "DictKeysType": type({}.keys()), + "DictValuesType": type({}.values()), + "DictItemsType": type({}.items()), - "OdictKeysType": type(x.keys()), - "OdictValuesType": type(x.values()), - "OdictItemsType": type(x.items()), - }) -else: - _incedental_reverse_typemap.update({ - "DictKeysType": type({}.viewkeys()), - "DictValuesType": type({}.viewvalues()), - "DictItemsType": type({}.viewitems()), - }) + "OdictKeysType": type(x.keys()), + "OdictValuesType": type(x.values()), + "OdictItemsType": type(x.items()), +}) if ExitType: _incedental_reverse_typemap['ExitType'] = ExitType @@ -761,7 +686,7 @@ def _create_typemap(): try: import winreg _incedental_reverse_typemap["HKEYType"] = winreg.HKEYType -except: +except ImportError: pass _reverse_typemap.update(_incedental_reverse_typemap) @@ -769,10 +694,7 @@ def _create_typemap(): del x -if PY3: - _typemap = dict((v, k) for k, v in _reverse_typemap.items()) -else: - _typemap = dict((v, k) for k, v in _reverse_typemap.iteritems()) +_typemap = dict((v, k) for k, v in _reverse_typemap.items()) def _unmarshal(string): return marshal.loads(string) @@ -805,7 +727,7 @@ def _create_code(*args): args = args[1:] else: # from < 3.10 (or pre-LNOTAB storage) LNOTAB = b'' - if PY3 and hasattr(args[-3], 'encode'): #NOTE: from PY2 fails (optcode) + if hasattr(args[-3], 'encode'): #NOTE: from PY2 fails (optcode) args = list(args) if len(args) == 20: # from 3.11a # obj.co_argcount, obj.co_posonlyargcount, @@ -933,7 +855,7 @@ def _create_code(*args): return CodeType(*args) elif len(args) == 15: return CodeType(args[0], 0, *args[1:]) # from 3.7 return CodeType(args[0], 0, 0, *args[1:]) # from 2.7 - elif hasattr(CodeType, 'co_kwonlyargcount'): # python 3.7 + else: # python 3.7 # obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals, # obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts, # obj.co_names, obj.co_varnames, obj.co_filename, @@ -953,24 +875,6 @@ def _create_code(*args): return CodeType(args[0], *argz) elif len(args) == 15: return CodeType(*args) return CodeType(args[0], 0, *args[1:]) # from 2.7 - # obj.co_argcount, obj.co_nlocals, obj.co_stacksize, obj.co_flags, - # obj.co_code, obj.co_consts, obj.co_names, obj.co_varnames, - # obj.co_filename, obj.co_name, obj.co_firstlineno, obj.co_lnotab, - # obj.co_freevars, obj.co_cellvars - if len(args) == 20: # from 3.11a - args = args[:1] + args[3:12] + args[13:14] + (LNOTAB,) + args[18:] - return CodeType(*args) - elif len(args) == 18: # from 3.11 - args = args[:1] + args[3:12] + args[13:14] + (LNOTAB,) + args[16:] - return CodeType(*args) - elif len(args) == 16: # from 3.10 or from 3.8 - if LNOTAB: # here and above uses stored LNOTAB - argz = args[3:-3] + (LNOTAB,) + args[-2:] - else: - argz = args[3:] - return CodeType(args[0], *argz) - elif len(args) == 15: return CodeType(args[0], *args[2:]) # from 3.7 - return CodeType(*args) def _create_ftype(ftypeobj, func, args, kwds): if kwds is None: @@ -1010,12 +914,9 @@ def _create_filehandle(name, mode, position, closed, open, strictio, fmode, fdat import tempfile f = tempfile.TemporaryFile(mode) else: - # treat x mode as w mode - if "x" in mode and sys.hexversion < 0x03030000: - raise ValueError("invalid mode: '%s'" % mode) try: exists = os.path.exists(name) - except: + except Exception: exists = False if not exists: if strictio: @@ -1044,6 +945,7 @@ def _create_filehandle(name, mode, position, closed, open, strictio, fmode, fdat elif name == '': # file did not exist import tempfile f = tempfile.TemporaryFile(mode) + # treat x mode as w mode elif fmode == CONTENTS_FMODE \ and ("w" in mode or "x" in mode): # stop truncation when opening @@ -1054,28 +956,9 @@ def _create_filehandle(name, mode, position, closed, open, strictio, fmode, fdat flags |= os.O_WRONLY f = os.fdopen(os.open(name, flags), mode) # set name to the correct value - if PY3: - r = getattr(f, "buffer", f) - r = getattr(r, "raw", r) - r.name = name - else: - if not HAS_CTYPES: - raise ImportError("No module named 'ctypes'") - class FILE(ctypes.Structure): - _fields_ = [("refcount", ctypes.c_long), - ("type_obj", ctypes.py_object), - ("file_pointer", ctypes.c_voidp), - ("name", ctypes.py_object)] - - class PyObject(ctypes.Structure): - _fields_ = [ - ("ob_refcnt", ctypes.c_int), - ("ob_type", ctypes.py_object) - ] - #FIXME: CONTENTS_FMODE fails for pypy due to issue #1233 - # https://bitbucket.org/pypy/pypy/issues/1233 - ctypes.cast(id(f), ctypes.POINTER(FILE)).contents.name = name - ctypes.cast(id(name), ctypes.POINTER(PyObject)).contents.ob_refcnt += 1 + r = getattr(f, "buffer", f) + r = getattr(r, "raw", r) + r.name = name assert f.name == name else: f = open(name, mode) @@ -1136,7 +1019,7 @@ def __ror__(self, a): # mapping referenced by the proxy. It may work for other implementations, # but is not guaranteed. MAPPING_PROXY_TRICK = __d is (DictProxyType(__d) | _dictproxy_helper_instance) -except: +except Exception: MAPPING_PROXY_TRICK = False del __d @@ -1147,26 +1030,15 @@ def __ror__(self, a): _CELL_REF = None _CELL_EMPTY = Sentinel('_CELL_EMPTY') -if PY3: - def _create_cell(contents=None): - if contents is not _CELL_EMPTY: - value = contents - return (lambda: value).__closure__[0] - -else: - def _create_cell(contents=None): - if contents is not _CELL_EMPTY: - value = contents - return (lambda: value).func_closure[0] - +def _create_cell(contents=None): + if contents is not _CELL_EMPTY: + value = contents + return (lambda: value).__closure__[0] def _create_weakref(obj, *args): from weakref import ref if obj is None: # it's dead - if PY3: - from collections import UserDict - else: - from UserDict import UserDict + from collections import UserDict return ref(UserDict(), *args) return ref(obj, *args) @@ -1174,10 +1046,7 @@ def _create_weakproxy(obj, callable=False, *args): from weakref import proxy if obj is None: # it's dead if callable: return proxy(lambda x:x, *args) - if PY3: - from collections import UserDict - else: - from UserDict import UserDict + from collections import UserDict return proxy(UserDict(), *args) return proxy(obj, *args) @@ -1198,37 +1067,24 @@ def _create_dtypemeta(scalar_type): return NumpyDType return type(NumpyDType(scalar_type)) -if OLD37: - def _create_namedtuple(name, fieldnames, modulename, defaults=None): - class_ = _import_module(modulename + '.' + name, safe=True) - if class_ is not None: - return class_ - import collections - t = collections.namedtuple(name, fieldnames) - t.__module__ = modulename - return t -else: - def _create_namedtuple(name, fieldnames, modulename, defaults=None): - class_ = _import_module(modulename + '.' + name, safe=True) - if class_ is not None: - return class_ - import collections - t = collections.namedtuple(name, fieldnames, defaults=defaults, module=modulename) - return t +def _create_namedtuple(name, fieldnames, modulename, defaults=None): + class_ = _import_module(modulename + '.' + name, safe=True) + if class_ is not None: + return class_ + import collections + t = collections.namedtuple(name, fieldnames, defaults=defaults, module=modulename) + return t def _create_capsule(pointer, name, context, destructor): attr_found = False try: # based on https://github.com/python/cpython/blob/f4095e53ab708d95e019c909d5928502775ba68f/Objects/capsule.c#L209-L231 - if PY3: - uname = name.decode('utf8') - else: - uname = name + uname = name.decode('utf8') for i in range(1, uname.count('.')+1): names = uname.rsplit('.', i) try: module = __import__(names[0]) - except: + except ImportError: pass obj = module for attr in names[1:]: @@ -1236,7 +1092,7 @@ def _create_capsule(pointer, name, context, destructor): capsule = obj attr_found = True break - except: + except Exception: pass if attr_found: @@ -1254,14 +1110,14 @@ def _getattr(objclass, name, repr_str): try: #XXX: works only for __builtin__ ? attr = repr_str.split("'")[3] return eval(attr+'.__dict__["'+name+'"]') - except: + except Exception: try: attr = objclass.__dict__ if type(attr) is DictProxyType: attr = attr[name] else: attr = getattr(objclass,name) - except: + except (AttributeError, KeyError): attr = getattr(objclass,name) return attr @@ -1314,7 +1170,7 @@ def _locate_function(obj, pickler=None): try: found, _ = _getattribute(module, obj.__qualname__) return found is obj - except: + except AttributeError: return False else: found = _import_module(module_name + '.' + obj.__name__, safe=True) @@ -1370,10 +1226,7 @@ def _save_with_postproc(pickler, reduction, is_pickler_dill=None, obj=Getattr.NO else: pickler.save_reduce(*reduction) # pop None created by calling preprocessing step off stack - if PY3: - pickler.write(bytes('0', 'UTF-8')) - else: - pickler.write('0') + pickler.write(bytes('0', 'UTF-8')) #@register(CodeType) #def save_code(pickler, obj): @@ -1389,62 +1242,54 @@ def _save_with_postproc(pickler, reduction, is_pickler_dill=None, obj=Getattr.NO @register(CodeType) def save_code(pickler, obj): logger.trace(pickler, "Co: %s", obj) - if PY3: - if hasattr(obj, "co_endlinetable"): # python 3.11a (20 args) - args = ( - obj.co_lnotab, # for < python 3.10 [not counted in args] - obj.co_argcount, obj.co_posonlyargcount, - obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, - obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, - obj.co_varnames, obj.co_filename, obj.co_name, obj.co_qualname, - obj.co_firstlineno, obj.co_linetable, obj.co_endlinetable, - obj.co_columntable, obj.co_exceptiontable, obj.co_freevars, - obj.co_cellvars - ) - elif hasattr(obj, "co_exceptiontable"): # python 3.11 (18 args) - args = ( - obj.co_lnotab, # for < python 3.10 [not counted in args] - obj.co_argcount, obj.co_posonlyargcount, - obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, - obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, - obj.co_varnames, obj.co_filename, obj.co_name, obj.co_qualname, - obj.co_firstlineno, obj.co_linetable, obj.co_exceptiontable, - obj.co_freevars, obj.co_cellvars - ) - elif hasattr(obj, "co_linetable"): # python 3.10 (16 args) - args = ( - obj.co_lnotab, # for < python 3.10 [not counted in args] - obj.co_argcount, obj.co_posonlyargcount, - obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, - obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, - obj.co_varnames, obj.co_filename, obj.co_name, - obj.co_firstlineno, obj.co_linetable, obj.co_freevars, - obj.co_cellvars - ) - elif hasattr(obj, "co_posonlyargcount"): # python 3.8 (16 args) - args = ( - obj.co_argcount, obj.co_posonlyargcount, - obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, - obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, - obj.co_varnames, obj.co_filename, obj.co_name, - obj.co_firstlineno, obj.co_lnotab, obj.co_freevars, - obj.co_cellvars - ) - else: # python 3.7 (15 args) - args = ( - obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals, - obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts, - obj.co_names, obj.co_varnames, obj.co_filename, - obj.co_name, obj.co_firstlineno, obj.co_lnotab, - obj.co_freevars, obj.co_cellvars - ) - else: # python 2.7 (14 args) + if hasattr(obj, "co_endlinetable"): # python 3.11a (20 args) args = ( - obj.co_argcount, obj.co_nlocals, obj.co_stacksize, obj.co_flags, - obj.co_code, obj.co_consts, obj.co_names, obj.co_varnames, - obj.co_filename, obj.co_name, obj.co_firstlineno, obj.co_lnotab, + obj.co_lnotab, # for < python 3.10 [not counted in args] + obj.co_argcount, obj.co_posonlyargcount, + obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, + obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, + obj.co_varnames, obj.co_filename, obj.co_name, obj.co_qualname, + obj.co_firstlineno, obj.co_linetable, obj.co_endlinetable, + obj.co_columntable, obj.co_exceptiontable, obj.co_freevars, + obj.co_cellvars + ) + elif hasattr(obj, "co_exceptiontable"): # python 3.11 (18 args) + args = ( + obj.co_lnotab, # for < python 3.10 [not counted in args] + obj.co_argcount, obj.co_posonlyargcount, + obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, + obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, + obj.co_varnames, obj.co_filename, obj.co_name, obj.co_qualname, + obj.co_firstlineno, obj.co_linetable, obj.co_exceptiontable, obj.co_freevars, obj.co_cellvars - ) + ) + elif hasattr(obj, "co_linetable"): # python 3.10 (16 args) + args = ( + obj.co_lnotab, # for < python 3.10 [not counted in args] + obj.co_argcount, obj.co_posonlyargcount, + obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, + obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, + obj.co_varnames, obj.co_filename, obj.co_name, + obj.co_firstlineno, obj.co_linetable, obj.co_freevars, + obj.co_cellvars + ) + elif hasattr(obj, "co_posonlyargcount"): # python 3.8 (16 args) + args = ( + obj.co_argcount, obj.co_posonlyargcount, + obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, + obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, + obj.co_varnames, obj.co_filename, obj.co_name, + obj.co_firstlineno, obj.co_lnotab, obj.co_freevars, + obj.co_cellvars + ) + else: # python 3.7 (15 args) + args = ( + obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals, + obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts, + obj.co_names, obj.co_varnames, obj.co_filename, + obj.co_name, obj.co_firstlineno, obj.co_lnotab, + obj.co_freevars, obj.co_cellvars + ) pickler.save_reduce(_create_code, args, obj=obj) logger.trace(pickler, "# Co") @@ -1459,26 +1304,17 @@ def save_module_dict(pickler, obj): if is_dill(pickler, child=False) and obj == pickler._main.__dict__ and \ not (pickler._session and pickler._first_pass): logger.trace(pickler, "D1: %s", _repr_dict(obj)) # obj - if PY3: - pickler.write(bytes('c__builtin__\n__main__\n', 'UTF-8')) - else: - pickler.write('c__builtin__\n__main__\n') + pickler.write(bytes('c__builtin__\n__main__\n', 'UTF-8')) logger.trace(pickler, "# D1") elif (not is_dill(pickler, child=False)) and (obj == _main_module.__dict__): logger.trace(pickler, "D3: %s", _repr_dict(obj)) # obj - if PY3: - pickler.write(bytes('c__main__\n__dict__\n', 'UTF-8')) - else: - pickler.write('c__main__\n__dict__\n') #XXX: works in general? + pickler.write(bytes('c__main__\n__dict__\n', 'UTF-8')) #XXX: works in general? logger.trace(pickler, "# D3") elif '__name__' in obj and obj != _main_module.__dict__ \ - and type(obj['__name__']) is str \ - and obj is getattr(_import_module(obj['__name__'],True), '__dict__', None): + and type(obj['__name__']) is str \ + and obj is getattr(_import_module(obj['__name__'],True), '__dict__', None): logger.trace(pickler, "D4: %s", _repr_dict(obj)) # obj - if PY3: - pickler.write(bytes('c%s\n__dict__\n' % obj['__name__'], 'UTF-8')) - else: - pickler.write('c%s\n__dict__\n' % obj['__name__']) + pickler.write(bytes('c%s\n__dict__\n' % obj['__name__'], 'UTF-8')) logger.trace(pickler, "# D4") else: logger.trace(pickler, "D2: %s", _repr_dict(obj)) # obj @@ -1572,38 +1408,17 @@ def save_rlock(pickler, obj): logger.trace(pickler, "RL: %s", obj) r = obj.__repr__() # don't use _release_save as it unlocks the lock count = int(r.split('count=')[1].split()[0].rstrip('>')) - owner = int(r.split('owner=')[1].split()[0]) if PY3 else getattr(obj, '_RLock__owner') + owner = int(r.split('owner=')[1].split()[0]) pickler.save_reduce(_create_rlock, (count,owner,), obj=obj) logger.trace(pickler, "# RL") return -if not IS_PYPY2: - #@register(SocketType) #FIXME: causes multiprocess test_pickling FAIL - def save_socket(pickler, obj): - logger.trace(pickler, "So: %s", obj) - pickler.save_reduce(*reduce_socket(obj)) - logger.trace(pickler, "# So") - return - -if sys.hexversion <= 0x3050000: - @register(ItemGetterType) - def save_itemgetter(pickler, obj): - logger.trace(pickler, "Ig: %s", obj) - helper = _itemgetter_helper() - obj(helper) - pickler.save_reduce(type(obj), tuple(helper.items), obj=obj) - logger.trace(pickler, "# Ig") - return - - @register(AttrGetterType) - def save_attrgetter(pickler, obj): - logger.trace(pickler, "Ag: %s", obj) - attrs = [] - helper = _attrgetter_helper(attrs) - obj(helper) - pickler.save_reduce(type(obj), tuple(attrs), obj=obj) - logger.trace(pickler, "# Ag") - return +#@register(SocketType) #FIXME: causes multiprocess test_pickling FAIL +def save_socket(pickler, obj): + logger.trace(pickler, "So: %s", obj) + pickler.save_reduce(*reduce_socket(obj)) + logger.trace(pickler, "# So") + return def _save_file(pickler, obj, open_): if obj.closed: @@ -1683,15 +1498,6 @@ def save_stringo(pickler, obj): logger.trace(pickler, "# Io") return -if 0x2050000 <= sys.hexversion < 0x3010000: - @register(PartialType) - def save_functor(pickler, obj): - logger.trace(pickler, "Fu: %s", obj) - pickler.save_reduce(_create_ftype, (type(obj), obj.func, obj.args, - obj.keywords), obj=obj) - logger.trace(pickler, "# Fu") - return - if LRUCacheType is not None: from functools import lru_cache @register(LRUCacheType) @@ -1718,32 +1524,6 @@ def save_super(pickler, obj): logger.trace(pickler, "# Su") return -if OLDER or not PY3: - @register(BuiltinMethodType) - def save_builtin_method(pickler, obj): - if obj.__self__ is not None: - if obj.__self__ is __builtin__: - module = 'builtins' if PY3 else '__builtin__' - _t = "B1" - logger.trace(pickler, "%s: %s", _t, obj) - else: - module = obj.__self__ - _t = "B3" - logger.trace(pickler, "%s: %s", _t, obj) - if is_dill(pickler, child=True): - _recurse = pickler._recurse - pickler._recurse = False - pickler.save_reduce(_get_attr, (module, obj.__name__), obj=obj) - if is_dill(pickler, child=True): - pickler._recurse = _recurse - logger.trace(pickler, "# %s", _t) - else: - logger.trace(pickler, "B2: %s", obj) - name = getattr(obj, '__qualname__', getattr(obj, '__name__', None)) - StockPickler.save_global(pickler, obj, name=name) - logger.trace(pickler, "# B2") - return - if IS_PYPY: @register(MethodType) def save_instancemethod0(pickler, obj): @@ -1751,50 +1531,39 @@ def save_instancemethod0(pickler, obj): if code is not None and type(code) is not CodeType \ and getattr(obj.__self__, obj.__name__) == obj: # Some PyPy builtin functions have no module name - logger.trace(pickler, "Me2: %s" % obj) + logger.trace(pickler, "Me2: %s", obj) # TODO: verify that this works for all PyPy builtin methods pickler.save_reduce(getattr, (obj.__self__, obj.__name__), obj=obj) logger.trace(pickler, "# Me2") return - logger.trace(pickler, "Me1: %s" % obj) + logger.trace(pickler, "Me1: %s", obj) pickler.save_reduce(MethodType, (obj.__func__, obj.__self__), obj=obj) logger.trace(pickler, "# Me1") return else: @register(MethodType) def save_instancemethod0(pickler, obj): - logger.trace(pickler, "Me1: %s" % obj) + logger.trace(pickler, "Me1: %s", obj) pickler.save_reduce(MethodType, (obj.__func__, obj.__self__), obj=obj) logger.trace(pickler, "# Me1") return -if sys.hexversion >= 0x20500f0: - if not IS_PYPY: - @register(MemberDescriptorType) - @register(GetSetDescriptorType) - @register(MethodDescriptorType) - @register(WrapperDescriptorType) - @register(ClassMethodDescriptorType) - def save_wrapper_descriptor(pickler, obj): - logger.trace(pickler, "Wr: %s", obj) - pickler.save_reduce(_getattr, (obj.__objclass__, obj.__name__, - obj.__repr__()), obj=obj) - logger.trace(pickler, "# Wr") - return - else: - @register(MemberDescriptorType) - @register(GetSetDescriptorType) - def save_wrapper_descriptor(pickler, obj): - logger.trace(pickler, "Wr: %s", obj) - pickler.save_reduce(_getattr, (obj.__objclass__, obj.__name__, - obj.__repr__()), obj=obj) - logger.trace(pickler, "# Wr") - return - -elif not IS_PYPY: +if not IS_PYPY: + @register(MemberDescriptorType) + @register(GetSetDescriptorType) @register(MethodDescriptorType) @register(WrapperDescriptorType) + @register(ClassMethodDescriptorType) + def save_wrapper_descriptor(pickler, obj): + logger.trace(pickler, "Wr: %s", obj) + pickler.save_reduce(_getattr, (obj.__objclass__, obj.__name__, + obj.__repr__()), obj=obj) + logger.trace(pickler, "# Wr") + return +else: + @register(MemberDescriptorType) + @register(GetSetDescriptorType) def save_wrapper_descriptor(pickler, obj): logger.trace(pickler, "Wr: %s", obj) pickler.save_reduce(_getattr, (obj.__objclass__, obj.__name__, @@ -1806,7 +1575,7 @@ def save_wrapper_descriptor(pickler, obj): def save_cell(pickler, obj): try: f = obj.cell_contents - except: + except ValueError: # cell is empty logger.trace(pickler, "Ce3: %s", obj) # _shims._CELL_EMPTY is defined in _shims.py to support PyPy 2.7. # It unpickles to a sentinel object _dill._CELL_EMPTY, also created in @@ -1822,10 +1591,7 @@ def save_cell(pickler, obj): # The result of this function call will be None pickler.save_reduce(_shims._delattr, (obj, 'cell_contents')) # pop None created by calling _delattr off stack - if PY3: - pickler.write(bytes('0', 'UTF-8')) - else: - pickler.write('0') + pickler.write(bytes('0', 'UTF-8')) logger.trace(pickler, "# Ce3") return if is_dill(pickler, child=True): @@ -1909,10 +1675,7 @@ def _locate_object(address, module=None): for obj in special: if address == id(obj): return obj if module: - if PY3: - objects = iter(module.__dict__.values()) - else: - objects = module.__dict__.itervalues() + objects = iter(module.__dict__.values()) else: objects = iter(gc.get_objects()) for obj in objects: if address == id(obj): return obj @@ -1988,7 +1751,7 @@ def save_module(pickler, obj): pickler.save_reduce(_import_module, (obj.__name__,), obj=obj, state=_main_dict) logger.trace(pickler, "# M1") - elif PY3 and obj.__name__ == "dill._dill": + elif obj.__name__ == "dill._dill": logger.trace(pickler, "M2: %s", obj) pickler.save_global(obj, name="_dill") logger.trace(pickler, "# M2") @@ -2002,7 +1765,7 @@ def save_module(pickler, obj): @register(TypeType) def save_type(pickler, obj, postproc_list=None): if obj in _typemap: - logger.trace(pickler, "T1: %s" % obj) + logger.trace(pickler, "T1: %s", obj) # if obj in _incedental_types: # warnings.warn('Type %r may only exist on this implementation of Python and cannot be unpickled in other implementations.' % (obj,), PicklingWarning) pickler.save_reduce(_load_type, (_typemap[obj],), obj=obj) @@ -2010,7 +1773,7 @@ def save_type(pickler, obj, postproc_list=None): elif obj.__bases__ == (tuple,) and all([hasattr(obj, attr) for attr in ('_fields','_asdict','_make','_replace')]): # special case: namedtuples logger.trace(pickler, "T6: %s", obj) - if OLD37 or (not obj._field_defaults): + if not obj._field_defaults: pickler.save_reduce(_create_namedtuple, (obj.__name__, obj._fields, obj.__module__), obj=obj) else: defaults = [obj._field_defaults[field] for field in obj._fields if field in obj._field_defaults] @@ -2022,10 +1785,7 @@ def save_type(pickler, obj, postproc_list=None): elif obj is type(None): logger.trace(pickler, "T7: %s", obj) #XXX: pickler.save_reduce(type, (None,), obj=obj) - if PY3: - pickler.write(bytes('c__builtin__\nNoneType\n', 'UTF-8')) - else: - pickler.write('c__builtin__\nNoneType\n') + pickler.write(bytes('c__builtin__\nNoneType\n', 'UTF-8')) logger.trace(pickler, "# T7") elif obj is NotImplementedType: logger.trace(pickler, "T7: %s", obj) @@ -2056,7 +1816,7 @@ def save_type(pickler, obj, postproc_list=None): #print ("%s\n%s" % (obj.__bases__, obj.__dict__)) for name in _dict.get("__slots__", []): del _dict[name] - if PY3 and obj_name != obj.__name__: + if obj_name != obj.__name__: if postproc_list is None: postproc_list = [] postproc_list.append((setattr, (obj, '__qualname__', obj_name))) @@ -2077,12 +1837,6 @@ def save_type(pickler, obj, postproc_list=None): logger.trace(pickler, "# T4") return -# Error in PyPy 2.7 when adding ABC support -if IS_PYPY2: - @register(FrameType) - def save_frame(pickler, obj): - raise PicklingError('Cannot pickle a Python stack frame') - @register(property) def save_property(pickler, obj): logger.trace(pickler, "Pr: %s", obj) @@ -2094,24 +1848,15 @@ def save_property(pickler, obj): @register(classmethod) def save_classmethod(pickler, obj): logger.trace(pickler, "Cm: %s", obj) - im_func = '__func__' if PY3 else 'im_func' - try: - orig_func = getattr(obj, im_func) - except AttributeError: # Python 2.6 - orig_func = obj.__get__(None, object) - if isinstance(obj, classmethod): - orig_func = getattr(orig_func, im_func) # Unbind - - # if PY3: - # if type(obj.__dict__) is dict: - # if obj.__dict__: - # state = obj.__dict__ - # else: - # state = None + orig_func = obj.__func__ + + # if type(obj.__dict__) is dict: + # if obj.__dict__: + # state = obj.__dict__ # else: - # state = (None, {'__dict__', obj.__dict__}) + # state = None # else: - # state = None + # state = (None, {'__dict__', obj.__dict__}) pickler.save_reduce(type(obj), (orig_func,), obj=obj) logger.trace(pickler, "# Cm") @@ -2131,16 +1876,16 @@ def save_function(pickler, obj): found, _ = _getattribute(module, obj.__qualname__) if getattr(found, '__func__', None) is obj: _pypy_builtin = True - except: + except AttributeError: pass if _pypy_builtin: - logger.trace(pickler, "F3: %s" % obj) + logger.trace(pickler, "F3: %s", obj) pickler.save_reduce(getattr, (found, '__func__'), obj=obj) logger.trace(pickler, "# F3") return - logger.trace(pickler, "F1: %s" % obj) + logger.trace(pickler, "F1: %s", obj) _recurse = getattr(pickler, '_recurse', None) _postproc = getattr(pickler, '_postproc', None) _main_modified = getattr(pickler, '_main_modified', None) @@ -2157,7 +1902,7 @@ def save_function(pickler, obj): # is created to correctly handle recursion. globs = {'__name__': obj.__module__} else: - globs_copy = obj.__globals__ if PY3 else obj.func_globals + globs_copy = obj.__globals__ # If the globals is the __dict__ from the module being saved as a # session, substitute it by the dictionary being actually saved. @@ -2175,10 +1920,7 @@ def save_function(pickler, obj): # In the case that the globals are copied, we need to ensure that # the globals dictionary is updated when all objects in the # dictionary are already created. - if PY3: - glob_ids = {id(g) for g in globs_copy.values()} - else: - glob_ids = {id(g) for g in globs_copy.itervalues()} + glob_ids = {id(g) for g in globs_copy.values()} for stack_element in _postproc: if stack_element in glob_ids: _postproc[stack_element].append((_setitems, (globs, globs_copy))) @@ -2186,42 +1928,28 @@ def save_function(pickler, obj): else: postproc_list.append((_setitems, (globs, globs_copy))) - if PY3: - closure = obj.__closure__ - state_dict = {} - for fattrname in ('__doc__', '__kwdefaults__', '__annotations__'): - fattr = getattr(obj, fattrname, None) - if fattr is not None: - state_dict[fattrname] = fattr - if obj.__qualname__ != obj.__name__: - state_dict['__qualname__'] = obj.__qualname__ - if '__name__' not in globs or obj.__module__ != globs['__name__']: - state_dict['__module__'] = obj.__module__ - - state = obj.__dict__ - if type(state) is not dict: - state_dict['__dict__'] = state - state = None - if state_dict: - state = state, state_dict - - _save_with_postproc(pickler, (_create_function, ( - obj.__code__, globs, obj.__name__, obj.__defaults__, - closure - ), state), obj=obj, postproc_list=postproc_list) - else: - closure = obj.func_closure - if obj.__doc__ is not None: - postproc_list.append((setattr, (obj, '__doc__', obj.__doc__))) - if '__name__' not in globs or obj.__module__ != globs['__name__']: - postproc_list.append((setattr, (obj, '__module__', obj.__module__))) - if obj.__dict__: - postproc_list.append((setattr, (obj, '__dict__', obj.__dict__))) - - _save_with_postproc(pickler, (_create_function, ( - obj.func_code, globs, obj.func_name, obj.func_defaults, + closure = obj.__closure__ + state_dict = {} + for fattrname in ('__doc__', '__kwdefaults__', '__annotations__'): + fattr = getattr(obj, fattrname, None) + if fattr is not None: + state_dict[fattrname] = fattr + if obj.__qualname__ != obj.__name__: + state_dict['__qualname__'] = obj.__qualname__ + if '__name__' not in globs or obj.__module__ != globs['__name__']: + state_dict['__module__'] = obj.__module__ + + state = obj.__dict__ + if type(state) is not dict: + state_dict['__dict__'] = state + state = None + if state_dict: + state = state, state_dict + + _save_with_postproc(pickler, (_create_function, ( + obj.__code__, globs, obj.__name__, obj.__defaults__, closure - )), obj=obj, postproc_list=postproc_list) + ), state), obj=obj, postproc_list=postproc_list) # Lift closure cell update to earliest function (#458) if _postproc: @@ -2237,10 +1965,7 @@ def save_function(pickler, obj): # Change the value of the cell pickler.save_reduce(*possible_postproc) # pop None created by calling preprocessing step off stack - if PY3: - pickler.write(bytes('0', 'UTF-8')) - else: - pickler.write('0') + pickler.write(bytes('0', 'UTF-8')) logger.trace(pickler, "# F1") else: @@ -2372,10 +2097,7 @@ def check(obj, *args, **kwds): # unpickle = "dill.loads(%s, ignore=%s)"%(repr(_obj), repr(ignore)) # cmd = [python, "-c", "import dill; print(%s)"%unpickle] # msg = "SUCCESS" if not subprocess.call(cmd) else "LOAD FAILED" - if verbose is None: - msg = "%s -c import dill; dill.loads(%s)" % (python, repr(_obj)) - else: - msg = "%s -c import dill; print(dill.loads(%s))" % (python, repr(_obj)) + msg = "%s -c import dill; print(dill.loads(%s))" % (python, repr(_obj)) msg = "SUCCESS" if not subprocess.call(msg.split(None,2)) else "LOAD FAILED" if verbose: print(msg) @@ -2384,7 +2106,7 @@ def check(obj, *args, **kwds): # use to protect against missing attributes def is_dill(pickler, child=None): "check the dill-ness of your pickler" - if (child is False) or PY34 or (not hasattr(pickler.__class__, 'mro')): + if child is False or not hasattr(pickler.__class__, 'mro'): return 'dill' in pickler.__module__ return Pickler in pickler.__class__.mro() @@ -2394,9 +2116,8 @@ def _extend(): for t,func in Pickler.dispatch.items(): try: StockPickler.dispatch[t] = func - except: #TypeError, PicklingError, UnpicklingError + except Exception: #TypeError, PicklingError, UnpicklingError logger.trace(pickler, "skip: %s", t) - else: pass return del diff, _use_diff, use_diff diff --git a/dill/_objects.py b/dill/_objects.py index 954a3c04..8efee787 100644 --- a/dill/_objects.py +++ b/dill/_objects.py @@ -15,22 +15,9 @@ # helper imports import warnings; warnings.filterwarnings("ignore", category=DeprecationWarning) import sys -PY3 = (hex(sys.hexversion) >= '0x30000f0') -if PY3: - import queue as Queue - import dbm as anydbm -else: - import Queue - import anydbm - import sets # deprecated/removed - import mutex # removed -try: - from cStringIO import StringIO # has StringI and StringO types -except ImportError: # only has StringIO type - if PY3: - from io import BytesIO as StringIO - else: - from StringIO import StringIO +import queue as Queue +import dbm as anydbm +from io import BytesIO as StringIO import re import array import collections @@ -41,6 +28,7 @@ import weakref import pprint import decimal +import numbers import functools import itertools import operator @@ -56,6 +44,7 @@ import hmac import os import logging +import logging.handlers import optparse #import __hello__ import threading @@ -64,8 +53,7 @@ try: import bz2 import sqlite3 - if PY3: import dbm.ndbm as dbm - else: import dbm + import dbm.ndbm as dbm HAS_ALL = True except ImportError: # Ubuntu HAS_ALL = False @@ -112,7 +100,7 @@ class _newclass2(object): def _function(x): yield x def _function2(): try: raise - except: + except Exception: from sys import exc_info e, er, tb = exc_info() return er, tb @@ -123,20 +111,12 @@ class _Struct(ctypes.Structure): _filedescrip, _tempfile = tempfile.mkstemp('r') # deleted in cleanup _tmpf = tempfile.TemporaryFile('w') -# put the objects in order, if possible -try: - from collections import OrderedDict as odict -except ImportError: - try: - from ordereddict import OrderedDict as odict - except ImportError: - odict = dict # objects used by dill for type declaration -registered = d = odict() +registered = d = {} # objects dill fails to pickle -failures = x = odict() +failures = x = {} # all other type objects -succeeds = a = odict() +succeeds = a = {} # types module (part of CH 8) a['BooleanType'] = bool(1) @@ -157,12 +137,8 @@ class _Struct(ctypes.Structure): a['StringType'] = _str = str(1) a['TupleType'] = _tuple = () a['TypeType'] = type -if PY3: - a['LongType'] = _int - a['UnicodeType'] = _str -else: - a['LongType'] = long(1) - a['UnicodeType'] = unicode(1) +a['LongType'] = _int +a['UnicodeType'] = _str # built-in constants (CH 4) a['CopyrightType'] = copyright # built-in types (CH 5) @@ -181,17 +157,13 @@ class _Struct(ctypes.Structure): a['TZInfoType'] = datetime.tzinfo() a['DateTimeType'] = datetime.datetime.today() a['CalendarType'] = calendar.Calendar() -if not PY3: - a['SetsType'] = sets.Set() - a['ImmutableSetType'] = sets.ImmutableSet() - a['MutexType'] = mutex.mutex() # numeric and mathematical types (CH 9) a['DecimalType'] = decimal.Decimal(1) a['CountType'] = itertools.count(0) # data compression and archiving (CH 12) a['TarInfoType'] = tarfile.TarInfo() # generic operating system services (CH 15) -a['LoggerType'] = logging.getLogger() +a['LoggerType'] = _logger = logging.getLogger() a['FormatterType'] = logging.Formatter() # pickle ok a['FilterType'] = logging.Filter() # pickle ok a['LogRecordType'] = logging.makeLogRecord(_dict) # pickle ok @@ -223,40 +195,33 @@ class _Struct(ctypes.Structure): #NOTE: ctypes.c_int._objects is memberdescriptor for object's __dict__ #NOTE: base class of all ctypes data types is non-public _CData -try: # python 2.6 - import fractions - import number - import io - from io import StringIO as TextIO - # built-in functions (CH 2) - a['ByteArrayType'] = bytearray([1]) - # numeric and mathematical types (CH 9) - a['FractionType'] = fractions.Fraction() - a['NumberType'] = numbers.Number() - # generic operating system services (CH 15) - a['IOBaseType'] = io.IOBase() - a['RawIOBaseType'] = io.RawIOBase() - a['TextIOBaseType'] = io.TextIOBase() - a['BufferedIOBaseType'] = io.BufferedIOBase() - a['UnicodeIOType'] = TextIO() # the new StringIO - a['LoggingAdapterType'] = logging.LoggingAdapter(_logger,_dict) # pickle ok - if HAS_CTYPES: - a['CBoolType'] = ctypes.c_bool(1) - a['CLongDoubleType'] = ctypes.c_longdouble() -except ImportError: - pass -try: # python 2.7 - import argparse - # data types (CH 8) - a['OrderedDictType'] = collections.OrderedDict(_dict) - a['CounterType'] = collections.Counter(_dict) - if HAS_CTYPES: - a['CSSizeTType'] = ctypes.c_ssize_t() - # generic operating system services (CH 15) - a['NullHandlerType'] = logging.NullHandler() # pickle ok # new 2.7 - a['ArgParseFileType'] = argparse.FileType() # pickle ok -except (AttributeError, ImportError): - pass +import fractions +import io +from io import StringIO as TextIO +# built-in functions (CH 2) +a['ByteArrayType'] = bytearray([1]) +# numeric and mathematical types (CH 9) +a['FractionType'] = fractions.Fraction() +a['NumberType'] = numbers.Number() +# generic operating system services (CH 15) +a['IOBaseType'] = io.IOBase() +a['RawIOBaseType'] = io.RawIOBase() +a['TextIOBaseType'] = io.TextIOBase() +a['BufferedIOBaseType'] = io.BufferedIOBase() +a['UnicodeIOType'] = TextIO() # the new StringIO +a['LoggerAdapterType'] = logging.LoggerAdapter(_logger,_dict) # pickle ok +if HAS_CTYPES: + a['CBoolType'] = ctypes.c_bool(1) + a['CLongDoubleType'] = ctypes.c_longdouble() +import argparse +# data types (CH 8) +a['OrderedDictType'] = collections.OrderedDict(_dict) +a['CounterType'] = collections.Counter(_dict) +if HAS_CTYPES: + a['CSSizeTType'] = ctypes.c_ssize_t() +# generic operating system services (CH 15) +a['NullHandlerType'] = logging.NullHandler() # pickle ok # new 2.7 +a['ArgParseFileType'] = argparse.FileType() # pickle ok # -- pickle fails on all below here ----------------------------------------- # types module (part of CH 8) @@ -288,31 +253,21 @@ class _Struct(ctypes.Structure): except ImportError: pass # other (concrete) object types -if PY3: - d['CellType'] = (_lambda)(0).__closure__[0] - a['XRangeType'] = _xrange = range(1) -else: - d['CellType'] = (_lambda)(0).func_closure[0] - a['XRangeType'] = _xrange = xrange(1) +d['CellType'] = (_lambda)(0).__closure__[0] +a['XRangeType'] = _xrange = range(1) a['MethodDescriptorType'] = type.__dict__['mro'] a['WrapperDescriptorType'] = type.__repr__ #a['WrapperDescriptorType2'] = type.__dict__['__module__']#XXX: GetSetDescriptor -a['ClassMethodDescriptorType'] = type.__dict__['__prepare__' if PY3 else 'mro'] +a['ClassMethodDescriptorType'] = type.__dict__['__prepare__'] # built-in functions (CH 2) -if PY3 or IS_PYPY: - _methodwrap = (1).__lt__ -else: - _methodwrap = (1).__cmp__ +_methodwrap = (1).__lt__ a['MethodWrapperType'] = _methodwrap a['StaticMethodType'] = staticmethod(_method) a['ClassMethodType'] = classmethod(_method) a['PropertyType'] = property() d['SuperType'] = super(Exception, _exception) # string services (CH 7) -if PY3: - _in = _bytes -else: - _in = _str +_in = _bytes a['InputType'] = _cstrI = StringIO(_in) a['OutputType'] = _cstrO = StringIO() # data types (CH 8) @@ -327,16 +282,12 @@ class _Struct(ctypes.Structure): a['QueueType'] = Queue.Queue() # numeric and mathematical types (CH 9) d['PartialType'] = functools.partial(int,base=2) -if PY3: - a['IzipType'] = zip('0','1') -else: - a['IzipType'] = itertools.izip('0','1') +a['IzipType'] = zip('0','1') a['ChainType'] = itertools.chain('0','1') d['ItemGetterType'] = operator.itemgetter(0) d['AttrGetterType'] = operator.attrgetter('__repr__') # file and directory access (CH 10) -if PY3: _fileW = _cstrO -else: _fileW = _tmpf +_fileW = _cstrO # data persistence (CH 11) if HAS_ALL: x['ConnectionType'] = _conn = sqlite3.connect(':memory:') @@ -344,8 +295,7 @@ class _Struct(ctypes.Structure): a['ShelveType'] = shelve.Shelf({}) # data compression and archiving (CH 12) if HAS_ALL: - if (hex(sys.hexversion) < '0x2070ef0') or PY3: - x['BZ2FileType'] = bz2.BZ2File(os.devnull) + x['BZ2FileType'] = bz2.BZ2File(os.devnull) x['BZ2CompressorType'] = bz2.BZ2Compressor() x['BZ2DecompressorType'] = bz2.BZ2Decompressor() #x['ZipFileType'] = _zip = zipfile.ZipFile(os.devnull,'w') @@ -362,17 +312,10 @@ class _Struct(ctypes.Structure): a['NamedLoggerType'] = _logger = logging.getLogger(__name__) #a['FrozenModuleType'] = __hello__ #FIXME: prints "Hello world..." # interprocess communication (CH 17) -if PY3: - x['SocketType'] = _socket = socket.socket() - x['SocketPairType'] = socket.socketpair()[0] -else: - x['SocketType'] = _socket = socket.socket() - x['SocketPairType'] = _socket._sock +x['SocketType'] = _socket = socket.socket() +x['SocketPairType'] = socket.socketpair()[0] # python runtime services (CH 27) -if PY3: - a['GeneratorContextManagerType'] = contextlib.contextmanager(max)([1]) -else: - a['GeneratorContextManagerType'] = contextlib.GeneratorContextManager(max) +a['GeneratorContextManagerType'] = contextlib.contextmanager(max)([1]) try: # ipython __IPYTHON__ is True # is ipython @@ -389,27 +332,21 @@ class _Struct(ctypes.Structure): a['NumpyInt32Type'] = _numpy_int32 except ImportError: pass -try: # python 2.6 - # numeric and mathematical types (CH 9) - a['ProductType'] = itertools.product('0','1') - # generic operating system services (CH 15) - a['FileHandlerType'] = logging.FileHandler(os.devnull) - a['RotatingFileHandlerType'] = logging.handlers.RotatingFileHandler(os.devnull) - a['SocketHandlerType'] = logging.handlers.SocketHandler('localhost',514) - a['MemoryHandlerType'] = logging.handlers.MemoryHandler(1) -except AttributeError: - pass -try: # python 2.7 - # data types (CH 8) - a['WeakSetType'] = weakref.WeakSet() # 2.7 -# # generic operating system services (CH 15) [errors when dill is imported] -# a['ArgumentParserType'] = _parser = argparse.ArgumentParser('PROG') -# a['NamespaceType'] = _parser.parse_args() # pickle ok -# a['SubParsersActionType'] = _parser.add_subparsers() -# a['MutuallyExclusiveGroupType'] = _parser.add_mutually_exclusive_group() -# a['ArgumentGroupType'] = _parser.add_argument_group() -except AttributeError: - pass +# numeric and mathematical types (CH 9) +a['ProductType'] = itertools.product('0','1') +# generic operating system services (CH 15) +a['FileHandlerType'] = logging.FileHandler(os.devnull) +a['RotatingFileHandlerType'] = logging.handlers.RotatingFileHandler(os.devnull) +a['SocketHandlerType'] = logging.handlers.SocketHandler('localhost',514) +a['MemoryHandlerType'] = logging.handlers.MemoryHandler(1) +# data types (CH 8) +a['WeakSetType'] = weakref.WeakSet() # 2.7 +# generic operating system services (CH 15) [errors when dill is imported] +#a['ArgumentParserType'] = _parser = argparse.ArgumentParser('PROG') +#a['NamespaceType'] = _parser.parse_args() # pickle ok +#a['SubParsersActionType'] = _parser.add_subparsers() +#a['MutuallyExclusiveGroupType'] = _parser.add_mutually_exclusive_group() +#a['ArgumentGroupType'] = _parser.add_argument_group() # -- dill fails in some versions below here --------------------------------- # types module (part of CH 8) @@ -431,14 +368,10 @@ class _Struct(ctypes.Structure): d["OdictItemsType"] = X.items() a["OdictIteratorType"] = iter(X.keys()) #FIXME: list_iterator del X -if PY3: #FIXME: list_iterator - a['DictionaryItemIteratorType'] = iter(type.__dict__.items()) - a['DictionaryKeyIteratorType'] = iter(type.__dict__.keys()) - a['DictionaryValueIteratorType'] = iter(type.__dict__.values()) -else: - a['DictionaryItemIteratorType'] = type.__dict__.iteritems() - a['DictionaryKeyIteratorType'] = type.__dict__.iterkeys() - a['DictionaryValueIteratorType'] = type.__dict__.itervalues() +#FIXME: list_iterator +a['DictionaryItemIteratorType'] = iter(type.__dict__.items()) +a['DictionaryKeyIteratorType'] = iter(type.__dict__.keys()) +a['DictionaryValueIteratorType'] = iter(type.__dict__.values()) if sys.hexversion >= 0x30800a0: a["DictReversekeyiteratorType"] = reversed({}.keys()) a["DictReversevalueiteratorType"] = reversed({}.values()) @@ -446,8 +379,9 @@ class _Struct(ctypes.Structure): try: import symtable + #FIXME: fails to pickle x["SymtableEntryType"] = symtable.symtable("", "string", "exec")._table -except: #FIXME: fails to pickle +except ImportError: pass if sys.hexversion >= 0x30a00a0: @@ -468,19 +402,12 @@ class _Struct(ctypes.Structure): x['GzipFileType'] = gzip.GzipFile(fileobj=_fileW) # generic operating system services (CH 15) a['StreamHandlerType'] = logging.StreamHandler() -try: # python 2.6 - # numeric and mathematical types (CH 9) - a['PermutationsType'] = itertools.permutations('0') - a['CombinationsType'] = itertools.combinations('0',1) -except AttributeError: - pass -try: # python 2.7 - # numeric and mathematical types (CH 9) - a['RepeatType'] = itertools.repeat(0) - a['CompressType'] = itertools.compress('0',[1]) - #XXX: ...and etc -except AttributeError: - pass +# numeric and mathematical types (CH 9) +a['PermutationsType'] = itertools.permutations('0') +a['CombinationsType'] = itertools.combinations('0',1) +a['RepeatType'] = itertools.repeat(0) +a['CompressType'] = itertools.compress('0',[1]) +#XXX: ...and etc # -- dill fails on all below here ------------------------------------------- # types module (part of CH 8) @@ -513,7 +440,7 @@ class _Struct(ctypes.Structure): x['CSVDictWriterType'] = csv.DictWriter(_cstrO,{}) # cryptographic services (CH 14) x['HashType'] = hashlib.md5() -if (hex(sys.hexversion) < '0x30800a1'): +if (sys.hexversion < 0x30800a1): x['HMACType'] = hmac.new(_in) else: x['HMACType'] = hmac.new(_in, digestmod='md5') @@ -544,38 +471,22 @@ class _Struct(ctypes.Structure): x['FieldType'] = _field = _Struct._field x['CFUNCTYPEType'] = _cfunc = ctypes.CFUNCTYPE(ctypes.c_char) x['CFunctionType'] = _cfunc(str) -try: # python 2.6 - # numeric and mathematical types (CH 9) - a['MethodCallerType'] = operator.methodcaller('mro') # 2.6 -except AttributeError: - pass -try: # python 2.7 - # built-in types (CH 5) - x['MemoryType'] = memoryview(_in) # 2.7 - x['MemoryType2'] = memoryview(bytearray(_in)) # 2.7 - if PY3: - d['DictItemsType'] = _dict.items() # 2.7 - d['DictKeysType'] = _dict.keys() # 2.7 - d['DictValuesType'] = _dict.values() # 2.7 - else: - d['DictItemsType'] = _dict.viewitems() # 2.7 - d['DictKeysType'] = _dict.viewkeys() # 2.7 - d['DictValuesType'] = _dict.viewvalues() # 2.7 - # generic operating system services (CH 15) - a['RawTextHelpFormatterType'] = argparse.RawTextHelpFormatter('PROG') - a['RawDescriptionHelpFormatterType'] = argparse.RawDescriptionHelpFormatter('PROG') - a['ArgDefaultsHelpFormatterType'] = argparse.ArgumentDefaultsHelpFormatter('PROG') -except NameError: - pass -try: # python 2.7 (and not 3.1) - x['CmpKeyType'] = _cmpkey = functools.cmp_to_key(_methodwrap) # 2.7, >=3.2 - x['CmpKeyObjType'] = _cmpkey('0') #2.7, >=3.2 -except AttributeError: - pass -if PY3: # oddities: removed, etc - x['BufferType'] = x['MemoryType'] -else: - x['BufferType'] = buffer('') +# numeric and mathematical types (CH 9) +a['MethodCallerType'] = operator.methodcaller('mro') # 2.6 +# built-in types (CH 5) +x['MemoryType'] = memoryview(_in) # 2.7 +x['MemoryType2'] = memoryview(bytearray(_in)) # 2.7 +d['DictItemsType'] = _dict.items() # 2.7 +d['DictKeysType'] = _dict.keys() # 2.7 +d['DictValuesType'] = _dict.values() # 2.7 +# generic operating system services (CH 15) +a['RawTextHelpFormatterType'] = argparse.RawTextHelpFormatter('PROG') +a['RawDescriptionHelpFormatterType'] = argparse.RawDescriptionHelpFormatter('PROG') +a['ArgDefaultsHelpFormatterType'] = argparse.ArgumentDefaultsHelpFormatter('PROG') +x['CmpKeyType'] = _cmpkey = functools.cmp_to_key(_methodwrap) # 2.7, >=3.2 +x['CmpKeyObjType'] = _cmpkey('0') #2.7, >=3.2 +# oddities: removed, etc +x['BufferType'] = x['MemoryType'] from dill._dill import _testcapsule if _testcapsule is not None: diff --git a/dill/_shims.py b/dill/_shims.py index 2da6f5be..2e6641ca 100644 --- a/dill/_shims.py +++ b/dill/_shims.py @@ -49,7 +49,8 @@ def _setattr(object, name, value): https://github.com/uqfoundation/dill/pull/443 """ -import inspect, sys +import inspect +import sys _dill = sys.modules['dill._dill'] @@ -188,105 +189,5 @@ def _create_class(): _CELL_EMPTY = register_shim('_CELL_EMPTY', None) -if _dill.OLD37: - if _dill.HAS_CTYPES and hasattr(_dill.ctypes, 'pythonapi') and hasattr(_dill.ctypes.pythonapi, 'PyCell_Set'): - # CPython - ctypes = _dill.ctypes - - _PyCell_Set = ctypes.pythonapi.PyCell_Set - - def _setattr(object, name, value): - if type(object) is _dill.CellType and name == 'cell_contents': - _PyCell_Set.argtypes = (ctypes.py_object, ctypes.py_object) - _PyCell_Set(object, value) - else: - setattr(object, name, value) - - def _delattr(object, name): - if type(object) is _dill.CellType and name == 'cell_contents': - _PyCell_Set.argtypes = (ctypes.py_object, ctypes.c_void_p) - _PyCell_Set(object, None) - else: - delattr(object, name) - - # General Python (not CPython) up to 3.6 is in a weird case, where it is - # possible to pickle recursive cells, but we can't assign directly to the - # cell. - elif _dill.PY3: - # Use nonlocal variables to reassign the cell value. - # https://stackoverflow.com/a/59276835 - __nonlocal = ('nonlocal cell',) - exec('''def _setattr(cell, name, value): - if type(cell) is _dill.CellType and name == 'cell_contents': - def cell_setter(value): - %s - cell = value # pylint: disable=unused-variable - func = _dill.FunctionType(cell_setter.__code__, globals(), "", None, (cell,)) # same as cell_setter, but with cell being the cell's contents - func(value) - else: - setattr(cell, name, value)''' % __nonlocal) - - exec('''def _delattr(cell, name): - if type(cell) is _dill.CellType and name == 'cell_contents': - try: - cell.cell_contents - except: - return - def cell_deleter(): - %s - del cell # pylint: disable=unused-variable - func = _dill.FunctionType(cell_deleter.__code__, globals(), "", None, (cell,)) # same as cell_deleter, but with cell being the cell's contents - func() - else: - delattr(cell, name)''' % __nonlocal) - - else: - # Likely PyPy 2.7. Simulate the nonlocal keyword with bytecode - # manipulation. - - # The following function is based on 'cell_set' from 'cloudpickle' - # https://github.com/cloudpipe/cloudpickle/blob/5d89947288a18029672596a4d719093cc6d5a412/cloudpickle/cloudpickle.py#L393-L482 - # Copyright (c) 2012, Regents of the University of California. - # Copyright (c) 2009 `PiCloud, Inc. `_. - # License: https://github.com/cloudpipe/cloudpickle/blob/master/LICENSE - def _setattr(cell, name, value): - if type(cell) is _dill.CellType and name == 'cell_contents': - _cell_set = _dill.FunctionType( - _cell_set_template_code, {}, '_cell_set', (), (cell,),) - _cell_set(value) - else: - setattr(cell, name, value) - - def _cell_set_factory(value): - lambda: cell - cell = value - - co = _cell_set_factory.__code__ - - _cell_set_template_code = _dill.CodeType( - co.co_argcount, - co.co_nlocals, - co.co_stacksize, - co.co_flags, - co.co_code, - co.co_consts, - co.co_names, - co.co_varnames, - co.co_filename, - co.co_name, - co.co_firstlineno, - co.co_lnotab, - co.co_cellvars, # co_freevars is initialized with co_cellvars - (), # co_cellvars is made empty - ) - - del co - - def _delattr(cell, name): - if type(cell) is _dill.CellType and name == 'cell_contents': - pass - else: - delattr(cell, name) - _setattr = register_shim('_setattr', setattr) _delattr = register_shim('_delattr', delattr) diff --git a/dill/detect.py b/dill/detect.py index 26d5b219..b6a6cb76 100644 --- a/dill/detect.py +++ b/dill/detect.py @@ -11,9 +11,8 @@ import dis from inspect import ismethod, isfunction, istraceback, isframe, iscode -from .pointers import parent, reference, at, parents, children -from ._dill import PY3 +from .pointers import parent, reference, at, parents, children from .logger import trace __all__ = ['baditems','badobjects','badtypes','code','errors','freevars', @@ -25,9 +24,7 @@ def getmodule(object, _filename=None, force=False): from inspect import getmodule as getmod module = getmod(object, _filename) if module or not force: return module - if PY3: builtins = 'builtins' - else: builtins = '__builtin__' - builtins = __import__(builtins) + import builtins from .source import getname name = getname(object, force=True) return builtins if name in vars(builtins).keys() else None @@ -37,26 +34,17 @@ def outermost(func): # is analogous to getsource(func,enclosing=True) NOTE: this is the object-equivalent of getsource(func, enclosing=True) """ - if PY3: - if ismethod(func): - _globals = func.__func__.__globals__ or {} - elif isfunction(func): - _globals = func.__globals__ or {} - else: - return #XXX: or raise? no matches - _globals = _globals.items() + if ismethod(func): + _globals = func.__func__.__globals__ or {} + elif isfunction(func): + _globals = func.__globals__ or {} else: - if ismethod(func): - _globals = func.im_func.func_globals or {} - elif isfunction(func): - _globals = func.func_globals or {} - else: - return #XXX: or raise? no matches - _globals = _globals.iteritems() + return #XXX: or raise? no matches + _globals = _globals.items() # get the enclosing source from .source import getsourcelines try: lines,lnum = getsourcelines(func, enclosing=True) - except: #TypeError, IOError + except Exception: #TypeError, IOError lines,lnum = [],None code = ''.join(lines) # get all possible names,objects that are named in the enclosing source @@ -65,7 +53,7 @@ def outermost(func): # is analogous to getsource(func,enclosing=True) for name,obj in _locals: #XXX: don't really need 'name' try: if getsourcelines(obj) == (lines,lnum): return obj - except: #TypeError, IOError + except Exception: #TypeError, IOError pass return #XXX: or raise? no matches @@ -83,18 +71,12 @@ def nestedcode(func, recurse=True): #XXX: or return dict of {co_name: co} ? return list(nested) def code(func): - '''get the code object for the given function or method + """get the code object for the given function or method NOTE: use dill.source.getsource(CODEOBJ) to get the source code - ''' - if PY3: - im_func = '__func__' - func_code = '__code__' - else: - im_func = 'im_func' - func_code = 'func_code' - if ismethod(func): func = getattr(func, im_func) - if isfunction(func): func = getattr(func, func_code) + """ + if ismethod(func): func = func.__func__ + if isfunction(func): func = func.__code__ if istraceback(func): func = func.tb_frame if isframe(func): func = func.f_code if iscode(func): return func @@ -109,13 +91,6 @@ def referrednested(func, recurse=True): #XXX: return dict of {__name__: obj} ? If possible, python builds code objects, but delays building functions until func() is called. """ - if PY3: - att1 = '__code__' - att0 = '__func__' - else: - att1 = 'func_code' # functions - att0 = 'im_func' # methods - import gc funcs = set() # get the code objects, and try to track down by referrence @@ -123,16 +98,16 @@ def referrednested(func, recurse=True): #XXX: return dict of {__name__: obj} ? # look for function objects that refer to the code object for obj in gc.get_referrers(co): # get methods - _ = getattr(obj, att0, None) # ismethod - if getattr(_, att1, None) is co: funcs.add(obj) + _ = getattr(obj, '__func__', None) # ismethod + if getattr(_, '__code__', None) is co: funcs.add(obj) # get functions - elif getattr(obj, att1, None) is co: funcs.add(obj) + elif getattr(obj, '__code__', None) is co: funcs.add(obj) # get frame objects elif getattr(obj, 'f_code', None) is co: funcs.add(obj) # get code objects elif hasattr(obj, 'co_code') and obj is co: funcs.add(obj) -# frameobjs => func.func_code.co_varnames not in func.func_code.co_cellvars -# funcobjs => func.func_code.co_cellvars not in func.func_code.co_varnames +# frameobjs => func.__code__.co_varnames not in func.__code__.co_cellvars +# funcobjs => func.__code__.co_cellvars not in func.__code__.co_varnames # frameobjs are not found, however funcobjs are... # (see: test_mixins.quad ... and test_mixins.wtf) # after execution, code objects get compiled, and then may be found by gc @@ -143,28 +118,20 @@ def freevars(func): """get objects defined in enclosing code that are referred to by func returns a dict of {name:object}""" - if PY3: - im_func = '__func__' - func_code = '__code__' - func_closure = '__closure__' - else: - im_func = 'im_func' - func_code = 'func_code' - func_closure = 'func_closure' - if ismethod(func): func = getattr(func, im_func) + if ismethod(func): func = func.__func__ if isfunction(func): - closures = getattr(func, func_closure) or () - func = getattr(func, func_code).co_freevars # get freevars + closures = func.__closure__ or () + func = func.__code__.co_freevars # get freevars else: return {} def get_cell_contents(): - for (name,c) in zip(func,closures): + for name, c in zip(func, closures): try: cell_contents = c.cell_contents - except: + except ValueError: # cell is empty continue - yield (name,c.cell_contents) + yield name, c.cell_contents return dict(get_cell_contents()) @@ -175,7 +142,7 @@ def nestedglobals(func, recurse=True): if func is None: return list() import sys from .temp import capture - CAN_NULL = sys.hexversion >= 51052711 #NULL may be prepended >= 3.11a7 + CAN_NULL = sys.hexversion >= 0x30b00a7 # NULL may be prepended >= 3.11a7 names = set() with capture('stdout') as out: dis.dis(func) #XXX: dis.dis(None) disassembles last traceback @@ -199,37 +166,27 @@ def globalvars(func, recurse=True, builtin=False): """get objects defined in global scope that are referred to by func return a dict of {name:object}""" - if PY3: - im_func = '__func__' - func_code = '__code__' - func_globals = '__globals__' - func_closure = '__closure__' - else: - im_func = 'im_func' - func_code = 'func_code' - func_globals = 'func_globals' - func_closure = 'func_closure' - if ismethod(func): func = getattr(func, im_func) + if ismethod(func): func = func.__func__ if isfunction(func): globs = vars(getmodule(sum)).copy() if builtin else {} # get references from within closure orig_func, func = func, set() - for obj in getattr(orig_func, func_closure) or {}: + for obj in orig_func.__closure__ or {}: try: cell_contents = obj.cell_contents - except: + except ValueError: # cell is empty pass else: _vars = globalvars(cell_contents, recurse, builtin) or {} func.update(_vars) #XXX: (above) be wary of infinte recursion? globs.update(_vars) # get globals - globs.update(getattr(orig_func, func_globals) or {}) + globs.update(orig_func.__globals__ or {}) # get names of references if not recurse: - func.update(getattr(orig_func, func_code).co_names) + func.update(orig_func.__code__.co_names) else: - func.update(nestedglobals(getattr(orig_func, func_code))) + func.update(nestedglobals(orig_func.__code__)) # find globals for all entries of func for key in func.copy(): #XXX: unnecessary...? nested_func = globs.get(key) @@ -254,7 +211,7 @@ def globalvars(func, recurse=True, builtin=False): func.update(globalvars(nested_func, True, builtin)) else: return {} - #NOTE: if name not in func_globals, then we skip it... + #NOTE: if name not in __globals__, then we skip it... return dict((name,globs[name]) for name in func if name in globs) diff --git a/dill/logger.py b/dill/logger.py index 7fa2856c..fedff6bf 100644 --- a/dill/logger.py +++ b/dill/logger.py @@ -215,9 +215,9 @@ def format(self, record): def trace(arg: Union[bool, TextIO, str, os.PathLike] = None, *, mode: str = 'a') -> NoReturn: """print a trace through the stack when pickling; useful for debugging - + With a single boolean argument, enable or disable the tracing. - + Example usage: >>> import dill diff --git a/dill/settings.py b/dill/settings.py index 4d0226b0..b105d2e8 100644 --- a/dill/settings.py +++ b/dill/settings.py @@ -9,10 +9,7 @@ global settings for Pickler """ -try: - from pickle import DEFAULT_PROTOCOL -except ImportError: - from pickle import HIGHEST_PROTOCOL as DEFAULT_PROTOCOL +from pickle import DEFAULT_PROTOCOL settings = { #'main' : None, diff --git a/dill/source.py b/dill/source.py index 47064a16..7e70a635 100644 --- a/dill/source.py +++ b/dill/source.py @@ -29,8 +29,6 @@ ismodule, istraceback) from tokenize import TokenError -from ._dill import PY3 - def isfrommain(obj): "check if object was built in __main__" @@ -58,7 +56,7 @@ def _matchlambda(func, line): lhs,rhs = line.split('lambda ',1)[-1].split(":", 1) #FIXME: if !1 inputs try: #FIXME: unsafe _ = eval("lambda %s : %s" % (lhs,rhs), globals(),locals()) - except: _ = dummy + except Exception: _ = dummy # get code objects, for comparison _, code = getcode(_).co_code, getcode(func).co_code # check if func is in closure @@ -80,7 +78,7 @@ def _matchlambda(func, line): _lhs,_rhs = rhs.split('lambda ',1)[-1].split(":",1) #FIXME: if !1 inputs try: #FIXME: unsafe _f = eval("lambda %s : %s" % (_lhs,_rhs), globals(),locals()) - except: _f = dummy + except Exception: _f = dummy # get code objects, for comparison _, code = getcode(_f).co_code, getcode(func).co_code if len(_) != len(code): return False @@ -120,7 +118,7 @@ def findsource(object): try: import readline err = '' - except: + except ImportError: import sys err = sys.exc_info()[1].args[0] if sys.platform[:3] == 'win': @@ -165,16 +163,14 @@ def findsource(object): name = object.__name__ if name == '': pat1 = r'(.*(?': pat1 = r'(.*(?', 'exec'), *a) __globals__ = globals() __locals__ = locals() -wrap2 = ''' -def _wrap(f): - """ encapsulate a function and it's __import__ """ - def func(*args, **kwds): - try: - # _ = eval(getsource(f, force=True)) #XXX: safer but less robust - exec getimportable(f, alias='_') in %s, %s - except: - raise ImportError('cannot import name ' + f.__name__) - return _(*args, **kwds) - func.__name__ = f.__name__ - func.__doc__ = f.__doc__ - return func -''' % ('__globals__', '__locals__') -wrap3 = ''' def _wrap(f): """ encapsulate a function and it's __import__ """ def func(*args, **kwds): try: # _ = eval(getsource(f, force=True)) #XXX: safer but less robust - exec(getimportable(f, alias='_'), %s, %s) - except: + exec(getimportable(f, alias='_'), __globals__, __locals__) + except Exception: raise ImportError('cannot import name ' + f.__name__) return _(*args, **kwds) func.__name__ = f.__name__ func.__doc__ = f.__doc__ return func -''' % ('__globals__', '__locals__') -if PY3: - exec(wrap3) -else: - exec(wrap2) -del wrap2, wrap3 def _enclose(object, alias=''): #FIXME: needs alias to hold returned object @@ -588,10 +561,7 @@ def dumpsource(object, alias='', new=False, enclose=True): else: #XXX: other cases where source code is needed??? code += getsource(object.__class__, alias='', lstrip=True, force=True) mod = repr(object.__module__) # should have a module (no builtins here) - if PY3: - code += pre + 'dill.loads(%s.replace(b%s,bytes(__name__,"UTF-8")))\n' % (pik,mod) - else: - code += pre + 'dill.loads(%s.replace(%s,__name__))\n' % (pik,mod) + code += pre + 'dill.loads(%s.replace(b%s,bytes(__name__,"UTF-8")))\n' % (pik,mod) #code += 'del %s' % object.__class__.__name__ #NOTE: kills any existing! if enclose: @@ -654,7 +624,7 @@ def _namespace(obj): if module in ['builtins','__builtin__']: # BuiltinFunctionType if _intypes(name): return ['types'] + [name] return qual + [name] #XXX: can be wrong for some aliased objects - except: pass + except Exception: pass # special case: numpy.inf and numpy.nan (we don't want them as floats) if str(obj) in ['inf','nan','Inf','NaN']: # is more, but are they needed? return ['numpy'] + [str(obj)] @@ -742,7 +712,7 @@ def getimport(obj, alias='', verify=True, builtin=False, enclosing=False): try: # look for '<...>' and be mindful it might be in lists, dicts, etc... name = repr(obj).split('<',1)[1].split('>',1)[1] name = None # we have a 'object'-style repr - except: # it's probably something 'importable' + except Exception: # it's probably something 'importable' if head in ['builtins','__builtin__']: name = repr(obj) #XXX: catch [1,2], (1,2), set([1,2])... others? else: @@ -800,7 +770,7 @@ def _importable(obj, alias='', source=None, enclosing=False, force=True, \ try: return getsource(obj, alias, enclosing=enclosing, \ force=force, lstrip=lstrip, builtin=builtin) - except: pass + except Exception: pass try: if not _isinstance(obj): return getimport(obj, alias, enclosing=enclosing, \ @@ -815,12 +785,12 @@ def _importable(obj, alias='', source=None, enclosing=False, force=True, \ if alias == name: _alias = "" return _import+_alias+"%s\n" % name - except: pass + except Exception: pass if not source: # try getsource, only if it hasn't been tried yet try: return getsource(obj, alias, enclosing=enclosing, \ force=force, lstrip=lstrip, builtin=builtin) - except: pass + except Exception: pass # get the name (of functions, lambdas, and classes) # or hope that obj can be built from the __repr__ #XXX: what to do about class instances and such? @@ -960,7 +930,7 @@ def importable(obj, alias='', source=None, builtin=True): if len(src) > 1: raise NotImplementedError('not implemented') return list(src.values())[0] - except: + except Exception: if tried_source: raise tried_import = True # we want the source @@ -999,7 +969,7 @@ def _code_stitcher(block): if not obj: return src if not src: return obj return obj + src - except: + except Exception: if tried_import: raise tried_source = True source = not source diff --git a/dill/temp.py b/dill/temp.py index 251a8e30..215ba63a 100644 --- a/dill/temp.py +++ b/dill/temp.py @@ -17,7 +17,6 @@ 'capture'] import contextlib -from ._dill import PY3 @contextlib.contextmanager @@ -32,10 +31,7 @@ def capture(stream='stdout'): """ import sys - if PY3: - from io import StringIO - else: - from StringIO import StringIO + from io import StringIO orig = getattr(sys, stream) setattr(sys, stream, StringIO()) try: @@ -176,10 +172,7 @@ def loadIO(buffer, **kwds): [1, 2, 3, 4, 5] """ import dill as pickle - if PY3: - from io import BytesIO as StringIO - else: - from StringIO import StringIO + from io import BytesIO as StringIO value = getattr(buffer, 'getvalue', buffer) # value or buffer.getvalue if value != buffer: value = value() # buffer.getvalue() return pickle.load(StringIO(value)) @@ -193,10 +186,7 @@ def dumpIO(object, **kwds): [1, 2, 3, 4, 5] """ import dill as pickle - if PY3: - from io import BytesIO as StringIO - else: - from StringIO import StringIO + from io import BytesIO as StringIO file = StringIO() pickle.dump(object, file) file.flush() @@ -217,7 +207,7 @@ def loadIO_source(buffer, **kwds): alias = kwds.pop('alias', None) source = getattr(buffer, 'getvalue', buffer) # source or buffer.getvalue if source != buffer: source = source() # buffer.getvalue() - if PY3: source = source.decode() # buffer to string + source = source.decode() # buffer to string if not alias: tag = source.strip().splitlines()[-1].split() if tag[0] != '#NAME:': @@ -243,10 +233,7 @@ def dumpIO_source(object, **kwds): If 'alias' is specified, the object will be renamed to the given string. """ from .source import importable, getname - if PY3: - from io import BytesIO as StringIO - else: - from StringIO import StringIO + from io import BytesIO as StringIO alias = kwds.pop('alias', '') #XXX: include an alias so a name is known name = str(alias) or getname(object) name = "\n#NAME: %s\n" % name diff --git a/docs/source/conf.py b/docs/source/conf.py index 19171caf..cbf7cf14 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -58,10 +58,10 @@ master_doc = 'index' # General information about the project. -project = u'dill' +project = 'dill' year = datetime.now().year -copyright = u'%d, The Uncertainty Quantification Foundation' % year -author = u'Mike McKerns' +copyright = '%d, The Uncertainty Quantification Foundation' % year +author = 'Mike McKerns' # extension config github_project_url = "https://github.com/uqfoundation/dill" @@ -191,8 +191,8 @@ # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'dill.tex', u'dill Documentation', - u'Mike McKerns', 'manual'), + (master_doc, 'dill.tex', 'dill Documentation', + 'Mike McKerns', 'manual'), ] @@ -201,7 +201,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - (master_doc, 'dill', u'dill Documentation', + (master_doc, 'dill', 'dill Documentation', [author], 1) ] @@ -212,7 +212,7 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'dill', u'dill Documentation', + (master_doc, 'dill', 'dill Documentation', author, 'dill', 'Serialize all of python.', 'Miscellaneous'), ] diff --git a/tests/__main__.py b/tests/__main__.py index 7f7be536..1570b399 100644 --- a/tests/__main__.py +++ b/tests/__main__.py @@ -27,4 +27,4 @@ p = sp.Popen([python, test], shell=shell).wait() if not p: print('.', end='', flush=True) - print('') + print() diff --git a/tests/test_check.py b/tests/test_check.py index 3b3fc590..0a22b276 100644 --- a/tests/test_check.py +++ b/tests/test_check.py @@ -10,7 +10,6 @@ import sys from dill.temp import capture -from dill._dill import PY3 #FIXME: this doesn't catch output... it's from the internal call diff --git a/tests/test_classdef.py b/tests/test_classdef.py index 47f0d00f..8edf5daf 100644 --- a/tests/test_classdef.py +++ b/tests/test_classdef.py @@ -90,19 +90,15 @@ def test_specialtypes(): assert dill.pickles(type(NotImplemented)) assert dill.pickles(type(Ellipsis)) -if hex(sys.hexversion) >= '0x20600f0': - from collections import namedtuple - Z = namedtuple("Z", ['a','b']) - Zi = Z(0,1) - X = namedtuple("Y", ['a','b']) - X.__name__ = "X" - if hex(sys.hexversion) >= '0x30300f0': - X.__qualname__ = "X" #XXX: name must 'match' or fails to pickle - Xi = X(0,1) - Bad = namedtuple("FakeName", ['a','b']) - Badi = Bad(0,1) -else: - Z = Zi = X = Xi = Bad = Badi = None +from collections import namedtuple +Z = namedtuple("Z", ['a','b']) +Zi = Z(0,1) +X = namedtuple("Y", ['a','b']) +X.__name__ = "X" +X.__qualname__ = "X" #XXX: name must 'match' or fails to pickle +Xi = X(0,1) +Bad = namedtuple("FakeName", ['a','b']) +Badi = Bad(0,1) # test namedtuple def test_namedtuple(): @@ -123,8 +119,7 @@ class B(namedtuple("B", ["one", "two"])): assert dill.copy(a) assert dill.copy(A.B).__name__ == 'B' - if dill._dill.PY3: - assert dill.copy(A.B).__qualname__.endswith('..A.B') + assert dill.copy(A.B).__qualname__.endswith('..A.B') assert dill.copy(A.B).__doc__ == 'docstring' assert dill.copy(A.B).__module__ == 'testing' @@ -168,12 +163,12 @@ def __getnewargs__(self): return np.asarray(self), self.color a1 = TestArray(np.zeros(100), color='green') - if dill._dill.PY3 and not dill._dill.IS_PYPY: + if not dill._dill.IS_PYPY: assert dill.pickles(a1) assert a1.__dict__ == dill.copy(a1).__dict__ a2 = a1[0:9] - if dill._dill.PY3 and not dill._dill.IS_PYPY: + if not dill._dill.IS_PYPY: assert dill.pickles(a2) assert a2.__dict__ == dill.copy(a2).__dict__ @@ -182,7 +177,7 @@ class TestArray2(np.ndarray): a3 = TestArray2([1,2,3,4,5]) a3.color = 'green' - if dill._dill.PY3 and not dill._dill.IS_PYPY: + if not dill._dill.IS_PYPY: assert dill.pickles(a3) assert a3.__dict__ == dill.copy(a3).__dict__ @@ -226,37 +221,21 @@ class A: assert dill.copy(v) == v def test_metaclass(): - if dill._dill.PY3: - class metaclass_with_new(type): - def __new__(mcls, name, bases, ns, **kwds): - cls = super().__new__(mcls, name, bases, ns, **kwds) - assert mcls is not None - assert cls.method(mcls) - return cls - def method(cls, mcls): - return isinstance(cls, mcls) - - l = locals() - exec("""class subclass_with_new(metaclass=metaclass_with_new): - def __new__(cls): - self = super().__new__(cls) - return self""", None, l) - subclass_with_new = l['subclass_with_new'] - else: - class metaclass_with_new(type): - def __new__(mcls, name, bases, ns, **kwds): - cls = super(mcls, metaclass_with_new).__new__(mcls, name, bases, ns, **kwds) - assert mcls is not None - assert cls.method(mcls) - return cls - def method(cls, mcls): - return isinstance(cls, mcls) - - class subclass_with_new: - __metaclass__ = metaclass_with_new - def __new__(cls): - self = super(subclass_with_new, cls).__new__(cls) - return self + class metaclass_with_new(type): + def __new__(mcls, name, bases, ns, **kwds): + cls = super().__new__(mcls, name, bases, ns, **kwds) + assert mcls is not None + assert cls.method(mcls) + return cls + def method(cls, mcls): + return isinstance(cls, mcls) + + l = locals() + exec("""class subclass_with_new(metaclass=metaclass_with_new): + def __new__(cls): + self = super().__new__(cls) + return self""", None, l) + subclass_with_new = l['subclass_with_new'] assert dill.copy(subclass_with_new()) diff --git a/tests/test_detect.py b/tests/test_detect.py index 06339acd..4dac7aaf 100644 --- a/tests/test_detect.py +++ b/tests/test_detect.py @@ -8,7 +8,7 @@ from dill.detect import baditems, badobjects, badtypes, errors, parent, at, globalvars from dill import settings -from dill._dill import IS_PYPY, IS_PYPY2 +from dill._dill import IS_PYPY from pickle import PicklingError import inspect @@ -21,7 +21,7 @@ def test_bad_things(): #assert baditems(globals()) == [f] #XXX assert badobjects(f) is f assert badtypes(f) == type(f) - assert type(errors(f)) is PicklingError if IS_PYPY2 else TypeError + assert type(errors(f)) is TypeError d = badtypes(f, 1) assert isinstance(d, dict) assert list(badobjects(f, 1).keys()) == list(d.keys()) @@ -30,7 +30,7 @@ def test_bad_things(): a = dict(s) if not os.environ.get('COVERAGE'): #XXX: travis-ci assert len(s) is len(a) # TypeError (and possibly PicklingError) - n = 1 if IS_PYPY2 else 2 + n = 2 assert len(a) is n if 'PicklingError' in a.keys() else n-1 def test_parent(): diff --git a/tests/test_diff.py b/tests/test_diff.py index 3277682b..652d819b 100644 --- a/tests/test_diff.py +++ b/tests/test_diff.py @@ -55,18 +55,15 @@ def test_diff(): assert changed[1] if not IS_PYPY: - try: - import abc - # make sure the "_abc_invaldation_counter" doesn't make test fail - diff.memorise(abc.ABCMeta, force=True) - assert not diff.has_changed(abc) - abc.ABCMeta.zzz = 1 - assert diff.has_changed(abc) - changed = diff.whats_changed(abc) - assert list(changed[0].keys()) == ["ABCMeta"] - assert not changed[1] - except ImportError: - pass + import abc + # make sure the "_abc_invaldation_counter" doesn't make test fail + diff.memorise(abc.ABCMeta, force=True) + assert not diff.has_changed(abc) + abc.ABCMeta.zzz = 1 + assert diff.has_changed(abc) + changed = diff.whats_changed(abc) + assert list(changed[0].keys()) == ["ABCMeta"] + assert not changed[1] ''' import Queue diff --git a/tests/test_extendpickle.py b/tests/test_extendpickle.py index 49b6f152..10dc0804 100644 --- a/tests/test_extendpickle.py +++ b/tests/test_extendpickle.py @@ -7,10 +7,7 @@ # - https://github.com/uqfoundation/dill/blob/master/LICENSE import dill as pickle -try: - from StringIO import StringIO -except ImportError: - from io import BytesIO as StringIO +from io import BytesIO as StringIO def my_fn(x): @@ -47,7 +44,7 @@ def test_isdill(): pickler = mp.reduction.ForkingPickler(obj_io) assert pickle._dill.is_dill(pickler, child=True) is True assert pickle._dill.is_dill(pickler, child=False) is False - except: + except Exception: pass diff --git a/tests/test_file.py b/tests/test_file.py index 8118e3a8..b766552e 100644 --- a/tests/test_file.py +++ b/tests/test_file.py @@ -19,8 +19,6 @@ fname = "_test_file.txt" rand_chars = list(string.ascii_letters) + ["\n"] * 40 # bias newline -if sys.hexversion < 0x03030000: - FileNotFoundError = IOError buffer_error = ValueError("invalid buffer size") dne_error = FileNotFoundError("[Errno 2] No such file or directory: '%s'" % fname) diff --git a/tests/test_functions.py b/tests/test_functions.py index a86cbde0..d82c37e3 100644 --- a/tests/test_functions.py +++ b/tests/test_functions.py @@ -11,10 +11,6 @@ dill.settings['recurse'] = True -def is_py3(): - return hex(sys.hexversion) >= '0x30000f0' - - def function_a(a): return a @@ -34,18 +30,17 @@ def function_d(d, d1, d2=1): function_d.__module__ = 'a module' -if is_py3(): - exec(''' +exec(''' def function_e(e, *e1, e2=1, e3=2): return e + sum(e1) + e2 + e3''') - globalvar = 0 +globalvar = 0 - @functools.lru_cache(None) - def function_with_cache(x): - global globalvar - globalvar += x - return globalvar +@functools.lru_cache(None) +def function_with_cache(x): + global globalvar + globalvar += x + return globalvar def function_with_unassigned_variable(): @@ -87,28 +82,26 @@ def test_functions(): assert dill.loads(dumped_func_d)(1, 2, 3) == 6 assert dill.loads(dumped_func_d)(1, 2, d2=3) == 6 - if is_py3(): - function_with_cache(1) - globalvar = 0 - dumped_func_cache = dill.dumps(function_with_cache) - assert function_with_cache(2) == 3 - assert function_with_cache(1) == 1 - assert function_with_cache(3) == 6 - assert function_with_cache(2) == 3 + function_with_cache(1) + globalvar = 0 + dumped_func_cache = dill.dumps(function_with_cache) + assert function_with_cache(2) == 3 + assert function_with_cache(1) == 1 + assert function_with_cache(3) == 6 + assert function_with_cache(2) == 3 empty_cell = function_with_unassigned_variable() cell_copy = dill.loads(dill.dumps(empty_cell)) assert 'empty' in str(cell_copy.__closure__[0]) try: cell_copy() - except: + except Exception: # this is good pass else: raise AssertionError('cell_copy() did not read an empty cell') - if is_py3(): - exec(''' + exec(''' dumped_func_e = dill.dumps(function_e) assert dill.loads(dumped_func_e)(1, 2) == 6 assert dill.loads(dumped_func_e)(1, 2, 3) == 9 diff --git a/tests/test_module.py b/tests/test_module.py index 5c5e000c..2d009f26 100644 --- a/tests/test_module.py +++ b/tests/test_module.py @@ -9,8 +9,7 @@ import sys import dill import test_mixins as module -try: from importlib import reload -except ImportError: pass +from importlib import reload dill.settings['recurse'] = True cached = (module.__cached__ if hasattr(module, "__cached__") diff --git a/tests/test_nested.py b/tests/test_nested.py index 144f54b0..950641ca 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -110,7 +110,7 @@ def test_pickled_inner(): def test_moduledict_where_not_main(): try: from . import test_moduledict - except: + except ImportError: import test_moduledict name = 'test_moduledict.py' if os.path.exists(name) and os.path.exists(name+'c'): diff --git a/tests/test_recursive.py b/tests/test_recursive.py index ee71a688..0f3a53a1 100644 --- a/tests/test_recursive.py +++ b/tests/test_recursive.py @@ -6,7 +6,6 @@ # - https://github.com/uqfoundation/dill/blob/master/LICENSE import dill -from dill._dill import PY3 from functools import partial import warnings @@ -15,7 +14,7 @@ def copy(obj, byref=False, recurse=False): if byref: try: return dill.copy(obj, byref=byref, recurse=recurse) - except: + except Exception: pass else: raise AssertionError('Copy of %s with byref=True should have given a warning!' % (obj,)) @@ -113,9 +112,8 @@ def __init__(self): def test_circular_reference(): assert copy(obj4()) obj4_copy = dill.loads(dill.dumps(obj4())) - if PY3: - assert type(obj4_copy) is type(obj4_copy).__init__.__closure__[0].cell_contents - assert type(obj4_copy.b) is type(obj4_copy.b).__init__.__closure__[0].cell_contents + assert type(obj4_copy) is type(obj4_copy).__init__.__closure__[0].cell_contents + assert type(obj4_copy.b) is type(obj4_copy.b).__init__.__closure__[0].cell_contents def f(): @@ -146,7 +144,7 @@ def test_recursive_function(): for _fib in (fib3, fib4): try: _fib(5) - except: + except Exception: # This is expected to fail because fib no longer exists pass else: diff --git a/tests/test_selected.py b/tests/test_selected.py index 9eef8d2d..59ff2ce1 100644 --- a/tests/test_selected.py +++ b/tests/test_selected.py @@ -19,7 +19,7 @@ def test_dict_contents(): for i,j in c.items(): #try: ok = dill.pickles(j) - #except: + #except Exception: # print ("FAIL: %s with %s" % (i, dill.detect.errors(j))) if verbose: print ("%s: %s, %s" % (ok, type(j), j)) assert ok @@ -29,7 +29,7 @@ def _g(x): yield x; def _f(): try: raise - except: + except Exception: from sys import exc_info e, er, tb = exc_info() return er, tb @@ -83,7 +83,7 @@ def test_frame_related(): g = _g(1) f = g.gi_frame e,t = _f() - _is = lambda ok: not ok if dill._dill.IS_PYPY2 else ok + _is = lambda ok: ok ok = dill.pickles(f) if verbose: print ("%s: %s, %s" % (ok, type(f), f)) assert not ok diff --git a/tests/test_session.py b/tests/test_session.py index 8d036fb5..689cc975 100644 --- a/tests/test_session.py +++ b/tests/test_session.py @@ -5,8 +5,12 @@ # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/dill/blob/master/LICENSE -from __future__ import print_function -import atexit, dill, os, sys, __main__ +import atexit +import os +import sys +import __main__ + +import dill session_file = os.path.join(os.path.dirname(__file__), 'session-byref-%s.pkl') @@ -114,9 +118,8 @@ def test_objects(main, copy_dict, byref): for obj in ('x', 'empty', 'names'): assert main_dict[obj] == copy_dict[obj] - globs = '__globals__' if dill._dill.PY3 else 'func_globals' for obj in ['squared', 'cubed']: - assert getattr(main_dict[obj], globs) is main_dict + assert main_dict[obj].__globals__ is main_dict assert main_dict[obj](3) == copy_dict[obj](3) assert main.Person.__module__ == main.__name__ diff --git a/tests/test_source.py b/tests/test_source.py index 57d25d12..01fc1eb3 100644 --- a/tests/test_source.py +++ b/tests/test_source.py @@ -11,9 +11,7 @@ from dill._dill import IS_PYPY import sys -PY3 = sys.version_info[0] >= 3 -IS_PYPY3 = IS_PYPY and PY3 -PY310b = '0x30a00b1' +PY310b = 0x30a00b1 f = lambda x: x**2 def g(x): return f(x) - x @@ -61,14 +59,12 @@ def test_itself(): # builtin functions and objects def test_builtin(): - if PY3: builtin = 'builtins' - else: builtin = '__builtin__' assert likely_import(pow) == 'pow\n' assert likely_import(100) == '100\n' assert likely_import(True) == 'True\n' - assert likely_import(pow, explicit=True) == 'from %s import pow\n' % builtin + assert likely_import(pow, explicit=True) == 'from builtins import pow\n' assert likely_import(100, explicit=True) == '100\n' - assert likely_import(True, explicit=True) == 'True\n' if PY3 else 'from %s import True\n' % builtin + assert likely_import(True, explicit=True) == 'True\n' # this is kinda BS... you can't import a None assert likely_import(None) == 'None\n' assert likely_import(None, explicit=True) == 'None\n' @@ -87,14 +83,9 @@ def test_dynamic(): # classes and class instances def test_classes(): - try: #XXX: should this be a 'special case'? - from StringIO import StringIO - y = "from StringIO import StringIO\n" - x = y - except ImportError: - from io import BytesIO as StringIO - y = "from _io import BytesIO\n" - x = y if (IS_PYPY3 or hex(sys.hexversion) >= PY310b) else "from io import BytesIO\n" + from io import BytesIO as StringIO + y = "from _io import BytesIO\n" + x = y if (IS_PYPY or sys.hexversion >= PY310b) else "from io import BytesIO\n" s = StringIO() assert likely_import(StringIO) == x diff --git a/tests/test_weakref.py b/tests/test_weakref.py index ada7d140..0e99f3ea 100644 --- a/tests/test_weakref.py +++ b/tests/test_weakref.py @@ -76,7 +76,7 @@ def test_dictproxy(): from dill._dill import DictProxyType try: m = DictProxyType({"foo": "bar"}) - except: + except Exception: m = type.__dict__ mp = dill.copy(m) assert mp.items() == m.items() From 3881a2b2109ed206fa5a081758d77902602b491e Mon Sep 17 00:00:00 2001 From: mmckerns Date: Sat, 2 Jul 2022 14:27:32 -0400 Subject: [PATCH 13/13] fix #521: support develop mode --- .travis.yml | 4 ++-- MANIFEST.in | 1 - {tests => dill/tests}/__init__.py | 0 {tests => dill/tests}/__main__.py | 0 {tests => dill/tests}/test_check.py | 0 {tests => dill/tests}/test_classdef.py | 0 {tests => dill/tests}/test_detect.py | 0 {tests => dill/tests}/test_dictviews.py | 0 {tests => dill/tests}/test_diff.py | 0 {tests => dill/tests}/test_extendpickle.py | 0 {tests => dill/tests}/test_fglobals.py | 0 {tests => dill/tests}/test_file.py | 0 {tests => dill/tests}/test_functions.py | 0 {tests => dill/tests}/test_functors.py | 0 {tests => dill/tests}/test_logger.py | 0 {tests => dill/tests}/test_mixins.py | 0 {tests => dill/tests}/test_module.py | 0 {tests => dill/tests}/test_moduledict.py | 0 {tests => dill/tests}/test_nested.py | 0 {tests => dill/tests}/test_objects.py | 0 {tests => dill/tests}/test_properties.py | 0 {tests => dill/tests}/test_pycapsule.py | 0 {tests => dill/tests}/test_recursive.py | 0 {tests => dill/tests}/test_restricted.py | 0 {tests => dill/tests}/test_selected.py | 0 {tests => dill/tests}/test_session.py | 0 {tests => dill/tests}/test_source.py | 0 {tests => dill/tests}/test_temp.py | 0 {tests => dill/tests}/test_weakref.py | 0 setup.py | 2 +- tox.ini | 2 +- 31 files changed, 4 insertions(+), 5 deletions(-) rename {tests => dill/tests}/__init__.py (100%) rename {tests => dill/tests}/__main__.py (100%) rename {tests => dill/tests}/test_check.py (100%) rename {tests => dill/tests}/test_classdef.py (100%) rename {tests => dill/tests}/test_detect.py (100%) rename {tests => dill/tests}/test_dictviews.py (100%) rename {tests => dill/tests}/test_diff.py (100%) rename {tests => dill/tests}/test_extendpickle.py (100%) rename {tests => dill/tests}/test_fglobals.py (100%) rename {tests => dill/tests}/test_file.py (100%) rename {tests => dill/tests}/test_functions.py (100%) rename {tests => dill/tests}/test_functors.py (100%) rename {tests => dill/tests}/test_logger.py (100%) rename {tests => dill/tests}/test_mixins.py (100%) rename {tests => dill/tests}/test_module.py (100%) rename {tests => dill/tests}/test_moduledict.py (100%) rename {tests => dill/tests}/test_nested.py (100%) rename {tests => dill/tests}/test_objects.py (100%) rename {tests => dill/tests}/test_properties.py (100%) rename {tests => dill/tests}/test_pycapsule.py (100%) rename {tests => dill/tests}/test_recursive.py (100%) rename {tests => dill/tests}/test_restricted.py (100%) rename {tests => dill/tests}/test_selected.py (100%) rename {tests => dill/tests}/test_session.py (100%) rename {tests => dill/tests}/test_source.py (100%) rename {tests => dill/tests}/test_temp.py (100%) rename {tests => dill/tests}/test_weakref.py (100%) diff --git a/.travis.yml b/.travis.yml index 4fbe2282..423c3a2e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,8 +40,8 @@ install: - python -m pip install . script: - - for test in tests/__init__.py; do echo $test ; if [[ $COVERAGE == "true" ]]; then coverage run -a $test > /dev/null; else python $test > /dev/null; fi ; done - - for test in tests/test_*.py; do echo $test ; if [[ $COVERAGE == "true" ]]; then coverage run -a $test > /dev/null; else python $test > /dev/null; fi ; done + - for test in dill/tests/__init__.py; do echo $test ; if [[ $COVERAGE == "true" ]]; then coverage run -a $test > /dev/null; else python $test > /dev/null; fi ; done + - for test in dill/tests/test_*.py; do echo $test ; if [[ $COVERAGE == "true" ]]; then coverage run -a $test > /dev/null; else python $test > /dev/null; fi ; done after_success: - if [[ $COVERAGE == "true" ]]; then bash <(curl -s https://codecov.io/bash); else echo ''; fi diff --git a/MANIFEST.in b/MANIFEST.in index 0cd47401..76ad6227 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,7 +4,6 @@ include MANIFEST.in include pyproject.toml include tox.ini include scripts/* -include tests/*py recursive-include docs * include .* prune .git diff --git a/tests/__init__.py b/dill/tests/__init__.py similarity index 100% rename from tests/__init__.py rename to dill/tests/__init__.py diff --git a/tests/__main__.py b/dill/tests/__main__.py similarity index 100% rename from tests/__main__.py rename to dill/tests/__main__.py diff --git a/tests/test_check.py b/dill/tests/test_check.py similarity index 100% rename from tests/test_check.py rename to dill/tests/test_check.py diff --git a/tests/test_classdef.py b/dill/tests/test_classdef.py similarity index 100% rename from tests/test_classdef.py rename to dill/tests/test_classdef.py diff --git a/tests/test_detect.py b/dill/tests/test_detect.py similarity index 100% rename from tests/test_detect.py rename to dill/tests/test_detect.py diff --git a/tests/test_dictviews.py b/dill/tests/test_dictviews.py similarity index 100% rename from tests/test_dictviews.py rename to dill/tests/test_dictviews.py diff --git a/tests/test_diff.py b/dill/tests/test_diff.py similarity index 100% rename from tests/test_diff.py rename to dill/tests/test_diff.py diff --git a/tests/test_extendpickle.py b/dill/tests/test_extendpickle.py similarity index 100% rename from tests/test_extendpickle.py rename to dill/tests/test_extendpickle.py diff --git a/tests/test_fglobals.py b/dill/tests/test_fglobals.py similarity index 100% rename from tests/test_fglobals.py rename to dill/tests/test_fglobals.py diff --git a/tests/test_file.py b/dill/tests/test_file.py similarity index 100% rename from tests/test_file.py rename to dill/tests/test_file.py diff --git a/tests/test_functions.py b/dill/tests/test_functions.py similarity index 100% rename from tests/test_functions.py rename to dill/tests/test_functions.py diff --git a/tests/test_functors.py b/dill/tests/test_functors.py similarity index 100% rename from tests/test_functors.py rename to dill/tests/test_functors.py diff --git a/tests/test_logger.py b/dill/tests/test_logger.py similarity index 100% rename from tests/test_logger.py rename to dill/tests/test_logger.py diff --git a/tests/test_mixins.py b/dill/tests/test_mixins.py similarity index 100% rename from tests/test_mixins.py rename to dill/tests/test_mixins.py diff --git a/tests/test_module.py b/dill/tests/test_module.py similarity index 100% rename from tests/test_module.py rename to dill/tests/test_module.py diff --git a/tests/test_moduledict.py b/dill/tests/test_moduledict.py similarity index 100% rename from tests/test_moduledict.py rename to dill/tests/test_moduledict.py diff --git a/tests/test_nested.py b/dill/tests/test_nested.py similarity index 100% rename from tests/test_nested.py rename to dill/tests/test_nested.py diff --git a/tests/test_objects.py b/dill/tests/test_objects.py similarity index 100% rename from tests/test_objects.py rename to dill/tests/test_objects.py diff --git a/tests/test_properties.py b/dill/tests/test_properties.py similarity index 100% rename from tests/test_properties.py rename to dill/tests/test_properties.py diff --git a/tests/test_pycapsule.py b/dill/tests/test_pycapsule.py similarity index 100% rename from tests/test_pycapsule.py rename to dill/tests/test_pycapsule.py diff --git a/tests/test_recursive.py b/dill/tests/test_recursive.py similarity index 100% rename from tests/test_recursive.py rename to dill/tests/test_recursive.py diff --git a/tests/test_restricted.py b/dill/tests/test_restricted.py similarity index 100% rename from tests/test_restricted.py rename to dill/tests/test_restricted.py diff --git a/tests/test_selected.py b/dill/tests/test_selected.py similarity index 100% rename from tests/test_selected.py rename to dill/tests/test_selected.py diff --git a/tests/test_session.py b/dill/tests/test_session.py similarity index 100% rename from tests/test_session.py rename to dill/tests/test_session.py diff --git a/tests/test_source.py b/dill/tests/test_source.py similarity index 100% rename from tests/test_source.py rename to dill/tests/test_source.py diff --git a/tests/test_temp.py b/dill/tests/test_temp.py similarity index 100% rename from tests/test_temp.py rename to dill/tests/test_temp.py diff --git a/tests/test_weakref.py b/dill/tests/test_weakref.py similarity index 100% rename from tests/test_weakref.py rename to dill/tests/test_weakref.py diff --git a/setup.py b/setup.py index 0d3e76cb..c0716c2a 100644 --- a/setup.py +++ b/setup.py @@ -97,7 +97,7 @@ 'Topic :: Software Development', ], packages = ['dill','dill.tests'], - package_dir = {'dill':'dill', 'dill.tests':'tests'}, + package_dir = {'dill':'dill', 'dill.tests':'dill/tests'}, scripts=['scripts/undill','scripts/get_objgraph'], ) diff --git a/tox.ini b/tox.ini index 55b8f077..a63d8067 100644 --- a/tox.ini +++ b/tox.ini @@ -18,5 +18,5 @@ whitelist_externals = bash commands = {envpython} -m pip install . - bash -c "failed=0; for test in tests/__main__.py; do echo $test; \ + bash -c "failed=0; for test in dill/tests/__main__.py; do echo $test; \ {envpython} $test || failed=1; done; exit $failed"