Merge pull request #55354 from Akm0d/msgpack

Use packaged msgpack

salt/cloud/clouds/ec2.py

@@ -92,7 +92,6 @@
 import binascii
 import datetime
 import base64
-import msgpack
 import re
 import decimal
 
@@ -102,6 +101,7 @@
 import salt.utils.files
 import salt.utils.hashutils
 import salt.utils.json
+import salt.utils.msgpack
 import salt.utils.stringutils
 import salt.utils.yaml
 from salt._compat import ElementTree as ET
@@ -5000,7 +5000,7 @@ def _parse_pricing(url, name):
         __opts__['cachedir'], 'ec2-pricing-{0}.p'.format(name)
     )
     with salt.utils.files.fopen(outfile, 'w') as fho:
-        msgpack.dump(regions, fho)
+        salt.utils.msgpack.dump(regions, fho)
 
     return True
 
@@ -5068,7 +5068,8 @@ def show_pricing(kwargs=None, call=None):
         update_pricing({'type': name}, 'function')
 
     with salt.utils.files.fopen(pricefile, 'r') as fhi:
-        ec2_price = salt.utils.stringutils.to_unicode(msgpack.load(fhi))
+        ec2_price = salt.utils.stringutils.to_unicode(
+            salt.utils.msgpack.load(fhi))
 
     region = get_location(profile)
     size = profile.get('size', None)

salt/cloud/clouds/gce.py

@@ -53,7 +53,6 @@
 import re
 import pprint
 import logging
-import msgpack
 from ast import literal_eval
 from salt.utils.versions import LooseVersion as _LooseVersion
 
@@ -91,6 +90,7 @@
 import salt.utils.cloud
 import salt.utils.files
 import salt.utils.http
+import salt.utils.msgpack
 import salt.config as config
 from salt.cloud.libcloudfuncs import *  # pylint: disable=redefined-builtin,wildcard-import,unused-wildcard-import
 from salt.exceptions import (
@@ -2629,7 +2629,7 @@ def update_pricing(kwargs=None, call=None):
         __opts__['cachedir'], 'gce-pricing.p'
     )
     with salt.utils.files.fopen(outfile, 'w') as fho:
-        msgpack.dump(price_json['dict'], fho)
+        salt.utils.msgpack.dump(price_json['dict'], fho)
 
     return True
 
@@ -2668,7 +2668,7 @@ def show_pricing(kwargs=None, call=None):
         update_pricing()
 
     with salt.utils.files.fopen(pricefile, 'r') as fho:
-        sizes = msgpack.load(fho)
+        sizes = salt.utils.msgpack.load(fho)
 
     per_hour = float(sizes['gcp_price_list'][size][region])
 

salt/engines/stalekey.py

@@ -28,11 +28,11 @@
 import salt.key
 import salt.utils.files
 import salt.utils.minions
+import salt.utils.msgpack
 import salt.wheel
 
 # Import 3rd-party libs
 from salt.ext import six
-import msgpack
 
 log = logging.getLogger(__name__)
 
@@ -60,7 +60,7 @@ def start(interval=3600, expire=604800):
         if os.path.exists(presence_file):
             try:
                 with salt.utils.files.fopen(presence_file, 'r') as f:
-                    minions = msgpack.load(f)
+                    minions = salt.utils.msgpack.load(f)
             except IOError as e:
                 log.error('Could not open presence file %s: %s', presence_file, e)
                 time.sleep(interval)
@@ -95,7 +95,7 @@ def start(interval=3600, expire=604800):
 
         try:
             with salt.utils.files.fopen(presence_file, 'w') as f:
-                msgpack.dump(minions, f)
+                salt.utils.msgpack.dump(minions, f)
         except IOError as e:
             log.error('Could not write to presence file %s: %s', presence_file, e)
         time.sleep(interval)

salt/log/handlers/fluent_mod.py

@@ -86,33 +86,14 @@
 # Import salt libs
 from salt.log.setup import LOG_LEVELS
 from salt.log.mixins import NewStyleClassMixIn
+import salt.utils.msgpack
 import salt.utils.network
 
 # Import Third party libs
 from salt.ext import six
 
 log = logging.getLogger(__name__)
 
-try:
-    # Attempt to import msgpack
-    import msgpack
-    # There is a serialization issue on ARM and potentially other platforms
-    # for some msgpack bindings, check for it
-    if msgpack.loads(msgpack.dumps([1, 2, 3]), use_list=True) is None:
-        raise ImportError
-except ImportError:
-    # Fall back to msgpack_pure
-    try:
-        import msgpack_pure as msgpack
-    except ImportError:
-        # TODO: Come up with a sane way to get a configured logfile
-        #       and write to the logfile when this error is hit also
-        LOG_FORMAT = '[%(levelname)-8s] %(message)s'
-        salt.log.setup_console_logger(log_format=LOG_FORMAT)
-        log.fatal('Unable to import msgpack or msgpack_pure python modules')
-        # Don't exit if msgpack is not available, this is to make local mode
-        # work without msgpack
-        #sys.exit(salt.exitcodes.EX_GENERIC)
 
 # Define the module's virtual name
 __virtualname__ = 'fluent'
@@ -455,7 +436,7 @@ def _make_packet(self, label, timestamp, data):
         packet = (tag, timestamp, data)
         if self.verbose:
             print(packet)
-        return msgpack.packb(packet)
+        return salt.utils.msgpack.packb(packet)
 
     def _send(self, bytes_):
         self.lock.acquire()

salt/modules/state.py

@@ -33,6 +33,7 @@
 import salt.utils.hashutils
 import salt.utils.jid
 import salt.utils.json
+import salt.utils.msgpack
 import salt.utils.platform
 import salt.utils.state
 import salt.utils.stringutils
@@ -45,7 +46,6 @@
 
 # Import 3rd-party libs
 from salt.ext import six
-import msgpack
 
 __proxyenabled__ = ['*']
 
@@ -185,7 +185,7 @@ def _get_pause(jid, state_id=None):
             data[state_id] = {}
     if os.path.exists(pause_path):
         with salt.utils.files.fopen(pause_path, 'rb') as fp_:
-            data = msgpack.loads(fp_.read())
+            data = salt.utils.msgpack.loads(fp_.read())
     return data, pause_path
 
 
@@ -256,7 +256,7 @@ def soft_kill(jid, state_id=None):
     data, pause_path = _get_pause(jid, state_id)
     data[state_id]['kill'] = True
     with salt.utils.files.fopen(pause_path, 'wb') as fp_:
-        fp_.write(msgpack.dumps(data))
+        fp_.write(salt.utils.msgpack.dumps(data))
 
 
 def pause(jid, state_id=None, duration=None):
@@ -291,7 +291,7 @@ def pause(jid, state_id=None, duration=None):
     if duration:
         data[state_id]['duration'] = int(duration)
     with salt.utils.files.fopen(pause_path, 'wb') as fp_:
-        fp_.write(msgpack.dumps(data))
+        fp_.write(salt.utils.msgpack.dumps(data))
 
 
 def resume(jid, state_id=None):
@@ -325,7 +325,7 @@ def resume(jid, state_id=None):
     if state_id == '__all__':
         data = {}
     with salt.utils.files.fopen(pause_path, 'wb') as fp_:
-        fp_.write(msgpack.dumps(data))
+        fp_.write(salt.utils.msgpack.dumps(data))
 
 
 def orchestrate(mods,

salt/modules/winrepo.py

@@ -32,11 +32,8 @@
     GLOBAL_ONLY
 )
 from salt.ext import six
-try:
-    import msgpack
-except ImportError:
-    import msgpack_pure as msgpack  # pylint: disable=import-error
 import salt.utils.gitfs
+import salt.utils.msgpack as msgpack
 # pylint: enable=unused-import
 
 log = logging.getLogger(__name__)

salt/payload.py

@@ -16,6 +16,7 @@
 import salt.log
 import salt.transport.frame
 import salt.utils.immutabletypes as immutabletypes
+import salt.utils.msgpack
 import salt.utils.stringutils
 from salt.exceptions import SaltReqTimeoutError, SaltDeserializationError
 from salt.utils.data import CaseInsensitiveDict
@@ -30,63 +31,20 @@
 
 log = logging.getLogger(__name__)
 
-HAS_MSGPACK = False
-try:
-    # Attempt to import msgpack
-    import msgpack
-    # There is a serialization issue on ARM and potentially other platforms
-    # for some msgpack bindings, check for it
-    if msgpack.version >= (0, 4, 0):
-        if msgpack.loads(msgpack.dumps([1, 2, 3], use_bin_type=False), use_list=True) is None:
-            raise ImportError
-    else:
-        if msgpack.loads(msgpack.dumps([1, 2, 3]), use_list=True) is None:
-            raise ImportError
-    HAS_MSGPACK = True
-except ImportError:
-    # Fall back to msgpack_pure
-    try:
-        import msgpack_pure as msgpack  # pylint: disable=import-error
-        HAS_MSGPACK = True
-    except ImportError:
-        # TODO: Come up with a sane way to get a configured logfile
-        #       and write to the logfile when this error is hit also
-        LOG_FORMAT = '[%(levelname)-8s] %(message)s'
-        salt.log.setup_console_logger(log_format=LOG_FORMAT)
-        log.fatal('Unable to import msgpack or msgpack_pure python modules')
-        # Don't exit if msgpack is not available, this is to make local mode
-        # work without msgpack
-        #sys.exit(salt.defaults.exitcodes.EX_GENERIC)
-
-
-if HAS_MSGPACK and not hasattr(msgpack, 'exceptions'):
-    class PackValueError(Exception):
-        '''
-        older versions of msgpack do not have PackValueError
-        '''
-
-    class exceptions(object):
-        '''
-        older versions of msgpack do not have an exceptions module
-        '''
-        PackValueError = PackValueError()
-
-    msgpack.exceptions = exceptions()
-
 
 def package(payload):
     '''
     This method for now just wraps msgpack.dumps, but it is here so that
     we can make the serialization a custom option in the future with ease.
     '''
-    return msgpack.dumps(payload)
+    return salt.utils.msgpack.dumps(payload)
 
 
 def unpackage(package_):
     '''
     Unpackages a payload
     '''
-    return msgpack.loads(package_, use_list=True)
+    return salt.utils.msgpack.loads(package_, use_list=True)
 
 
 def format_payload(enc, **kwargs):
@@ -142,27 +100,27 @@ def ext_type_decoder(code, data):
             gc.disable()  # performance optimization for msgpack
             loads_kwargs = {'use_list': True,
                             'ext_hook': ext_type_decoder}
-            if msgpack.version >= (0, 4, 0):
+            if salt.utils.msgpack.version >= (0, 4, 0):
                 # msgpack only supports 'encoding' starting in 0.4.0.
                 # Due to this, if we don't need it, don't pass it at all so
                 # that under Python 2 we can still work with older versions
                 # of msgpack.
-                if msgpack.version >= (0, 5, 2):
+                if salt.utils.msgpack.version >= (0, 5, 2):
                     if encoding is None:
                         loads_kwargs['raw'] = True
                     else:
                         loads_kwargs['raw'] = False
                 else:
                     loads_kwargs['encoding'] = encoding
                 try:
-                    ret = msgpack.loads(msg, **loads_kwargs)
+                    ret = salt.utils.msgpack.unpackb(msg, **loads_kwargs)
                 except UnicodeDecodeError:
                     # msg contains binary data
                     loads_kwargs.pop('raw', None)
                     loads_kwargs.pop('encoding', None)
-                    ret = msgpack.loads(msg, **loads_kwargs)
+                    ret = salt.utils.msgpack.loads(msg, **loads_kwargs)
             else:
-                ret = msgpack.loads(msg, **loads_kwargs)
+                ret = salt.utils.msgpack.loads(msg, **loads_kwargs)
             if six.PY3 and encoding is None and not raw:
                 ret = salt.transport.frame.decode_embedded_strs(ret)
         except Exception as exc:
@@ -216,7 +174,7 @@ def ext_type_encoder(obj):
                 # msgpack doesn't support datetime.datetime and datetime.date datatypes.
                 # So here we have converted these types to custom datatype
                 # This is msgpack Extended types numbered 78
-                return msgpack.ExtType(78, salt.utils.stringutils.to_bytes(
+                return salt.utils.msgpack.ExtType(78, salt.utils.stringutils.to_bytes(
                     obj.strftime('%Y%m%dT%H:%M:%S.%f')))
             # The same for immutable types
             elif isinstance(obj, immutabletypes.ImmutableDict):
@@ -232,15 +190,8 @@ def ext_type_encoder(obj):
             return obj
 
         try:
-            if msgpack.version >= (0, 4, 0):
-                # msgpack only supports 'use_bin_type' starting in 0.4.0.
-                # Due to this, if we don't need it, don't pass it at all so
-                # that under Python 2 we can still work with older versions
-                # of msgpack.
-                return msgpack.dumps(msg, default=ext_type_encoder, use_bin_type=use_bin_type)
-            else:
-                return msgpack.dumps(msg, default=ext_type_encoder)
-        except (OverflowError, msgpack.exceptions.PackValueError):
+            return salt.utils.msgpack.packb(msg, default=ext_type_encoder, use_bin_type=use_bin_type)
+        except (OverflowError, salt.utils.msgpack.exceptions.PackValueError):
             # msgpack<=0.4.6 don't call ext encoder on very long integers raising the error instead.
             # Convert any very long longs to strings and call dumps again.
             def verylong_encoder(obj, context):
@@ -267,10 +218,7 @@ def verylong_encoder(obj, context):
                     return obj
 
             msg = verylong_encoder(msg, set())
-            if msgpack.version >= (0, 4, 0):
-                return msgpack.dumps(msg, default=ext_type_encoder, use_bin_type=use_bin_type)
-            else:
-                return msgpack.dumps(msg, default=ext_type_encoder)
+            return salt.utils.msgpack.packb(msg, default=ext_type_encoder, use_bin_type=use_bin_type)
 
     def dump(self, msg, fn_):
         '''

salt/renderers/msgpack.py

@@ -1,10 +1,8 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import, print_function, unicode_literals
 
-# Import third party libs
-import msgpack
-
 # Import salt libs
+import salt.utils.msgpack
 from salt.ext import six
 
 
@@ -28,4 +26,4 @@ def render(msgpack_data, saltenv='base', sls='', **kws):
         msgpack_data = msgpack_data[(msgpack_data.find('\n') + 1):]
     if not msgpack_data.strip():
         return {}
-    return msgpack.loads(msgpack_data)
+    return salt.utils.msgpack.loads(msgpack_data)