Backport #51677 to master

salt/config/__init__.py

@@ -1169,9 +1169,6 @@ def _gather_buffer_space():
     # Subconfig entries can be specified by using the ':' notation (e.g. key:subkey)
     'pass_to_ext_pillars': (six.string_types, list),
 
-    # Used by salt.modules.dockermod.compare_container_networks to specify which keys are compared
-    'docker.compare_container_networks': dict,
-
     # SSDP discovery publisher description.
     # Contains publisher configuration and minion mapping.
     # Setting it to False disables discovery
@@ -1482,11 +1479,6 @@ def _gather_buffer_space():
     'extmod_whitelist': {},
     'extmod_blacklist': {},
     'minion_sign_messages': False,
-    'docker.compare_container_networks': {
-        'static': ['Aliases', 'Links', 'IPAMConfig'],
-        'automatic': ['IPAddress', 'Gateway',
-                      'GlobalIPv6Address', 'IPv6Gateway'],
-    },
     'discovery': False,
     'schedule': {},
     'ssh_merge_pillar': True

salt/modules/config.py

@@ -6,6 +6,7 @@
 # Import python libs
 from __future__ import absolute_import, print_function, unicode_literals
 import copy
+import fnmatch
 import re
 import os
 import logging
@@ -77,6 +78,12 @@
             'aliases.file': '/etc/aliases',
             'virt': {'tunnel': False,
                      'images': os.path.join(syspaths.SRV_ROOT_DIR, 'salt-images')},
+
+            'docker.exec_driver': 'docker-exec',
+            'docker.compare_container_networks': {
+                'static': ['Aliases', 'Links', 'IPAMConfig'],
+                'automatic': ['IPAddress', 'Gateway',
+                              'GlobalIPv6Address', 'IPv6Gateway']},
             }
 
 
@@ -130,31 +137,110 @@ def valid_fileproto(uri):
 
 def option(
         value,
-        default='',
+        default=None,
         omit_opts=False,
+        omit_grains=False,
+        omit_pillar=False,
         omit_master=False,
-        omit_pillar=False):
+        omit_all=False,
+        wildcard=False):
     '''
-    Pass in a generic option and receive the value that will be assigned
+    Returns the setting for the specified config value. The priority for
+    matches is the same as in :py:func:`config.get <salt.modules.config.get>`,
+    only this function does not recurse into nested data structures. Another
+    difference between this function and :py:func:`config.get
+    <salt.modules.config.get>` is that it comes with a set of "sane defaults".
+    To view these, you can run the following command:
+
+    .. code-block:: bash
+
+        salt '*' config.option '*' omit_all=True wildcard=True
+
+    default
+        The default value if no match is found. If not specified, then the
+        fallback default will be an empty string, unless ``wildcard=True``, in
+        which case the return will be an empty dictionary.
+
+    omit_opts : False
+        Pass as ``True`` to exclude matches from the minion configuration file
+
+    omit_grains : False
+        Pass as ``True`` to exclude matches from the grains
+
+    omit_pillar : False
+        Pass as ``True`` to exclude matches from the pillar data
+
+    omit_master : False
+        Pass as ``True`` to exclude matches from the master configuration file
+
+    omit_all : True
+        Shorthand to omit all of the above and return matches only from the
+        "sane defaults".
+
+        .. versionadded:: Neon
+
+    wildcard : False
+        If used, this will perform pattern matching on keys. Note that this
+        will also significantly change the return data. Instead of only a value
+        being returned, a dictionary mapping the matched keys to their values
+        is returned. For example, using ``wildcard=True`` with a ``key`` of
+        ``'foo.ba*`` could return a dictionary like so:
+
+        .. code-block:: python
+
+            {'foo.bar': True, 'foo.baz': False}
+
+        .. versionadded:: Neon
 
     CLI Example:
 
     .. code-block:: bash
 
         salt '*' config.option redis.host
     '''
-    if not omit_opts:
-        if value in __opts__:
-            return __opts__[value]
-    if not omit_master:
-        if value in __pillar__.get('master', {}):
-            return __pillar__['master'][value]
-    if not omit_pillar:
-        if value in __pillar__:
-            return __pillar__[value]
-    if value in DEFAULTS:
-        return DEFAULTS[value]
-    return default
+    if omit_all:
+        omit_opts = omit_grains = omit_pillar = omit_master = True
+
+    if default is None:
+        default = '' if not wildcard else {}
+
+    if not wildcard:
+        if not omit_opts:
+            if value in __opts__:
+                return __opts__[value]
+        if not omit_grains:
+            if value in __grains__:
+                return __grains__[value]
+        if not omit_pillar:
+            if value in __pillar__:
+                return __pillar__[value]
+        if not omit_master:
+            if value in __pillar__.get('master', {}):
+                return __pillar__['master'][value]
+        if value in DEFAULTS:
+            return DEFAULTS[value]
+
+        # No match
+        return default
+    else:
+        # We need to do the checks in the reverse order so that minion opts
+        # takes precedence
+        ret = {}
+        for omit, data in ((omit_master, __pillar__.get('master', {})),
+                           (omit_pillar, __pillar__),
+                           (omit_grains, __grains__),
+                           (omit_opts, __opts__)):
+            if not omit:
+                ret.update(
+                    {x: data[x] for x in fnmatch.filter(data, value)}
+                )
+        # Check the DEFAULTS as well to see if the pattern matches it
+        for item in (x for x in fnmatch.filter(DEFAULTS, value)
+                     if x not in ret):
+            ret[item] = DEFAULTS[item]
+
+        # If no matches, return the default
+        return ret or default
 
 
 def merge(value,
@@ -219,9 +305,9 @@ def get(key, default='', delimiter=':', merge=None, omit_opts=False,
     .. versionadded: 0.14.0
 
     Attempt to retrieve the named value from the minion config file, pillar,
-    grains or the master config. If the named value is not available, return the
-    value specified by ``default``. If not specified, the default is an empty
-    string.
+    grains or the master config. If the named value is not available, return
+    the value specified by the ``default`` argument. If this argument is not
+    specified, ``default`` falls back to an empty string.
 
     Values can also be retrieved from nested dictionaries. Assume the below
     data structure:

salt/modules/dockermod.py

@@ -50,7 +50,23 @@
 If you have previously performed a ``docker login`` from the minion, then the
 credentials saved in ``~/.docker/config.json`` will be used for any actions
 which require authentication. If not, then credentials can be configured in
-Pillar data. The configuration schema is as follows:
+any of the following locations:
+
+- Minion config file
+- Grains
+- Pillar data
+- Master config file (requires :conf_minion:`pillar_opts` to be set to ``True``
+  in Minion config file in order to work)
+
+.. important::
+    Versions prior to Neon require that Docker credentials are configured in
+    Pillar data. Be advised that Pillar data is still recommended though,
+    because this keeps the configuration from being stored on the Minion.
+
+    Also, keep in mind that if one gets your ``~/.docker/config.json``, the
+    password can be decoded from its contents.
+
+The configuration schema is as follows:
 
 .. code-block:: yaml
 
@@ -346,7 +362,7 @@ def _get_client(timeout=NOTSET, **kwargs):
         client_kwargs['timeout'] = timeout
     for key, val in (('base_url', 'docker.url'),
                      ('version', 'docker.version')):
-        param = __salt__['config.get'](val, NOTSET)
+        param = __salt__['config.option'](val, NOTSET)
         if param is not NOTSET:
             client_kwargs[key] = param
 
@@ -359,7 +375,7 @@ def _get_client(timeout=NOTSET, **kwargs):
         # it's not defined by user.
         client_kwargs['version'] = 'auto'
 
-    docker_machine = __salt__['config.get']('docker.machine', NOTSET)
+    docker_machine = __salt__['config.option']('docker.machine', NOTSET)
 
     if docker_machine is not NOTSET:
         docker_machine_json = __salt__['cmd.run'](
@@ -455,7 +471,8 @@ def _check_update_mine():
     try:
         ret = __context__['docker.update_mine']
     except KeyError:
-        ret = __context__['docker.update_mine'] = __salt__['config.get']('docker.update_mine', default=True)
+        ret = __context__['docker.update_mine'] = __salt__[
+            'config.option']('docker.update_mine', default=True)
     return ret
 
 
@@ -521,7 +538,7 @@ def _get_exec_driver():
     '''
     contextkey = 'docker.exec_driver'
     if contextkey not in __context__:
-        from_config = __salt__['config.get'](contextkey, None)
+        from_config = __salt__['config.option'](contextkey, None)
         # This if block can be removed once we make docker-exec a default
         # option, as it is part of the logic in the commented block above.
         if from_config is not None:
@@ -1037,6 +1054,11 @@ def compare_container_networks(first, second):
     than waiting for a new Salt release one can just set
     :conf_minion:`docker.compare_container_networks`.
 
+    .. versionchanged:: Neon
+        This config option can now also be set in pillar data and grains.
+        Additionally, it can be set in the master config file, provided that
+        :conf_minion:`pillar_opts` is enabled on the minion.
+
     .. note::
         The checks for automatic IP configuration described above only apply if
         ``IPAMConfig`` is among the keys set for static IP checks in
@@ -1058,7 +1080,8 @@ def compare_container_networks(first, second):
     def _get_nets(data):
         return data.get('NetworkSettings', {}).get('Networks', {})
 
-    compare_keys = __opts__['docker.compare_container_networks']
+    compare_keys = __salt__['config.option']('docker.compare_container_networks')
+
     result1 = inspect_container(first) \
         if not isinstance(first, dict) \
         else first
@@ -1396,23 +1419,21 @@ def login(*registries):
     # NOTE: This function uses the "docker login" CLI command so that login
     # information is added to the config.json, since docker-py isn't designed
     # to do so.
-    registry_auth = __pillar__.get('docker-registries', {})
+    registry_auth = __salt__['config.get']('docker-registries', {})
     ret = {'retcode': 0}
     errors = ret.setdefault('Errors', [])
     if not isinstance(registry_auth, dict):
         errors.append('\'docker-registries\' Pillar value must be a dictionary')
         registry_auth = {}
-    for key, data in six.iteritems(__pillar__):
+    for reg_name, reg_conf in six.iteritems(
+            __salt__['config.option']('*-docker-registries', wildcard=True)):
         try:
-            if key.endswith('-docker-registries'):
-                try:
-                    registry_auth.update(data)
-                except TypeError:
-                    errors.append(
-                        '\'{0}\' Pillar value must be a dictionary'.format(key)
-                    )
-        except AttributeError:
-            pass
+            registry_auth.update(reg_conf)
+        except TypeError:
+            errors.append(
+                'Docker registry \'{0}\' was not specified as a '
+                'dictionary'.format(reg_name)
+            )
 
     # If no registries passed, we will auth to all of them
     if not registries:

salt/states/docker_container.py

@@ -2062,7 +2062,8 @@ def _get_nets():
                 __context__[contextkey] = new_container_info.get(
                     'NetworkSettings', {}).get('Networks', {})
             return __context__[contextkey]
-        autoip_keys = __opts__['docker.compare_container_networks'].get('automatic', [])
+        autoip_keys = __salt__['config.option'](
+            'docker.compare_container_networks').get('automatic', [])
         for net_name, net_changes in six.iteritems(
                 ret['changes'].get('container', {}).get('Networks', {})):
             if 'IPConfiguration' in net_changes \

tests/integration/states/test_docker_container.py

@@ -26,6 +26,7 @@
 import salt.utils.network
 import salt.utils.path
 from salt.exceptions import CommandExecutionError
+from salt.modules.config import DEFAULTS as _config_defaults
 
 # Import 3rd-party libs
 from salt.ext import six
@@ -772,7 +773,7 @@ def _test_running(self, container_name, *nets):
         container_netinfo = self.run_function(
             'docker.inspect_container',
             [container_name]).get('NetworkSettings', {}).get('Networks', {})[nets[-1].name]
-        autoip_keys = self.minion_opts['docker.compare_container_networks']['automatic']
+        autoip_keys = _config_defaults['docker.compare_container_networks']['automatic']
         autoip_config = {
             x: y for x, y in six.iteritems(container_netinfo)
             if x in autoip_keys and y
@@ -1150,12 +1151,18 @@ def test_run_failhard(self, name):
         Test to make sure that we fail a state when the container exits with
         nonzero status if failhard is set to True, and that we don't when it is
         set to False.
+
+        NOTE: We can't use RUNTIME_VARS.SHELL_FALSE_PATH here because the image
+        we build on-the-fly here is based on busybox and does not include
+        /usr/bin/false. Therefore, when the host machine running the tests
+        has /usr/bin/false, it will not exist in the container and the Docker
+        Engine API will cause an exception to be raised.
         '''
         ret = self.run_state(
             'docker_container.run',
             name=name,
             image=self.image,
-            command=RUNTIME_VARS.SHELL_FALSE_PATH,
+            command='/bin/false',
             failhard=True)
         self.assertSaltFalseReturn(ret)
         ret = ret[next(iter(ret))]
@@ -1171,7 +1178,7 @@ def test_run_failhard(self, name):
             'docker_container.run',
             name=name,
             image=self.image,
-            command=RUNTIME_VARS.SHELL_FALSE_PATH,
+            command='/bin/false',
             failhard=False)
         self.assertSaltTrueReturn(ret)
         ret = ret[next(iter(ret))]