Merge pull request #195 from mirceaulinic/develop

Release 2020.10.2

docs/best_practices.rst

@@ -174,3 +174,50 @@ depending on your operating system and configuration, you may hit the hard
 limit for max open files. For example, on Unix operating systems, ``ulimit 
 -Hn`` will tell you the max open files number. If you hit any issues, consider 
 increasing this limit.
+
+Pillar Compilation Errors
+-------------------------
+
+Proxy Minions typically require Pillars for the authentication details. It 
+often happens that you may want your Pillars to use Grain data; but Grains are 
+typically collected *after* the connection has been established, or for the 
+connection to be established it requires the authentication details from the 
+Pillar. This results in a chicken and egg type dependency.
+
+If in your Pillar you have a block requiring device-specific Grains 
+like:
+
+.. code-block:: sls
+
+  {% if grains['model'] == 'VMX' %}
+  target_software_version: 17.4
+  {% endif %}
+
+During the first execution, at least, you'll see an error looking as below:
+
+.. code-block:: text
+
+  Traceback (most recent call last):
+    File "/usr/local/lib/python3.7/dist-packages/salt/pillar/__init__.py", line 884, in render_pstate
+      **defaults
+    File "/usr/local/lib/python3.7/dist-packages/salt/template.py", line 101, in compile_template
+      ret = render(input_data, saltenv, sls, **render_kwargs)
+    File "/usr/local/lib/python3.7/dist-packages/salt/renderers/jinja.py", line 79, in render
+      **kws
+    File "/usr/local/lib/python3.7/dist-packages/salt/utils/templates.py", line 260, in render_tmpl
+      output = render_str(tmplstr, context, tmplpath)
+    File "/usr/local/lib/python3.7/dist-packages/salt/utils/templates.py", line 505, in render_jinja_tmpl
+      raise SaltRenderError("Jinja variable {}{}".format(exc, out), buf=tmplstr)
+  salt.exceptions.SaltRenderError: Jinja variable 'salt.utils.odict.OrderedDict object' has no attribute 'model'
+
+This is normal, a cosmetical error, raised during the first Pillar compilation, 
+which you can ignore.
+
+To avoid having this sort of errors, you will want to have your Pillar 
+transformed to:
+
+.. code-block:: sls
+
+  {% if grains.get('model') == 'VMX' %}
+  target_software_version: 17.4
+  {% endif %}

docs/index.rst

@@ -223,7 +223,7 @@ The following targeting options are available:
 
 .. warning::
 
-    Some of the targeting options above may not be avaialble for some Roster
+    Some of the targeting options above may not be available for some Roster
     modules.
 
 To use a specific Roster, configure the ``proxy_roster`` (or simply ``roster``)

docs/man/salt-sproxy.1

@@ -227,7 +227,7 @@ use a range expression to identify targets. Range expressions look like
 \fBWARNING:\fP
 .INDENT 0.0
 .INDENT 3.5
-Some of the targeting options above may not be avaialble for some Roster
+Some of the targeting options above may not be available for some Roster
 modules.
 .UNINDENT
 .UNINDENT

pypi.rst

@@ -93,7 +93,7 @@ The following targeting options are available:
 
 .. warning::
 
-    Some of the targeting options above may not be avaialble for some Roster
+    Some of the targeting options above may not be available for some Roster
     modules.
 
 To use a specific Roster, configure the ``proxy_roster`` option into your

salt_sproxy/_runners/proxy.py

@@ -258,10 +258,21 @@ def gen_modules(self, initial_load=False):  # pylint: disable=arguments-differ
                 self.opts['roster_opts']['grains'], self.opts['grains']
             )
 
+        cached_grains = None
+        if self.opts.get('proxy_use_cached_grains', True):
+            cached_grains = self.opts.pop('proxy_cached_grains', None)
+
+        initial_grains = copy.deepcopy(self.opts['grains'])
+        if cached_grains:
+            # Merging the collected Grains into the cached Grains, but only for
+            # the initial Pillar compilation, to ensure we only do so to avoid
+            # any processing errors.
+            initial_grains = salt.utils.dictupdate.merge(cached_grains, initial_grains)
+
         if self.opts.get('proxy_load_pillar', True):
             self.opts['pillar'] = salt.pillar.get_pillar(
                 self.opts,
-                self.opts['grains'],
+                initial_grains,
                 self.opts['id'],
                 saltenv=self.opts['saltenv'],
                 pillarenv=self.opts.get('pillarenv'),
@@ -583,9 +594,13 @@ def salt_call(
     opts['proxy_cache_pillar'] = cache_pillar
     opts['preload_targeting'] = preload_targeting
     opts['invasive_targeting'] = invasive_targeting
-    opts['proxy_use_cached_grains'] = use_cached_grains
     opts['proxy_no_connect'] = no_connect
     opts['proxy_test_ping'] = test_ping
+    opts['proxy_use_cached_grains'] = use_cached_grains
+    if use_cached_grains:
+        opts['proxy_cached_grains'] = __salt__['cache.fetch'](
+            'minions/{}/data'.format(minion_id), 'grains'
+        )
     opts['roster_opts'] = roster_opts
     opts['returner'] = returner
     if not returner_kwargs:
@@ -1354,6 +1369,15 @@ def execute(
         # indefinitely.
         timeout = None
 
+    if tgt_type == 'pillar_target':
+        # When using the -I option on the CLI, the tgt_type passed on is called
+        # `pillar_target`:
+        # https://github.com/saltstack/salt/blob/e9e48b7fb6a688f4f22d74a849d58c1c156563d1/salt/utils/parsers.py#L1266
+        # While if we want to use this against existing Minions, the option
+        # needs to be just `pillar`:
+        # https://github.com/saltstack/salt/blob/99385b50718d70d93fd5b83e61c0f4b3a402490c/salt/utils/minions.py#L359
+        tgt_type = 'pillar'
+
     if preload_targeting or invasive_targeting:
         _tgt = '*'
         _tgt_type = 'glob'

salt_sproxy/cli.py

@@ -334,6 +334,7 @@ def run(self):
             'grain',
             'pcre',
             'grain_pcre',
+            'pillar',
             'pillar_pcre',
             'pillar_target',
             'nodegroup',

setup.py

@@ -20,7 +20,7 @@
 
 setup(
     name=name,
-    version='2020.10.1',
+    version='2020.10.2',
     namespace_packages=['salt_sproxy'],
     packages=find_packages(),
     author='Mircea Ulinic',