Merge pull request #95 from salt-formulas/develop

Release 1.7

README-extensions.rst

@@ -33,7 +33,7 @@ To control the feature there are two options available:
 .. code-block:: yaml
 
   ignore_class_notfound: False
-  ignore_class_regexp: ['.*']
+  ignore_class_notfound_regexp: ['.*']
 
 If you set regexp pattern to ``service.*`` all missing classes starting 'service.' will be logged with warning, but will not
 fail to return rendered reclass. Assuming all parameter interpolation passes.
@@ -691,3 +691,38 @@ classes for the pre-prod environment to use a directory on the local disc:
         storage_type: yaml_fs
         # options for yaml_fs storage type
         uri: /srv/salt/env/pre-prod/classes
+
+
+Support to use current node parameters as references in class name
+------------------------------------------------------------------
+
+With the following reclass config:
+
+.. code-block::
+
+    => /etc/reclass/nodes/mynode.yml
+    classes:
+      - common
+    parameters:
+      project: myproject
+
+    => /etc/reclass/classes/common.yml
+    class:
+      - ${project}
+
+    => /etc/reclass/classes/myproject.yml
+    parameters:
+      some:
+        project: parameters
+
+
+Will get the following result for the parameters:
+
+.. code-block:: yaml
+
+    parameters:
+      project: myproject
+      some:
+        project: parameters
+
+

doc/source/changelog.rst

@@ -5,6 +5,10 @@ ChangeLog
 ========= ========== ========================================================
 Version   Date       Changes
 ========= ========== ========================================================
+1.7.0     2020-10-02 Fixes and few new features:
+                     * Allow class mappings to wildcard match against either the node name and class
+                     * Support for .yaml along with .yml
+                     * Support to use current node parameters as references in class name
 1.6.0     2018-11-06 * Python code and parser refactoring by a-ovchinnikov
                      * Improvements in yaml_git and mixed setup by Andrew Pickford
                      * Relative paths in class names by Petr Michalec, Martin Polreich and Andrew Pickford

doc/source/operations.rst

@@ -101,6 +101,13 @@ end with ``.ch`` (again, note the escaped leading asterisk). Multiple classes
 can be assigned to each mapping by providing a space-separated list (class
 names cannot contain spaces anyway).
 
+By default the class mappings regex match is done against the node name. This can
+be changed to do the match against the path of the node file from the classes
+directory, but dropping the .yml extension at the end of the node file. This is
+controlled with the setting class_mappings_match_path. When False (the
+default) the match is done again the node name and when true the match is done
+against the node file path.
+
 .. warning::
 
   The class mappings do not really belong in the configuration file, as they

reclass/core.py

@@ -25,6 +25,7 @@
 from reclass.settings import Settings
 from reclass.datatypes import Entity, Classes, Parameters, Exports
 from reclass.errors import MappingFormatError, ClassNameResolveError, ClassNotFound, InvQueryClassNameResolveError, InvQueryClassNotFound, InvQueryError, InterpolationError, ResolveError
+from reclass.values import NodeInventory
 from reclass.values.parser import Parser
 
 
@@ -72,26 +73,30 @@ def _shlex_split(instr):
             key = '/{0}/'.format(key)
         return key, list(lexer)
 
-    def _get_class_mappings_entity(self, nodename):
+    def _get_class_mappings_entity(self, entity):
         if not self._class_mappings:
             return Entity(self._settings, name='empty (class mappings)')
         c = Classes()
+        if self._settings.class_mappings_match_path:
+            matchname = entity.pathname
+        else:
+            matchname = entity.name
         for mapping in self._class_mappings:
             matched = False
             key, klasses = Core._shlex_split(mapping)
             if key[0] == ('/'):
-                matched = Core._match_regexp(key[1:-1], nodename)
+                matched = Core._match_regexp(key[1:-1], matchname)
                 if matched:
                     for klass in klasses:
                         c.append_if_new(matched.expand(klass))
 
             else:
-                if Core._match_glob(key, nodename):
+                if Core._match_glob(key, matchname):
                     for klass in klasses:
                         c.append_if_new(klass)
 
         return Entity(self._settings, classes=c,
-                      name='class mappings for node {0}'.format(nodename))
+                      name='class mappings for node {0}'.format(entity.name))
 
     def _get_input_data_entity(self):
         if not self._input_data:
@@ -167,7 +172,29 @@ def _get_automatic_parameters(self, nodename, environment):
         else:
             return Parameters({}, self._settings, '')
 
+    def _get_scalar_parameters(self, node_parameters):
+        if self._settings.scalar_parameters:
+            scalars = node_parameters.as_dict().get(
+                    self._settings.scalar_parameters, {})
+            return Parameters(
+                    {self._settings.scalar_parameters: scalars}, self._settings, '__scalar__')
+        else:
+            return Parameters({}, self._settings, '')
+
     def _get_inventory(self, all_envs, environment, queries):
+        '''
+            Returns a dictionary of NodeInventory objects, one per matching node. Exports
+            which are required for the given queries (or all exports if the queries is None)
+            are rendered, remaining exports are left unrendered.
+
+            Args:
+              all_envs - if True match export values from nodes in any environment
+                         else if False match only for nodes in the same environment as the
+                         environment parameter
+              environment - node environment to match against if all_envs is False
+              queries - list of inventory queries to determine required export values
+                        or if None match all exports defined by a node
+        '''
         inventory = {}
         for nodename in self._storage.enumerate_nodes():
             try:
@@ -187,6 +214,7 @@ def _get_inventory(self, all_envs, environment, queries):
                 except ClassNameResolveError as e:
                     raise InvQueryClassNameResolveError(e)
                 if queries is None:
+                    # This only happens if reclass is called with the --inventory option
                     try:
                         node.interpolate_exports()
                     except InterpolationError as e:
@@ -199,17 +227,18 @@ def _get_inventory(self, all_envs, environment, queries):
                         except InterpolationError as e:
                             e.nodename = nodename
                             raise InvQueryError(q.contents, e, context=p, uri=q.uri)
-                inventory[nodename] = node.exports.as_dict()
+                inventory[nodename] = NodeInventory(node.exports.as_dict(), node_base.environment == environment)
         return inventory
 
     def _node_entity(self, nodename):
         node_entity = self._storage.get_node(nodename, self._settings)
         if node_entity.environment == None:
             node_entity.environment = self._settings.default_environment
         base_entity = Entity(self._settings, name='base')
-        base_entity.merge(self._get_class_mappings_entity(node_entity.name))
+        base_entity.merge(self._get_class_mappings_entity(node_entity))
         base_entity.merge(self._get_input_data_entity())
         base_entity.merge_parameters(self._get_automatic_parameters(nodename, node_entity.environment))
+        base_entity.merge_parameters(self._get_scalar_parameters(node_entity.parameters))
         seen = {}
         merge_base = self._recurse_entity(base_entity, seen=seen, nodename=nodename,
                                           environment=node_entity.environment)

reclass/datatypes/entity.py

@@ -24,9 +24,10 @@ class Entity(object):
     '''
     def __init__(self, settings, classes=None, applications=None,
                  parameters=None, exports=None, uri=None, name=None,
-                 environment=None):
+                 pathname=None, environment=None):
         self._uri = '' if uri is None else uri
         self._name = '' if name is None else name
+        self._pathname = '' if pathname is None else pathname
         self._classes = self._set_field(classes, Classes)
         self._applications = self._set_field(applications, Applications)
         pars = [None, settings, uri]
@@ -36,6 +37,7 @@ def __init__(self, settings, classes=None, applications=None,
 
     name = property(lambda s: s._name)
     uri = property(lambda s: s._uri)
+    pathname = property(lambda s: s._pathname)
     classes = property(lambda s: s._classes)
     applications = property(lambda s: s._applications)
     parameters = property(lambda s: s._parameters)
@@ -101,10 +103,10 @@ def __ne__(self, other):
         return not self.__eq__(other)
 
     def __repr__(self):
-        return "%s(%r, %r, %r, %r, uri=%r, name=%r, environment=%r)" % (
+        return "%s(%r, %r, %r, %r, uri=%r, name=%r, pathname=%r, environment=%r)" % (
                    self.__class__.__name__, self.classes, self.applications,
                    self.parameters, self.exports, self.uri, self.name,
-                   self.environment)
+                   self.pathname, self.environment)
 
     def as_dict(self):
         return {'classes': self._classes.as_list(),

reclass/datatypes/tests/test_entity.py

@@ -11,9 +11,12 @@
 from __future__ import print_function
 from __future__ import unicode_literals
 
+from six import iteritems
+
 from reclass.settings import Settings
 from reclass.datatypes import Entity, Classes, Parameters, Applications, Exports
 from reclass.errors import ResolveError
+from reclass.values import NodeInventory
 import unittest
 
 try:
@@ -167,40 +170,47 @@ def test_as_dict(self, **types):
 
 class TestEntityNoMock(unittest.TestCase):
 
+    def _make_inventory(self, nodes):
+        return { name: NodeInventory(node, True) for name, node in iteritems(nodes) }
+
     def test_interpolate_list_types(self):
         node1_exports = Exports({'exps': [ '${one}' ] }, SETTINGS, 'first')
         node1_parameters = Parameters({'alpha': [ '${two}', '${three}' ], 'one': 1, 'two': 2, 'three': 3 }, SETTINGS, 'first')
         node1_entity = Entity(SETTINGS, classes=None, applications=None, parameters=node1_parameters, exports=node1_exports)
         node2_exports = Exports({'exps': '${alpha}' }, SETTINGS, 'second')
         node2_parameters = Parameters({}, SETTINGS, 'second')
         node2_entity = Entity(SETTINGS, classes=None, applications=None, parameters=node2_parameters, exports=node2_exports)
-        r = {'exps': [ 1, 2, 3 ]}
+        result = {'exps': [ 1, 2, 3 ]}
         node1_entity.merge(node2_entity)
         node1_entity.interpolate(None)
         self.assertIs(type(node1_entity.exports.as_dict()['exps']), list)
-        self.assertDictEqual(node1_entity.exports.as_dict(), r)
+        self.assertDictEqual(node1_entity.exports.as_dict(), result)
 
     def test_exports_with_refs(self):
-        inventory = {'node1': {'a': 1, 'b': 2}, 'node2': {'a': 3, 'b': 4}}
+        inventory = self._make_inventory({'node1': {'a': 1, 'b': 2}, 'node2': {'a': 3, 'b': 4}})
         node3_exports = Exports({'a': '${a}', 'b': '${b}'}, SETTINGS, '')
         node3_parameters = Parameters({'name': 'node3', 'a': '${c}', 'b': 5}, SETTINGS, '')
         node3_parameters.merge({'c': 3})
         node3_entity = Entity(SETTINGS, classes=None, applications=None, parameters=node3_parameters, exports=node3_exports)
         node3_entity.interpolate_exports()
-        inventory['node3'] = node3_entity.exports.as_dict()
-        r = {'node1': {'a': 1, 'b': 2}, 'node2': {'a': 3, 'b': 4}, 'node3': {'a': 3, 'b': 5}}
-        self.assertDictEqual(inventory, r)
+        inventory['node3'] = NodeInventory(node3_entity.exports.as_dict(), True)
+        result = { 'node1': NodeInventory({'a': 1, 'b': 2}, True),
+                   'node2': NodeInventory({'a': 3, 'b': 4}, True),
+                   'node3': NodeInventory({'a': 3, 'b': 5}, True) }
+        self.assertDictEqual(inventory, result)
 
     def test_reference_to_an_export(self):
-        inventory = {'node1': {'a': 1, 'b': 2}, 'node2': {'a': 3, 'b': 4}}
+        inventory = self._make_inventory({'node1': {'a': 1, 'b': 2}, 'node2': {'a': 3, 'b': 4}})
         node3_exports = Exports({'a': '${a}', 'b': '${b}'}, SETTINGS, '')
         node3_parameters = Parameters({'name': 'node3', 'ref': '${exp}', 'a': '${c}', 'b': 5}, SETTINGS, '')
         node3_parameters.merge({'c': 3, 'exp': '$[ exports:a ]'})
         node3_entity = Entity(SETTINGS, classes=None, applications=None, parameters=node3_parameters, exports=node3_exports)
         node3_entity.interpolate_exports()
-        inventory['node3'] = node3_entity.exports.as_dict()
+        inventory['node3'] = NodeInventory(node3_entity.exports.as_dict(), True)
         node3_entity.interpolate(inventory)
-        res_inv = {'node1': {'a': 1, 'b': 2}, 'node2': {'a': 3, 'b': 4}, 'node3': {'a': 3, 'b': 5}}
+        res_inv = { 'node1': NodeInventory({'a': 1, 'b': 2}, True),
+                    'node2': NodeInventory({'a': 3, 'b': 4}, True),
+                    'node3': NodeInventory({'a': 3, 'b': 5}, True) }
         res_params = {'a': 3, 'c': 3, 'b': 5, 'name': 'node3', 'exp': {'node1': 1, 'node3': 3, 'node2': 3}, 'ref': {'node1': 1, 'node3': 3, 'node2': 3}}
         self.assertDictEqual(node3_parameters.as_dict(), res_params)
         self.assertDictEqual(inventory, res_inv)
@@ -218,40 +228,61 @@ def test_exports_multiple_nodes(self):
         for p, q in queries:
             node1_entity.interpolate_single_export(q)
             node2_entity.interpolate_single_export(q)
-        res_inv = {'node1': {'a': {'test': 1}}, 'node2': {'a': {'test': 2}}}
-        res_params = {'a': {'test': 1}, 'b': 1, 'name': 'node1', 'exp': {'node1': {'test': 1}, 'node2': {'test': 2}}}
-        inventory = {}
-        inventory['node1'] = node1_entity.exports.as_dict()
-        inventory['node2'] = node2_entity.exports.as_dict()
+        res_inv = { 'node1': NodeInventory({'a': {'test': 1}}, True),
+                    'node2': NodeInventory({'a': {'test': 2}}, True) }
+        res_params = { 'name': 'node1',
+                       'a': {'test': 1},
+                       'b': 1,
+                       'exp': {'node1': {'test': 1}, 'node2': {'test': 2}} }
+        inventory = self._make_inventory({'node1': node1_entity.exports.as_dict(), 'node2': node2_entity.exports.as_dict()})
         node1_entity.interpolate(inventory)
         self.assertDictEqual(node1_parameters.as_dict(), res_params)
         self.assertDictEqual(inventory, res_inv)
 
     def test_exports_with_ancestor_references(self):
-        inventory = {'node1': {'alpha' : {'beta': {'a': 1, 'b': 2}}}, 'node2': {'alpha' : {'beta': {'a': 3, 'b': 4}}}}
+        inventory = self._make_inventory({'node1': {'alpha' : {'beta': {'a': 1, 'b': 2}}}, 'node2': {'alpha' : {'beta': {'a': 3, 'b': 4}}}})
         node3_exports = Exports({'alpha': '${alpha}'}, SETTINGS, '')
         node3_parameters = Parameters({'name': 'node3', 'alpha': {'beta' : {'a': 5, 'b': 6}}, 'exp': '$[ exports:alpha:beta ]'}, SETTINGS, '')
         node3_entity = Entity(SETTINGS, classes=None, applications=None, parameters=node3_parameters, exports=node3_exports)
-        res_params = {'exp': {'node1': {'a': 1, 'b': 2}, 'node2': {'a': 3, 'b': 4}, 'node3': {'a': 5, 'b': 6}}, 'name': 'node3', 'alpha': {'beta': {'a': 5, 'b': 6}}}
-        res_inv = {'node1': {'alpha' : {'beta': {'a': 1, 'b': 2}}}, 'node2': {'alpha' : {'beta': {'a': 3, 'b': 4}}}, 'node3': {'alpha' : {'beta': {'a': 5, 'b': 6}}}}
+        res_params = { 'name': 'node3',
+                       'exp': {'node1': {'a': 1, 'b': 2},
+                       'node2': {'a': 3, 'b': 4},
+                       'node3': {'a': 5, 'b': 6}},
+                       'alpha': {'beta': {'a': 5, 'b': 6}} }
+        res_inv = { 'node1': NodeInventory({'alpha' : {'beta': {'a': 1, 'b': 2}}}, True),
+                    'node2': NodeInventory({'alpha' : {'beta': {'a': 3, 'b': 4}}}, True),
+                    'node3': NodeInventory({'alpha' : {'beta': {'a': 5, 'b': 6}}}, True) }
         node3_entity.initialise_interpolation()
         queries = node3_entity.parameters.get_inv_queries()
         for p, q in queries:
             node3_entity.interpolate_single_export(q)
-        inventory['node3'] = node3_entity.exports.as_dict()
+        inventory['node3'] = NodeInventory(node3_entity.exports.as_dict(), True)
         node3_entity.interpolate(inventory)
         self.assertDictEqual(node3_parameters.as_dict(), res_params)
         self.assertDictEqual(inventory, res_inv)
 
     def test_exports_with_nested_references(self):
-        inventory = {'node1': {'alpha': {'a': 1, 'b': 2}}, 'node2': {'alpha': {'a': 3, 'b': 4}}}
+        inventory = self._make_inventory({'node1': {'alpha': {'a': 1, 'b': 2}}, 'node2': {'alpha': {'a': 3, 'b': 4}}})
         node3_exports = Exports({'alpha': '${alpha}'}, SETTINGS, '')
-        node3_parameters = Parameters({'name': 'node3', 'alpha': {'a': '${one}', 'b': '${two}'}, 'beta': '$[ exports:alpha ]', 'one': '111', 'two': '${three}', 'three': '123'}, SETTINGS, '')
+        node3_parameters = Parameters({ 'name': 'node3',
+                                        'alpha': {'a': '${one}', 'b': '${two}'},
+                                        'beta': '$[ exports:alpha ]',
+                                        'one': '111',
+                                        'two': '${three}',
+                                        'three': '123'},
+                                      SETTINGS, '')
         node3_entity = Entity(SETTINGS, classes=None, applications=None, parameters=node3_parameters, exports=node3_exports)
-        res_params = {'beta': {'node1': {'a': 1, 'b': 2}, 'node3': {'a': '111', 'b': '123'}, 'node2': {'a': 3, 'b': 4}}, 'name': 'node3', 'alpha': {'a': '111', 'b': '123'}, 'three': '123', 'two': '123', 'one': '111'}
-        res_inv = {'node1': {'alpha': {'a': 1, 'b': 2}}, 'node2': {'alpha': {'a': 3, 'b': 4}}, 'node3': {'alpha': {'a': '111', 'b': '123'}}}
+        res_params = { 'name': 'node3',
+                       'alpha': { 'a': '111', 'b': '123' },
+                       'beta': { 'node1': {'a': 1, 'b': 2 }, 'node2': { 'a': 3, 'b': 4}, 'node3': { 'a': '111', 'b': '123' } },
+                       'one': '111',
+                       'two': '123',
+                       'three': '123' }
+        res_inv = { 'node1': NodeInventory({'alpha': {'a': 1, 'b': 2}}, True),
+                    'node2': NodeInventory({'alpha': {'a': 3, 'b': 4}}, True),
+                    'node3': NodeInventory({'alpha': {'a': '111', 'b': '123'}}, True) }
         node3_entity.interpolate_exports()
-        inventory['node3'] = node3_entity.exports.as_dict()
+        inventory['node3'] = NodeInventory(node3_entity.exports.as_dict(), True)
         node3_entity.interpolate(inventory)
         self.assertDictEqual(node3_parameters.as_dict(), res_params)
         self.assertDictEqual(inventory, res_inv)
@@ -285,11 +316,12 @@ def test_exports_failed_render_ignore(self):
         for p, q in queries:
             node1_entity.interpolate_single_export(q)
             node2_entity.interpolate_single_export(q)
-        res_inv = {'node1': {'a': 1}, 'node2': {}}
-        res_params = { 'a': 1, 'name': 'node1', 'exp': {'node1': 1} }
-        inventory = {}
-        inventory['node1'] = node1_entity.exports.as_dict()
-        inventory['node2'] = node2_entity.exports.as_dict()
+        res_inv = { 'node1': NodeInventory({'a': 1}, True),
+                    'node2': NodeInventory({}, True) }
+        res_params = { 'name': 'node1',
+                       'a': 1,
+                       'exp': {'node1': 1} }
+        inventory = self._make_inventory({'node1': node1_entity.exports.as_dict(), 'node2': node2_entity.exports.as_dict()})
         node1_entity.interpolate(inventory)
         self.assertDictEqual(node1_parameters.as_dict(), res_params)
         self.assertDictEqual(inventory, res_inv)