sonic-cfggen POC (sonic-net#15)

Don't write to the config DB keys of disabled features

Signed-off-by: Maksym Hedeon <maksym@githedgehog.com>

src/sonic-config-engine/data/flags_to_keys_map.yaml

@@ -0,0 +1,59 @@
+INCLUDE_DATABASE:
+  - ''
+INCLUDE_DHCP_RELAY:
+  - ''
+INCLUDE_FRR_BFD:
+  - ''
+INCLUDE_FRR_BGP:
+  - BGP_NEIGHBOR
+  - DEVICE_METADATA.localhost.bgp_asn
+INCLUDE_FRR_OSPF:
+  - ''
+INCLUDE_FRR_PBR:
+  - ''
+INCLUDE_FRR_VRRP:
+  - ''
+INCLUDE_ICCPD:
+  - ''
+INCLUDE_KUBERNETES:
+  - ''
+INCLUDE_LLDP:
+  - ''
+INCLUDE_MACSEC:
+  - ''
+INCLUDE_MGMT_FRAMEWORK:
+  - ''
+INCLUDE_MUX:
+  - ''
+INCLUDE_NAT:
+  - ''
+INCLUDE_NTP:
+  - ''
+INCLUDE_P4RT:
+  - ''
+INCLUDE_PMON:
+  - ''
+INCLUDE_RADIUS:
+  - ''
+INCLUDE_RESTAPI:
+  - ''
+INCLUDE_ROUTER_ADVERTISER:
+  - ''
+INCLUDE_ROUTING_STACK:
+  - ''
+INCLUDE_SFLOW:
+  - ''
+INCLUDE_SNMP:
+  - ''
+INCLUDE_SSH:
+  - ''
+INCLUDE_SWSS:
+  - ''
+INCLUDE_SYNCD:
+  - ''
+INCLUDE_SYSLOG:
+  - ''
+INCLUDE_SYSTEM_TELEMETRY:
+  - ''
+INCLUDE_TEAMD:
+  - ''

src/sonic-config-engine/setup.py

@@ -63,7 +63,8 @@
     ],
     install_requires = dependencies,
     data_files = [
-        ('/usr/share/sonic/templates', glob.glob('data/*')),
+        ('/usr/share/sonic/templates', glob.glob('data/*.j2')),
+        ('/etc/sonic', glob.glob('data/flags_to_keys_map.yaml'))
     ],
     setup_requires= [
         'pytest-runner',

src/sonic-config-engine/sonic-cfggen

@@ -32,10 +32,9 @@ from functools import partial
 from minigraph import minigraph_encoder, parse_xml, parse_device_desc_xml, parse_asic_sub_role, parse_asic_switch_type
 from portconfig import get_port_config, get_breakout_mode
 from sonic_py_common.multi_asic import get_asic_id_from_name, get_asic_device_id
-from sonic_py_common import device_info
+from sonic_py_common import device_info, cleanup_common
 from swsscommon.swsscommon import ConfigDBConnector, SonicDBConfig, ConfigDBPipeConnector
 
-
 PY3x = sys.version_info >= (3, 0)
 
 # TODO: Remove STR_TYPE, FILE_TYPE once SONiC moves to Python 3.x
@@ -187,8 +186,7 @@ TODO(taoyl): Current version of config db only supports BGP admin states.
                         data[table][new_key] = data[table].pop(key)
         return data
 
-
-def deep_update(dst, src):
+def deep_update(dst, src, cleanup = False):
     """ Deep update of dst dict with contest of src dict"""
     pending_nodes = [(dst, src)]
     while len(pending_nodes) > 0:
@@ -199,6 +197,12 @@ def deep_update(dst, src):
                 pending_nodes.append((node, value))
             else:
                 d[key] = value
+
+    if cleanup:
+        disabled_flags = list(cleanup_common.read_metadata_conifg(key_pattern="INCLUDE_", value_pattern="^n$"))
+        keys_list = cleanup_common.read_map(disabled_flags)
+        cleanup_common.clean_cfggen_dict(dst, keys_list)
+
     return dst
 
 # sort_data is required as it is being imported by config/config_mgmt module in sonic_utilities
@@ -428,6 +432,7 @@ def main():
             SonicDBConfig.load_sonic_global_db_config(namespace=args.namespace)
             configdb = ConfigDBPipeConnector(use_unix_socket_path=True, namespace=args.namespace, **db_kwargs)
 
+        deep_update(data, data, cleanup=True)
         configdb.connect(False)
         configdb.mod_config(FormatConverter.output_to_db(data))
 
@@ -436,6 +441,7 @@ def main():
 
     if args.preset is not None:
         data = generate_sample_config(data, args.preset)
+        deep_update(data, data, cleanup=True)
         print(json.dumps(FormatConverter.to_serialized(data), indent=4, cls=minigraph_encoder))
 
 

src/sonic-py-common/sonic_py_common/cleanup_common.py

@@ -0,0 +1,125 @@
+import yaml
+import re
+import os
+
+PATH_BUILD_METADATA = "/etc/sonic/build_metadata.yaml"
+PATH_CONFIG = "/sonic/rules/config"
+PATH_CONFIG_USER = PATH_CONFIG + ".user"
+PATH_FLAG_TO_KEY_MAP = "/etc/sonic/flags_to_keys_map.yaml"
+PATH_DIST_FLAG_TO_KEY_MAP = os.path.dirname(os.path.abspath(__file__)) + "/.." + PATH_FLAG_TO_KEY_MAP
+PATH_CFGGEN_KEYS_CLEANER = "/tmp/keys_to_clean"
+
+def read_rules_config(path = None):
+    '''
+    Read config file (e.g. rules/config) and
+    return in dictionary format
+    '''
+
+    if path is None:
+        path = PATH_CONFIG
+
+    if os.path.exists(path) is False:
+        return {}
+
+    with open(path) as file:
+        file = file.read().split("\n")
+
+    dict = {}
+
+    for line in file:
+        # non empty line and not comment
+        if re.search("^$", line) is None and \
+            re.search("^#", line) is None:
+            line = line.replace("?=", "=", 1)
+            # clear from spaces
+            line = line.replace(" ", "")
+            key_value = line.split("=")
+            dict[key_value[0]] = key_value[1]
+
+    return dict
+
+def read_metadata_conifg(key_pattern = "", value_pattern = "", path = None):
+    '''
+    pattern: default is any string. "^$" - empty string
+    "word" - substring, "^word$" - definite string
+
+    return value: dict from metadata
+    '''
+
+    if path is None:
+        path = PATH_BUILD_METADATA
+
+    # Check if path exists
+    if os.path.exists(path) is True:
+        with open(path) as file:
+            config = yaml.safe_load(file)
+        config = config['Configuration']
+    # probably this is build unit test
+    # read rules/config from sonic-buildimage
+    else:
+        config = {}
+        config = read_rules_config(PATH_CONFIG)
+        config_user = read_rules_config(PATH_CONFIG_USER)
+        if config_user is not None:
+            config.update(config_user)
+
+    for key in list(config):
+        if re.search(key_pattern, key) is None or \
+            (re.search(key_pattern, key) is not None and \
+            re.search(value_pattern, config[key]) is None):
+            del config[key]
+
+    return config
+
+def read_map(keys, path = None):
+    '''
+    Get list of values from key list
+    '''
+
+    if path is None:
+        path = PATH_DIST_FLAG_TO_KEY_MAP
+
+    if keys is None or os.path.exists(path) is False:
+        return []
+
+    with open(path) as file:
+        map = yaml.safe_load(file)
+
+    values = []
+    for key in keys:
+        values.extend(map[key])
+
+    # remove empty strings
+    values = list(filter(None, values))
+
+    return values
+
+def rm_top_nested_key(dict, key):
+    '''
+    provide dot separated key string to go through path and remove top nested key from dictionary
+    e.g. DEVICE_METADATA.localhost.bgp_asn
+    e.g. BGP_NEIGHBOR
+    '''
+
+    if "." in key:
+        keys_splitted = key.split(".", 1)
+        if keys_splitted[0] in dict:
+            dict = dict[keys_splitted[0]]
+            key = keys_splitted[1]
+            rm_top_nested_key(dict, key)
+    else:
+        dict.pop(key, None)
+
+def clean_cfggen_dict(dict, keys):
+    '''
+    Helper function for sonic-cfggen. Clear keys for
+    disabled build flags (e.g. for INCLUDE_FRR_BGP).
+    dict: sonic-cfggen dictionary
+    keys: dot separated keys (see rm_top_nested_key)
+    '''
+
+    if keys is None:
+        return
+
+    for key in keys:
+        rm_top_nested_key(dict, key)