From e7b131602bebbc10f7b83a9314b817a0c32297b5 Mon Sep 17 00:00:00 2001 From: Lakshita Jain Date: Tue, 2 Aug 2022 16:51:46 +0530 Subject: [PATCH 1/7] CORTX-32806: QUery Deployment api Signed-off-by: Lakshita Jain --- .../src/utils/query_deployment/__init__.py | 18 +++ .../query_deployment/query_deployment.py | 145 ++++++++++++++++++ 2 files changed, 163 insertions(+) create mode 100644 py-utils/src/utils/query_deployment/__init__.py create mode 100644 py-utils/src/utils/query_deployment/query_deployment.py diff --git a/py-utils/src/utils/query_deployment/__init__.py b/py-utils/src/utils/query_deployment/__init__.py new file mode 100644 index 000000000..d334026bd --- /dev/null +++ b/py-utils/src/utils/query_deployment/__init__.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 + +# CORTX-Py-Utils: CORTX Python common library. +# Copyright (c) 2022 Seagate Technology LLC and/or its Affiliates +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# For any questions about this software or licensing, +# please email opensource@seagate.com or cortx-questions@seagate.com. + +from cortx.utils.query_deployment.query_deployment import QueryDeployment \ No newline at end of file diff --git a/py-utils/src/utils/query_deployment/query_deployment.py b/py-utils/src/utils/query_deployment/query_deployment.py new file mode 100644 index 000000000..aa39835f6 --- /dev/null +++ b/py-utils/src/utils/query_deployment/query_deployment.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 + +# CORTX-Py-Utils: CORTX Python common library. +# Copyright (c) 2022 Seagate Technology LLC and/or its Affiliates +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# For any questions about this software or licensing, +# please email opensource@seagate.com or cortx-questions@seagate.com. + +import os +import json +import errno +from collections import defaultdict +from cortx.utils.conf_store import Conf + +class Topology: + topology = { + "cortx": { + "common": { + "release": {} + } + }, + "cluster": [], + "nodes": [] + } +class QueryConfData: + """Query Data.""" + _query_idx = "query_idx" + _data_idx = "data_idx" + _local_file = '/tmp/local_conf.conf' + _local_conf = "yaml://" + _local_file + + def __init__(self): + _f = QueryConfData._local_file + if os.path.exists(_f): os.remove(_f) + + def get_data(self, kv_url: str): + """Get data related to the parent key from config.""" + return self._get_data(kv_url) + + def _get_data(self, kv_url: str): + """Return data in dict format.""" + not_required_keys=['num_','nodes'] + Conf.load(QueryConfData._query_idx, kv_url) + _data_keys = Conf.get_keys(QueryConfData._query_idx) + + Conf.load(QueryConfData._data_idx, QueryConfData._local_conf) + Conf.copy(QueryConfData._query_idx, QueryConfData._data_idx, _data_keys) + Conf.save(QueryConfData._data_idx) + for key in Conf.get_keys(QueryConfData._data_idx): + if 'num_'in key: + Conf.delete(QueryConfData._data_idx, key) + # if 'nodes'in key: + # Conf.delete(QueryConfData._data_idx, key,force=True) + Conf.save(QueryConfData._data_idx) + + from cortx.utils.conf_store import ConfStore + _cs = ConfStore() + _cs.load(QueryConfData._data_idx, QueryConfData._local_conf) + _data = _cs.get_data(QueryConfData._data_idx) + return _data.get_data() + + +class QueryDeployment: + """ Query Deployment """ + _query_conf = None + @staticmethod + def init(**kwargs): + """ Static init for initialising and setting attributes.""" + if QueryDeployment._query_conf is None: + QueryDeployment._query_conf = QueryConfData() + + @staticmethod + def get_cortx_topology(kv_url: str) -> dict: + """ get cluster toplogy """ + if QueryDeployment._query_conf is None: + QueryDeployment.init() + + _data = {} + _data = QueryDeployment._query_conf.get_data(kv_url) + if not len(_data) > 0: + raise QueryDeploymentError(errno.EINVAL, f"Invalid data in {kv_url}") + + return QueryDeployment._get_cortx_topology(_data) + + def _get_cortx_topology(data: dict) -> dict: + """ Map gconf fields to topology """ + nd=lambda: defaultdict(nd) + _config=Topology.topology + + #cortx_info + _config["cortx"]["common"]["release"]=data["cortx"]["common"]["release"] + + #cluster info + for cluster_key, cluster_val in data['cluster'].items(): + cluster_info=nd() + storage_set_info=nd() + storage_set_list=[] + cluster_info['security']=data['cortx']['common']['security'] + if cluster_key=='storage_set': + for storage_info in data['cluster']['storage_set']: + for storage_key,storage_val in storage_info.items(): + if storage_key!='nodes': + storage_set_info[storage_key]=storage_val + storage_set_list.append(storage_set_info) + cluster_info['storage_set']=storage_set_list + else: + cluster_info[cluster_key]=cluster_val + _config['cluster'].append((json.dumps(cluster_info))) + + # Nodes Info + for nodes_key, nodes_value in data['node'].items(): + nodes_info=nd() + nodes_info['machine_id'] = nodes_key + for key, val in data['node'][nodes_key].items(): + if key=='provisioning': + # TODO: uncomment below once deployment time is supported by provisioner + # nodes_info['deployment_time']=data['node'][nodes_key]['provisioning']['time'] + nodes_info['version']=data['node'][nodes_key]['provisioning']['version'] + else: + nodes_info[key]=val + #TBD json dumps to dict + _config["nodes"].append(json.dumps(nodes_info)) + return _config + +class QueryDeploymentError(Exception): + """Generic Exception with error code and output.""" + + def __init__(self, rc, message, *args): + """Initialize self.""" + self._rc = rc + self._desc = message % (args) + + def __str__(self): + """Return str(self).""" + if self._rc == 0: return self._desc + return "error(%d): %s" % (self._rc, self._desc) From e517d4444860bf5c3f86bd10a283994aae975186 Mon Sep 17 00:00:00 2001 From: Lakshita Jain Date: Tue, 2 Aug 2022 18:51:36 +0530 Subject: [PATCH 2/7] CORTX-32806:resolving codacy issues Signed-off-by: Lakshita Jain --- .../src/utils/query_deployment/__init__.py | 2 +- .../query_deployment/query_deployment.py | 50 +++++++++---------- 2 files changed, 24 insertions(+), 28 deletions(-) diff --git a/py-utils/src/utils/query_deployment/__init__.py b/py-utils/src/utils/query_deployment/__init__.py index d334026bd..987d0bb60 100644 --- a/py-utils/src/utils/query_deployment/__init__.py +++ b/py-utils/src/utils/query_deployment/__init__.py @@ -15,4 +15,4 @@ # For any questions about this software or licensing, # please email opensource@seagate.com or cortx-questions@seagate.com. -from cortx.utils.query_deployment.query_deployment import QueryDeployment \ No newline at end of file +from cortx.utils.query_deployment.query_deployment import QueryDeployment diff --git a/py-utils/src/utils/query_deployment/query_deployment.py b/py-utils/src/utils/query_deployment/query_deployment.py index aa39835f6..bc3e1842a 100644 --- a/py-utils/src/utils/query_deployment/query_deployment.py +++ b/py-utils/src/utils/query_deployment/query_deployment.py @@ -28,8 +28,8 @@ class Topology: "release": {} } }, - "cluster": [], - "nodes": [] + "cluster": [], + "nodes": [], } class QueryConfData: """Query Data.""" @@ -48,7 +48,6 @@ def get_data(self, kv_url: str): def _get_data(self, kv_url: str): """Return data in dict format.""" - not_required_keys=['num_','nodes'] Conf.load(QueryConfData._query_idx, kv_url) _data_keys = Conf.get_keys(QueryConfData._query_idx) @@ -58,8 +57,6 @@ def _get_data(self, kv_url: str): for key in Conf.get_keys(QueryConfData._data_idx): if 'num_'in key: Conf.delete(QueryConfData._data_idx, key) - # if 'nodes'in key: - # Conf.delete(QueryConfData._data_idx, key,force=True) Conf.save(QueryConfData._data_idx) from cortx.utils.conf_store import ConfStore @@ -88,45 +85,44 @@ def get_cortx_topology(kv_url: str) -> dict: _data = QueryDeployment._query_conf.get_data(kv_url) if not len(_data) > 0: raise QueryDeploymentError(errno.EINVAL, f"Invalid data in {kv_url}") - return QueryDeployment._get_cortx_topology(_data) def _get_cortx_topology(data: dict) -> dict: """ Map gconf fields to topology """ - nd=lambda: defaultdict(nd) - _config=Topology.topology + nd = lambda: defaultdict(nd) + _config = Topology.topology - #cortx_info - _config["cortx"]["common"]["release"]=data["cortx"]["common"]["release"] - - #cluster info + # to fetch cortx info + _config["cortx"]["common"]["release"] = data["cortx"]["common"]["release"] + + # to fetch cluster_info for cluster_key, cluster_val in data['cluster'].items(): - cluster_info=nd() - storage_set_info=nd() + cluster_info = nd() + storage_set_info = nd() storage_set_list=[] - cluster_info['security']=data['cortx']['common']['security'] - if cluster_key=='storage_set': + cluster_info['security'] = data['cortx']['common']['security'] + if cluster_key == 'storage_set': for storage_info in data['cluster']['storage_set']: for storage_key,storage_val in storage_info.items(): - if storage_key!='nodes': - storage_set_info[storage_key]=storage_val + if storage_key != 'nodes': + storage_set_info[storage_key] = storage_val storage_set_list.append(storage_set_info) - cluster_info['storage_set']=storage_set_list + cluster_info['storage_set'] = storage_set_list else: - cluster_info[cluster_key]=cluster_val + cluster_info[cluster_key] = cluster_val _config['cluster'].append((json.dumps(cluster_info))) - # Nodes Info - for nodes_key, nodes_value in data['node'].items(): - nodes_info=nd() + # to fetch Nodes Info + for nodes_key in data['node'].keys(): + nodes_info = nd() nodes_info['machine_id'] = nodes_key for key, val in data['node'][nodes_key].items(): - if key=='provisioning': + if key =='provisioning': # TODO: uncomment below once deployment time is supported by provisioner - # nodes_info['deployment_time']=data['node'][nodes_key]['provisioning']['time'] - nodes_info['version']=data['node'][nodes_key]['provisioning']['version'] + # nodes_info['deployment_time'] = data['node'][nodes_key]['provisioning']['time'] + nodes_info['version'] = data['node'][nodes_key]['provisioning']['version'] else: - nodes_info[key]=val + nodes_info[key] = val #TBD json dumps to dict _config["nodes"].append(json.dumps(nodes_info)) return _config From b8d4138f2314d3a1ea1dc76171a28df1e8187584 Mon Sep 17 00:00:00 2001 From: Lakshita Jain Date: Tue, 2 Aug 2022 19:02:27 +0530 Subject: [PATCH 3/7] CORTX-32806: resolving codacy issue Signed-off-by: Lakshita Jain --- py-utils/src/utils/query_deployment/query_deployment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py-utils/src/utils/query_deployment/query_deployment.py b/py-utils/src/utils/query_deployment/query_deployment.py index bc3e1842a..fbae8bb09 100644 --- a/py-utils/src/utils/query_deployment/query_deployment.py +++ b/py-utils/src/utils/query_deployment/query_deployment.py @@ -35,7 +35,7 @@ class QueryConfData: """Query Data.""" _query_idx = "query_idx" _data_idx = "data_idx" - _local_file = '/tmp/local_conf.conf' + _local_file = "/tmp/local_conf.conf" _local_conf = "yaml://" + _local_file def __init__(self): From b4589a574b3d8369a98b329ab0cc463b9e60d492 Mon Sep 17 00:00:00 2001 From: Lakshita Jain Date: Tue, 2 Aug 2022 19:24:50 +0530 Subject: [PATCH 4/7] codacy issues Signed-off-by: Lakshita Jain --- py-utils/src/utils/query_deployment/query_deployment.py | 1 - 1 file changed, 1 deletion(-) diff --git a/py-utils/src/utils/query_deployment/query_deployment.py b/py-utils/src/utils/query_deployment/query_deployment.py index fbae8bb09..484c4d8e0 100644 --- a/py-utils/src/utils/query_deployment/query_deployment.py +++ b/py-utils/src/utils/query_deployment/query_deployment.py @@ -91,7 +91,6 @@ def _get_cortx_topology(data: dict) -> dict: """ Map gconf fields to topology """ nd = lambda: defaultdict(nd) _config = Topology.topology - # to fetch cortx info _config["cortx"]["common"]["release"] = data["cortx"]["common"]["release"] From 15f9257a5573a5e893e39cf860461345fcb78ac0 Mon Sep 17 00:00:00 2001 From: Lakshita Jain Date: Tue, 2 Aug 2022 19:43:03 +0530 Subject: [PATCH 5/7] CORTX-32806: removing trailing whitespace Signed-off-by: Lakshita Jain --- py-utils/src/utils/query_deployment/query_deployment.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/py-utils/src/utils/query_deployment/query_deployment.py b/py-utils/src/utils/query_deployment/query_deployment.py index 484c4d8e0..fd8460011 100644 --- a/py-utils/src/utils/query_deployment/query_deployment.py +++ b/py-utils/src/utils/query_deployment/query_deployment.py @@ -93,7 +93,6 @@ def _get_cortx_topology(data: dict) -> dict: _config = Topology.topology # to fetch cortx info _config["cortx"]["common"]["release"] = data["cortx"]["common"]["release"] - # to fetch cluster_info for cluster_key, cluster_val in data['cluster'].items(): cluster_info = nd() @@ -110,7 +109,6 @@ def _get_cortx_topology(data: dict) -> dict: else: cluster_info[cluster_key] = cluster_val _config['cluster'].append((json.dumps(cluster_info))) - # to fetch Nodes Info for nodes_key in data['node'].keys(): nodes_info = nd() From 7b42dc78f8d51a456b7d156aa125f0478712efb9 Mon Sep 17 00:00:00 2001 From: Lakshita Jain Date: Tue, 2 Aug 2022 21:59:50 +0530 Subject: [PATCH 6/7] CORTX-32806: added query_deployment to packages Signed-off-by: Lakshita Jain --- py-utils/setup.py | 3 +- py-utils/src/utils/query_deployment/error.py | 29 ++++++++++++ .../query_deployment/query_deployment.py | 44 +++++++------------ 3 files changed, 47 insertions(+), 29 deletions(-) create mode 100644 py-utils/src/utils/query_deployment/error.py diff --git a/py-utils/setup.py b/py-utils/setup.py index ddd3ea603..1131034af 100644 --- a/py-utils/setup.py +++ b/py-utils/setup.py @@ -114,7 +114,8 @@ def get_requirements_files() -> list: 'cortx.utils.support_framework', 'cortx.utils.manifest', 'cortx.utils.audit_log', 'cortx.utils.cortx', - 'cortx.utils.http', 'cortx.utils.s3', 'cortx.utils.activity_tracker' + 'cortx.utils.http', 'cortx.utils.s3', 'cortx.utils.activity_tracker', + 'cortx.utils.query_deployment' ], package_data={ 'cortx': ['py.typed'], diff --git a/py-utils/src/utils/query_deployment/error.py b/py-utils/src/utils/query_deployment/error.py new file mode 100644 index 000000000..b40c3d59b --- /dev/null +++ b/py-utils/src/utils/query_deployment/error.py @@ -0,0 +1,29 @@ +#!/bin/env python3 + +# CORTX Python common library. +# Copyright (c) 2022 Seagate Technology LLC and/or its Affiliates +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# For any questions about this software or licensing, +# please email opensource@seagate.com or cortx-questions@seagate.com. +from cortx.utils.errors import UtilsError + +class QueryDeploymentError(UtilsError): + """Generic Exception with error code and output.""" + + def __init__(self, rc, message, *args): + """Initialize SetupError.""" + super().__init__(rc, message, *args) + + def __str__(self): + """Return str(self).""" + if self._rc == 0: return self._desc + return "error(%d): %s" % (self._rc, self._desc) diff --git a/py-utils/src/utils/query_deployment/query_deployment.py b/py-utils/src/utils/query_deployment/query_deployment.py index fd8460011..c199a4dba 100644 --- a/py-utils/src/utils/query_deployment/query_deployment.py +++ b/py-utils/src/utils/query_deployment/query_deployment.py @@ -20,6 +20,8 @@ import errno from collections import defaultdict from cortx.utils.conf_store import Conf +from cortx.utils.conf_store import ConfStore +from cortx.utils.query_deployment.error import QueryDeploymentError class Topology: topology = { @@ -32,15 +34,14 @@ class Topology: "nodes": [], } class QueryConfData: - """Query Data.""" + """Query Configuration Data.""" _query_idx = "query_idx" _data_idx = "data_idx" _local_file = "/tmp/local_conf.conf" _local_conf = "yaml://" + _local_file def __init__(self): - _f = QueryConfData._local_file - if os.path.exists(_f): os.remove(_f) + pass def get_data(self, kv_url: str): """Get data related to the parent key from config.""" @@ -58,8 +59,6 @@ def _get_data(self, kv_url: str): if 'num_'in key: Conf.delete(QueryConfData._data_idx, key) Conf.save(QueryConfData._data_idx) - - from cortx.utils.conf_store import ConfStore _cs = ConfStore() _cs.load(QueryConfData._data_idx, QueryConfData._local_conf) _data = _cs.get_data(QueryConfData._data_idx) @@ -89,14 +88,14 @@ def get_cortx_topology(kv_url: str) -> dict: def _get_cortx_topology(data: dict) -> dict: """ Map gconf fields to topology """ - nd = lambda: defaultdict(nd) + nested_dict = lambda: defaultdict(nested_dict) _config = Topology.topology - # to fetch cortx info - _config["cortx"]["common"]["release"] = data["cortx"]["common"]["release"] - # to fetch cluster_info + _config["cortx"]["common"]["release"] = data['cortx']['common']['release'] + + # To fetch cluster_info for cluster_key, cluster_val in data['cluster'].items(): - cluster_info = nd() - storage_set_info = nd() + cluster_info = nested_dict() + storage_set_info = nested_dict() storage_set_list=[] cluster_info['security'] = data['cortx']['common']['security'] if cluster_key == 'storage_set': @@ -109,30 +108,19 @@ def _get_cortx_topology(data: dict) -> dict: else: cluster_info[cluster_key] = cluster_val _config['cluster'].append((json.dumps(cluster_info))) - # to fetch Nodes Info + + # To fetch Nodes Info for nodes_key in data['node'].keys(): - nodes_info = nd() + nodes_info = nested_dict() nodes_info['machine_id'] = nodes_key for key, val in data['node'][nodes_key].items(): - if key =='provisioning': + if key == 'provisioning': # TODO: uncomment below once deployment time is supported by provisioner # nodes_info['deployment_time'] = data['node'][nodes_key]['provisioning']['time'] nodes_info['version'] = data['node'][nodes_key]['provisioning']['version'] else: nodes_info[key] = val - #TBD json dumps to dict _config["nodes"].append(json.dumps(nodes_info)) + if os.path.exists(QueryConfData._local_file): + os.remove(QueryConfData._local_file) return _config - -class QueryDeploymentError(Exception): - """Generic Exception with error code and output.""" - - def __init__(self, rc, message, *args): - """Initialize self.""" - self._rc = rc - self._desc = message % (args) - - def __str__(self): - """Return str(self).""" - if self._rc == 0: return self._desc - return "error(%d): %s" % (self._rc, self._desc) From 42a91effbe462712979e358c5017adf01fb3477f Mon Sep 17 00:00:00 2001 From: Lakshita Jain Date: Tue, 2 Aug 2022 23:55:04 +0530 Subject: [PATCH 7/7] CORTX-32806: updated comments Signed-off-by: Lakshita Jain --- py-utils/src/utils/query_deployment/query_deployment.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/py-utils/src/utils/query_deployment/query_deployment.py b/py-utils/src/utils/query_deployment/query_deployment.py index c199a4dba..2c0c4a3ee 100644 --- a/py-utils/src/utils/query_deployment/query_deployment.py +++ b/py-utils/src/utils/query_deployment/query_deployment.py @@ -90,6 +90,8 @@ def _get_cortx_topology(data: dict) -> dict: """ Map gconf fields to topology """ nested_dict = lambda: defaultdict(nested_dict) _config = Topology.topology + + # To fetch common_info _config["cortx"]["common"]["release"] = data['cortx']['common']['release'] # To fetch cluster_info