From fa0a3a061aeb6f4d73fb9dac1f557c968d4f08db Mon Sep 17 00:00:00 2001 From: liuh-80 Date: Fri, 9 Sep 2022 08:31:33 +0000 Subject: [PATCH 01/10] Test disable swsssdk for none UT usage. --- .gitmodules | 3 ++- src/sonic-py-swsssdk | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 09ebc6da2373..0ef8c88c4a5d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -21,7 +21,8 @@ url = https://github.com/sonic-net/sonic-dbsyncd [submodule "src/sonic-py-swsssdk"] path = src/sonic-py-swsssdk - url = https://github.com/sonic-net/sonic-py-swsssdk.git + url = https://github.com/liuh-80/sonic-py-swsssdk.git + branch = /dev/liuh/disable-sdk [submodule "src/sonic-snmpagent"] path = src/sonic-snmpagent url = https://github.com/sonic-net/sonic-snmpagent diff --git a/src/sonic-py-swsssdk b/src/sonic-py-swsssdk index cc847a2e11b3..cdec1549d4d4 160000 --- a/src/sonic-py-swsssdk +++ b/src/sonic-py-swsssdk @@ -1 +1 @@ -Subproject commit cc847a2e11b3dd6d7f9630bfcac2ac3af2912eba +Subproject commit cdec1549d4d49f1ca0cedf8d4ad1aaf3095f63b0 From 41781fe0ef37d585b4376d304b67dd7c50cd7e98 Mon Sep 17 00:00:00 2001 From: liuh-80 Date: Fri, 9 Sep 2022 08:34:55 +0000 Subject: [PATCH 02/10] update reference --- src/sonic-py-swsssdk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sonic-py-swsssdk b/src/sonic-py-swsssdk index cdec1549d4d4..a490bda3c8c0 160000 --- a/src/sonic-py-swsssdk +++ b/src/sonic-py-swsssdk @@ -1 +1 @@ -Subproject commit cdec1549d4d49f1ca0cedf8d4ad1aaf3095f63b0 +Subproject commit a490bda3c8c034708b9bda67c91dba11cc64f29d From 750168f30b649efb02d7e4a4e9cdc25badb46111 Mon Sep 17 00:00:00 2001 From: liuh-80 Date: Tue, 13 Sep 2022 06:46:43 +0000 Subject: [PATCH 03/10] Improve code --- src/sonic-py-swsssdk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sonic-py-swsssdk b/src/sonic-py-swsssdk index a490bda3c8c0..e4c9989ae60c 160000 --- a/src/sonic-py-swsssdk +++ b/src/sonic-py-swsssdk @@ -1 +1 @@ -Subproject commit a490bda3c8c034708b9bda67c91dba11cc64f29d +Subproject commit e4c9989ae60c94dc0c00f42cab22a15825e4a1ac From 2857b84fe5eb871e0ac777ce9e14ccbe52161561 Mon Sep 17 00:00:00 2001 From: liuh-80 Date: Tue, 13 Sep 2022 07:16:10 +0000 Subject: [PATCH 04/10] Update reference --- src/sonic-py-swsssdk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sonic-py-swsssdk b/src/sonic-py-swsssdk index e4c9989ae60c..fd415ba47ca0 160000 --- a/src/sonic-py-swsssdk +++ b/src/sonic-py-swsssdk @@ -1 +1 @@ -Subproject commit e4c9989ae60c94dc0c00f42cab22a15825e4a1ac +Subproject commit fd415ba47ca0d4b3e505c222d0ecd62a138d3db6 From 5f685421f4798e648d2d11bdb15fd17625a94d0e Mon Sep 17 00:00:00 2001 From: liuh-80 Date: Thu, 15 Sep 2022 09:23:53 +0000 Subject: [PATCH 05/10] Update reference --- src/sonic-py-swsssdk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sonic-py-swsssdk b/src/sonic-py-swsssdk index fd415ba47ca0..2e9e4eaaa719 160000 --- a/src/sonic-py-swsssdk +++ b/src/sonic-py-swsssdk @@ -1 +1 @@ -Subproject commit fd415ba47ca0d4b3e505c222d0ecd62a138d3db6 +Subproject commit 2e9e4eaaa719e82e2d13fbe0c791e941848bbe56 From f4ac20e4963273ccaea128fe3bb619d3e1dda068 Mon Sep 17 00:00:00 2001 From: liuh-80 Date: Mon, 19 Sep 2022 07:55:07 +0000 Subject: [PATCH 06/10] update submodule --- src/sonic-py-swsssdk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sonic-py-swsssdk b/src/sonic-py-swsssdk index 2e9e4eaaa719..a8b5097f43c6 160000 --- a/src/sonic-py-swsssdk +++ b/src/sonic-py-swsssdk @@ -1 +1 @@ -Subproject commit 2e9e4eaaa719e82e2d13fbe0c791e941848bbe56 +Subproject commit a8b5097f43c6358d624e75f30a573590a3dc43f9 From 1a1d04e9e3e001fe22e1ea2b2008cfe8e22d4ab9 Mon Sep 17 00:00:00 2001 From: liuh-80 Date: Wed, 21 Sep 2022 08:35:27 +0000 Subject: [PATCH 07/10] Porting sonic-db-dump-load to sonic-py-common --- src/sonic-py-common/setup.py | 6 + .../sonic_py_common/sonic_db_dump_load.py | 139 ++++++++++++++++++ src/sonic-py-swsssdk | 2 +- 3 files changed, 146 insertions(+), 1 deletion(-) create mode 100755 src/sonic-py-common/sonic_py_common/sonic_db_dump_load.py diff --git a/src/sonic-py-common/setup.py b/src/sonic-py-common/setup.py index f12c0d83cb77..a6eb24f211b0 100644 --- a/src/sonic-py-common/setup.py +++ b/src/sonic-py-common/setup.py @@ -27,6 +27,12 @@ 'pytest', 'mock==3.0.5' # For python 2. Version >=4.0.0 drops support for py2 ], + entry_points={ + 'console_scripts': [ + 'sonic-db-load = sonic_py_common:sonic_db_dump_load', + 'sonic-db-dump = sonic_py_common:sonic_db_dump_load', + ], + }, classifiers=[ 'Intended Audience :: Developers', 'Operating System :: Linux', diff --git a/src/sonic-py-common/sonic_py_common/sonic_db_dump_load.py b/src/sonic-py-common/sonic_py_common/sonic_db_dump_load.py new file mode 100755 index 000000000000..74678cc3c21c --- /dev/null +++ b/src/sonic-py-common/sonic_py_common/sonic_db_dump_load.py @@ -0,0 +1,139 @@ +## ref: https://github.com/p/redis-dump-load/blob/7bbdb1eaea0a51ed4758d3ce6ca01d497a4e7429/redisdl.py + +def sonic_db_dump_load(): + import optparse + import os.path + import re + import sys + from redisdl import dump, load + from swsscommon.swsscommon import SonicDBConfig + + DUMP = 1 + LOAD = 2 + + def options_to_kwargs(options): + args = {} + if options.password: + args['password'] = options.password + if options.encoding: + args['encoding'] = options.encoding + # dump only + if hasattr(options, 'pretty') and options.pretty: + args['pretty'] = True + if hasattr(options, 'keys') and options.keys: + args['keys'] = options.keys + # load only + if hasattr(options, 'use_expireat') and options.use_expireat: + args['use_expireat'] = True + if hasattr(options, 'empty') and options.empty: + args['empty'] = True + if hasattr(options, 'backend') and options.backend: + args['streaming_backend'] = options.backend + if hasattr(options, 'dbname') and options.dbname: + if options.conntype == 'tcp': + args['host'] = SonicDBConfig.get_hostname(options.dbname) + args['port'] = SonicDBConfig.get_port(options.dbname) + args['db'] = SonicDBConfig.get_dbid(options.dbname) + args['unix_socket_path'] = None + elif options.conntype == "unix_socket": + args['host'] = None + args['port'] = None + args['db'] = SonicDBConfig.get_dbid(options.dbname) + args['unix_socket_path'] = SonicDBConfig.get_socket(options.dbname) + else: + raise TypeError('redis connection type is tcp or unix_socket') + + return args + + def do_dump(options): + if options.output: + output = open(options.output, 'w') + else: + output = sys.stdout + + kwargs = options_to_kwargs(options) + dump(output, **kwargs) + + if options.output: + output.close() + + def do_load(options, args): + if len(args) > 0: + input = open(args[0], 'rb') + else: + input = sys.stdin + + kwargs = options_to_kwargs(options) + load(input, **kwargs) + + if len(args) > 0: + input.close() + + script_name = os.path.basename(sys.argv[0]) + if re.search(r'load(?:$|\.)', script_name): + action = help = LOAD + elif re.search(r'dump(?:$|\.)', script_name): + action = help = DUMP + else: + # default is dump, however if dump is specifically requested + # we don't show help text for toggling between dumping and loading + action = DUMP + help = None + + if help == LOAD: + usage = "Usage: %prog [options] [FILE]" + usage += "\n\nLoad data from FILE (which must be a JSON dump previously created" + usage += "\nby redisdl) into specified or default redis." + usage += "\n\nIf FILE is omitted standard input is read." + elif help == DUMP: + usage = "Usage: %prog [options]" + usage += "\n\nDump data from specified or default redis." + usage += "\n\nIf no output file is specified, dump to standard output." + else: + usage = "Usage: %prog [options]" + usage += "\n %prog -l [options] [FILE]" + usage += "\n\nDump data from redis or load data into redis." + usage += "\n\nIf input or output file is specified, dump to standard output and load" + usage += "\nfrom standard input." + parser = optparse.OptionParser(usage=usage) + parser.add_option('-w', '--password', help='connect with PASSWORD') + if help == DUMP: + parser.add_option('-n', '--dbname', help='dump DATABASE (APPL_DB/ASIC_DB...)') + parser.add_option('-t', '--conntype', help='indicate redis connection type (tcp[default] or unix_socket)', default='tcp') + parser.add_option('-k', '--keys', help='dump only keys matching specified glob-style pattern') + parser.add_option('-o', '--output', help='write to OUTPUT instead of stdout') + parser.add_option('-y', '--pretty', help='split output on multiple lines and indent it', action='store_true') + parser.add_option('-E', '--encoding', help='set encoding to use while decoding data from redis', default='utf-8') + elif help == LOAD: + parser.add_option('-n', '--dbname', help='dump DATABASE (APPL_DB/ASIC_DB...)') + parser.add_option('-t', '--conntype', help='indicate redis connection type (tcp[default] or unix_socket)', default='tcp') + parser.add_option('-e', '--empty', help='delete all keys in destination db prior to loading', action='store_true') + parser.add_option('-E', '--encoding', help='set encoding to use while encoding data to redis', default='utf-8') + parser.add_option('-B', '--backend', help='use specified streaming backend') + parser.add_option('-A', '--use-expireat', help='use EXPIREAT rather than TTL/EXPIRE', action='store_true') + else: + parser.add_option('-l', '--load', help='load data into redis (default is to dump data from redis)', action='store_true') + parser.add_option('-n', '--dbname', help='dump DATABASE (APPL_DB/ASIC_DB/COUNTERS_DB/LOGLEVEL_DB/CONFIG_DB...)') + parser.add_option('-t', '--conntype', help='indicate redis connection type (tcp[default] or unix_socket)', default='tcp') + parser.add_option('-k', '--keys', help='dump only keys matching specified glob-style pattern') + parser.add_option('-o', '--output', help='write to OUTPUT instead of stdout (dump mode only)') + parser.add_option('-y', '--pretty', help='split output on multiple lines and indent it (dump mode only)', action='store_true') + parser.add_option('-e', '--empty', help='delete all keys in destination db prior to loading (load mode only)', action='store_true') + parser.add_option('-E', '--encoding', help='set encoding to use while decoding data from redis', default='utf-8') + parser.add_option('-A', '--use-expireat', help='use EXPIREAT rather than TTL/EXPIRE', action='store_true') + parser.add_option('-B', '--backend', help='use specified streaming backend (load mode only)') + options, args = parser.parse_args() + + if hasattr(options, 'load') and options.load: + action = LOAD + + if action == DUMP: + if len(args) > 0: + parser.print_help() + exit(4) + do_dump(options) + else: + if len(args) > 1: + parser.print_help() + exit(4) + do_load(options, args) diff --git a/src/sonic-py-swsssdk b/src/sonic-py-swsssdk index a8b5097f43c6..2dc0dabf8ec2 160000 --- a/src/sonic-py-swsssdk +++ b/src/sonic-py-swsssdk @@ -1 +1 @@ -Subproject commit a8b5097f43c6358d624e75f30a573590a3dc43f9 +Subproject commit 2dc0dabf8ec2ca5e5902cb819becb019a47cff53 From 2749d1fb3d5fc359addedbf9791cfbef4410eb8c Mon Sep 17 00:00:00 2001 From: liuh-80 Date: Thu, 22 Sep 2022 02:22:47 +0000 Subject: [PATCH 08/10] Fix entry point --- src/sonic-py-common/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sonic-py-common/setup.py b/src/sonic-py-common/setup.py index a6eb24f211b0..144cf61f52c4 100644 --- a/src/sonic-py-common/setup.py +++ b/src/sonic-py-common/setup.py @@ -29,8 +29,8 @@ ], entry_points={ 'console_scripts': [ - 'sonic-db-load = sonic_py_common:sonic_db_dump_load', - 'sonic-db-dump = sonic_py_common:sonic_db_dump_load', + 'sonic-db-load = sonic_py_common.sonic_db_dump_load:sonic_db_dump_load', + 'sonic-db-dump = sonic_py_common.sonic_db_dump_load:sonic_db_dump_load', ], }, classifiers=[ From d715879674eb9de0845907296b710bcbc84a4992 Mon Sep 17 00:00:00 2001 From: liuh-80 Date: Mon, 26 Sep 2022 07:16:16 +0000 Subject: [PATCH 09/10] Fix code issue --- .../sonic_py_common/sonic_db_dump_load.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/sonic-py-common/sonic_py_common/sonic_db_dump_load.py b/src/sonic-py-common/sonic_py_common/sonic_db_dump_load.py index 74678cc3c21c..126dd2bc112e 100755 --- a/src/sonic-py-common/sonic_py_common/sonic_db_dump_load.py +++ b/src/sonic-py-common/sonic_py_common/sonic_db_dump_load.py @@ -31,15 +31,15 @@ def options_to_kwargs(options): args['streaming_backend'] = options.backend if hasattr(options, 'dbname') and options.dbname: if options.conntype == 'tcp': - args['host'] = SonicDBConfig.get_hostname(options.dbname) - args['port'] = SonicDBConfig.get_port(options.dbname) - args['db'] = SonicDBConfig.get_dbid(options.dbname) + args['host'] = SonicDBConfig.getDbHostname(options.dbname) + args['port'] = SonicDBConfig.getDbPort(options.dbname) + args['db'] = SonicDBConfig.getDbId(options.dbname) args['unix_socket_path'] = None elif options.conntype == "unix_socket": args['host'] = None args['port'] = None - args['db'] = SonicDBConfig.get_dbid(options.dbname) - args['unix_socket_path'] = SonicDBConfig.get_socket(options.dbname) + args['db'] = SonicDBConfig.getDbId(options.dbname) + args['unix_socket_path'] = SonicDBConfig.getDbSock(options.dbname) else: raise TypeError('redis connection type is tcp or unix_socket') From 5f586057833b5696e9c3369b4041286249c5b240 Mon Sep 17 00:00:00 2001 From: liuh-80 Date: Mon, 26 Sep 2022 07:21:26 +0000 Subject: [PATCH 10/10] Revert swsssdk change --- .gitmodules | 3 +-- src/sonic-py-swsssdk | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index 0ef8c88c4a5d..09ebc6da2373 100644 --- a/.gitmodules +++ b/.gitmodules @@ -21,8 +21,7 @@ url = https://github.com/sonic-net/sonic-dbsyncd [submodule "src/sonic-py-swsssdk"] path = src/sonic-py-swsssdk - url = https://github.com/liuh-80/sonic-py-swsssdk.git - branch = /dev/liuh/disable-sdk + url = https://github.com/sonic-net/sonic-py-swsssdk.git [submodule "src/sonic-snmpagent"] path = src/sonic-snmpagent url = https://github.com/sonic-net/sonic-snmpagent diff --git a/src/sonic-py-swsssdk b/src/sonic-py-swsssdk index 2dc0dabf8ec2..cc847a2e11b3 160000 --- a/src/sonic-py-swsssdk +++ b/src/sonic-py-swsssdk @@ -1 +1 @@ -Subproject commit 2dc0dabf8ec2ca5e5902cb819becb019a47cff53 +Subproject commit cc847a2e11b3dd6d7f9630bfcac2ac3af2912eba