[MultiDB] add MultiDB warmboot support - backing up database (#1205)

* lua script to backup all database into one database and create rdb file - following earlier multiDB warmboot design at https://github.com/Azure/SONiC/blob/master/doc/database/multi_database_instances.md
* copy this rdb file as before to WARM_DIR
* restoring database part is in another PR at sonic-buildimage  sonic-net/sonic-buildimage#5773, they depend on each other

scripts/centralize_database

@@ -0,0 +1,62 @@
+#!/usr/bin/python
+from __future__ import print_function
+import sys
+import swsssdk
+import redis
+import argparse
+
+def centralize_to_target_db(target_dbname):
+    target_dbport = swsssdk.SonicDBConfig.get_port(target_dbname)
+    target_dbhost = swsssdk.SonicDBConfig.get_hostname(target_dbname)
+
+    dblists = swsssdk.SonicDBConfig.get_dblist()
+    for dbname in dblists:
+        dbport = swsssdk.SonicDBConfig.get_port(dbname)
+        dbhost = swsssdk.SonicDBConfig.get_hostname(dbname)
+        # if the db is on the same instance, no need to move
+        if dbport == target_dbport and dbhost == target_dbhost:
+            continue
+
+        dbsocket = swsssdk.SonicDBConfig.get_socket(dbname)
+        dbid = swsssdk.SonicDBConfig.get_dbid(dbname)
+
+        r = redis.Redis(host=dbhost, unix_socket_path=dbsocket, db=dbid)
+
+        script = """
+            local cursor = 0;
+            repeat
+                local  dat = redis.call('SCAN', cursor, 'COUNT', 7000);
+                     cursor = dat[1];
+                redis.call('MIGRATE', KEYS[1], KEYS[2], '', KEYS[3], 5000, 'COPY', 'REPLACE', 'KEYS', unpack(dat[2]));
+            until cursor == '0';
+        """
+        r.eval(script, 3, target_dbhost, target_dbport, dbid)
+
+    #SAVE rdb file
+    r = redis.Redis(host=target_dbhost, port=target_dbport)
+    r.save()
+
+def main():
+    parser = argparse.ArgumentParser(description='centralize all db data into one db instances',
+                                     formatter_class=argparse.RawTextHelpFormatter,
+                                     epilog=
+"""
+Example : centralize_database APPL_DB
+""")
+    parser.add_argument('target_db', type=str, help='move all db data into the instance where target db locates')
+    args = parser.parse_args()
+
+    if args.target_db:
+        try:
+            centralize_to_target_db(args.target_db)
+            print(swsssdk.SonicDBConfig.get_instancename(args.target_db))
+        except Exception as ex:
+            template = "An exception of type {0} occurred. Arguments:\n{1!r}"
+            message = template.format(type(ex).__name__, ex.args)
+            print(message, file=sys.stderr)
+            sys.exit(1)
+    else:
+        parser.print_help()
+
+if __name__ == "__main__":
+    main()

scripts/fast-reboot

@@ -247,10 +247,13 @@ function backup_database()
             end
         end
     " 0 > /dev/null
-    sonic-db-cli SAVE > /dev/null
-    #TODO : need a script to copy all rdb files if there is multiple db instances config
-    docker cp database:/var/lib/redis/$REDIS_FILE $WARM_DIR
-    docker exec -i database rm /var/lib/redis/$REDIS_FILE
+
+    # move all db data into the instance where APPL_DB locates
+    target_db_inst=`centralize_database APPL_DB`
+
+    # Dump redis content to a file 'dump.rdb' in warmboot directory
+    docker cp database:/var/lib/$target_db_inst/$REDIS_FILE $WARM_DIR
+    docker exec -i database rm /var/lib/$target_db_inst/$REDIS_FILE
 }
 
 function setup_control_plane_assistant()

setup.py

@@ -110,7 +110,8 @@
         'scripts/warm-reboot',
         'scripts/watermarkstat',
         'scripts/watermarkcfg',
-        'scripts/sonic-kdump-config'
+        'scripts/sonic-kdump-config',
+        'scripts/centralize_database'
     ],
     entry_points={
         'console_scripts': [