From 7849e18de8fb7ba5ec650a5f8ec620e81d2bcc65 Mon Sep 17 00:00:00 2001
From: Stepan Blyshchak <38952541+stepanblyschak@users.noreply.github.com>
Date: Tue, 7 Feb 2023 18:07:52 +0200
Subject: [PATCH] [db_migrator] make LOG_LEVEL_DB migration more robust (#2651)

It could be that LOG_LEVEL_DB includes some invalid data and/or a KEY_SET that is not cleaned up due to an issue, for example we observed _gearsyncd_KEY_SET set included in the LOG_LEVEL_DB and preserved in warm reboot. However, this key is not of type hash which leads to an exception and migration failure. The migration logic should be more robust allowing users to upgrade even though some daemon has left overs in the LOG_LEVEL_DB or invalid data is written.

- What I did
To fix migration issue that leads to device configuration being lost.

- How I did it
Wrap the logic in try/except/finally.

- How to verify it
202205 -> 202211/master upgrade.

Signed-off-by: Stepan Blyschak <stepanb@nvidia.com>
---
 scripts/db_migrator.py                        | 20 +++++++++++--------
 .../loglevel_db/logger_tables_input.json      |  7 +++++--
 2 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/scripts/db_migrator.py b/scripts/db_migrator.py
index 6c8ef21b6f..c52e38bd63 100755
--- a/scripts/db_migrator.py
+++ b/scripts/db_migrator.py
@@ -822,14 +822,18 @@ def version_3_0_5(self):
             keys = self.loglevelDB.keys(self.loglevelDB.LOGLEVEL_DB, "*")
             if keys is not None:
                 for key in keys:
-                    if key != "JINJA2_CACHE":
-                        fvs = self.loglevelDB.get_all(self.loglevelDB.LOGLEVEL_DB, key)
-                        component = key.split(":")[1]
-                        loglevel = fvs[loglevel_field]
-                        logoutput = fvs[logoutput_field]
-                        self.configDB.set(self.configDB.CONFIG_DB, '{}|{}'.format(table_name, component), loglevel_field, loglevel)
-                        self.configDB.set(self.configDB.CONFIG_DB, '{}|{}'.format(table_name, component), logoutput_field, logoutput)
-                    self.loglevelDB.delete(self.loglevelDB.LOGLEVEL_DB, key)
+                    try:
+                        if key != "JINJA2_CACHE":
+                            fvs = self.loglevelDB.get_all(self.loglevelDB.LOGLEVEL_DB, key)
+                            component = key.split(":")[1]
+                            loglevel = fvs[loglevel_field]
+                            logoutput = fvs[logoutput_field]
+                            self.configDB.set(self.configDB.CONFIG_DB, '{}|{}'.format(table_name, component), loglevel_field, loglevel)
+                            self.configDB.set(self.configDB.CONFIG_DB, '{}|{}'.format(table_name, component), logoutput_field, logoutput)
+                    except Exception as err:
+                        log.log_warning('Error occured during LOGLEVEL_DB migration for {}. Ignoring key {}'.format(err, key))
+                    finally:
+                        self.loglevelDB.delete(self.loglevelDB.LOGLEVEL_DB, key)
         self.set_version('version_3_0_6')
         return 'version_3_0_6'
 
diff --git a/tests/db_migrator_input/loglevel_db/logger_tables_input.json b/tests/db_migrator_input/loglevel_db/logger_tables_input.json
index 02377ea0a4..ed1bc8057f 100644
--- a/tests/db_migrator_input/loglevel_db/logger_tables_input.json
+++ b/tests/db_migrator_input/loglevel_db/logger_tables_input.json
@@ -7,5 +7,8 @@
         "LOGLEVEL": "SAI_LOG_LEVEL_NOTICE",
         "LOGOUTPUT": "SYSLOG"
     },
-    "JINJA2_CACHE": {}
-}
\ No newline at end of file
+    "JINJA2_CACHE": {},
+    "INVALID:INVALID": {
+        "invalid": "invalid"
+    }
+}