[pytest] Test the feature of monitoring critical processes by Supervisord.

tests/common/devices.py

@@ -643,6 +643,43 @@ def get_swss_docker_names(self):
                 swss_docker_names.append("swss{}".format(asic))
         return swss_docker_names
 
+    def get_namespace_ids(self, container_name):
+        """
+        Gets ids of namespace where the container should reside in.
+
+        Returns:
+            A list contains ids of namespace such as [DEFAULT_ASIC_ID, "0", "1", ...]}
+        """
+        has_global_scope = ""
+        has_per_asic_scope = ""
+        namespace_ids = []
+
+        num_asics = int(self.facts["num_asic"])
+        command_config_entry = "sonic-db-cli CONFIG_DB hgetall \"FEATURE|{}\"".format(container_name)
+        command_output = self.shell(command_config_entry)
+        exit_code = command_output["rc"]
+        if exit_code != 0:
+            return namespace_ids, False
+
+        config_info = command_output["stdout_lines"]
+        for index, item in enumerate(config_info):
+            if item == "has_global_scope":
+                has_global_scope = config_info[index + 1]
+            elif item == "has_per_asic_scope":
+                has_per_asic_scope = config_info[index + 1]
+
+        if num_asics > 1:
+            if has_global_scope == "True":
+                namespace_ids.append(DEFAULT_ASIC_ID)
+            if has_per_asic_scope == "True":
+                for asic_id in range(0, num_asics):
+                    namespace_ids.append(str(asic_id))
+        else:
+            namespace_ids.append(DEFAULT_ASIC_ID)
+
+        return namespace_ids, True
+
+
     def get_up_time(self):
         up_time_text = self.command("uptime -s")["stdout"]
         return datetime.strptime(up_time_text, "%Y-%m-%d %H:%M:%S")

tests/common/helpers/dut_utils.py

@@ -1,5 +1,8 @@
+import logging
+
 from tests.common.utilities import get_host_visible_vars
 
+logger = logging.getLogger(__name__)
 
 def is_supervisor_node(inv_files, hostname):
     """Check if the current node is a supervisor node in case of multi-DUT.
@@ -27,3 +30,72 @@ def is_frontend_node(inv_files, hostname):
           node. If we add more types of nodes, then we need to exclude them from this method as well.
     """
     return not is_supervisor_node(inv_files, hostname)
+
+
+def get_group_program_info(duthost, container_name, group_name):
+    """Gets program names, running status and their pids by analyzing the command
+       output of "docker exec <container_name> supervisorctl status". Program name
+       at here represents a program which is part of group <group_name>
+
+    Args:
+        duthost: Hostname of DUT.
+        container_name: A string shows container name.
+        program_name: A string shows process name.
+
+    Returns:
+        A dictionary where keys are the program names and values are their running
+        status and pids.
+    """
+    group_program_info = defaultdict(list)
+    program_name = None
+    program_status = None
+    program_pid = None
+
+    program_list = duthost.shell("docker exec {} supervisorctl status".format(container_name), module_ignore_errors=True)
+    for program_info in program_list["stdout_lines"]:
+        if program_info.find(group_name) != -1:
+            program_name = program_info.split()[0].split(':')[1].strip()
+            program_status = program_info.split()[1].strip()
+            if program_status in ["EXITED", "STOPPED", "STARTING"]:
+                program_pid = -1
+            else:
+                program_pid = int(program_info.split()[3].strip(','))
+
+            group_program_info[program_name].append(program_status)
+            group_program_info[program_name].append(program_pid)
+
+            if program_pid != -1:
+                logger.info("Found program '{}' in the '{}' state with pid {}"
+                            .format(program_name, program_status, program_pid))
+
+    return group_program_info
+
+
+def get_program_info(duthost, container_name, program_name):
+    """Gets program running status and its pid by analyzing the command
+       output of "docker exec <container_name> supervisorctl status"
+
+    Args:
+        duthost: Hostname of DUT.
+        container_name: A string shows container name.
+        program_name: A string shows process name.
+
+    Return:
+        Program running status and its pid.
+    """
+    program_status = None
+    program_pid = -1
+
+    program_list = duthost.shell("docker exec {} supervisorctl status".format(container_name), module_ignore_errors=True)
+    for program_info in program_list["stdout_lines"]:
+        if program_info.find(program_name) != -1:
+            program_status = program_info.split()[1].strip()
+            if program_status == "RUNNING":
+                program_pid = int(program_info.split()[3].strip(','))
+            break
+
+    if program_pid != -1:
+        logger.info("Found program '{}' in the '{}' state with pid {}"
+                    .format(program_name, program_status, program_pid))
+
+    return program_status, program_pid

tests/kvmtest.sh

@@ -124,7 +124,8 @@ test_t0() {
     test_procdockerstatsd.py \
     iface_namingmode/test_iface_namingmode.py \
     platform_tests/test_cpu_memory_usage.py \
-    bgp/test_bgpmon.py"
+    bgp/test_bgpmon.py \
+    process_monitoring/test_critical_process_monitoring.py"
 
     pushd $SONIC_MGMT_DIR/tests
     ./run_tests.sh $RUNTEST_CLI_COMMON_OPTS -c "$tests" -p logs/$tgname
@@ -154,7 +155,8 @@ test_t1_lag() {
     lldp/test_lldp.py \
     route/test_default_route.py \
     platform_tests/test_cpu_memory_usage.py \
-    bgp/test_bgpmon.py"
+    bgp/test_bgpmon.py \
+    process_monitoring/test_critical_process_monitoring.py"
 
     pushd $SONIC_MGMT_DIR/tests
     ./run_tests.sh $RUNTEST_CLI_COMMON_OPTS -c "$tests" -p logs/$tgname