[chassis]: remote cli commands infra for sonic chassis (#2850)

What I did
Microsoft ADO 17792956

Since each Linecard is running an independent SONiC Instance, the user needs to login to a linecard to run any CLI command
The user can login to each Linecard 2 ways

Ssh directly to the linecard using the management IP address
Ssh to supervisor and from supervisor ssh to the Linecard using the Linecard’s internal IP address
To simplify the user experience and allow scripting agents to execute commands on all linecards.
Two new commands are being added
rexec <linecard_name|all> -c <cli_command> This command will execute the command on specified linecards or all linecards.

rshell <linecard_name> connects to the linecard for interactive shell

This PR is adding the changes of PR #2701

How to verify it
UT and tested chassis

Signed-off-by: Arvindsrinivasan Lakshmi Narasimhan <arlakshm@microsoft.com>

rcli/linecard.py

@@ -0,0 +1,151 @@
+import click
+import os
+import paramiko
+import sys
+import select
+import socket
+import sys
+import termios
+import tty
+
+from .utils import get_linecard_ip
+from paramiko.py3compat import u
+from paramiko import Channel
+
+EMPTY_OUTPUTS = ['', '\x1b[?2004l\r']
+
+class Linecard:
+
+    def __init__(self, linecard_name, username, password):
+        """
+        Initialize Linecard object and store credentials, connection, and channel
+        
+        :param linecard_name: The name of the linecard you want to connect to
+        :param username: The username to use to connect to the linecard
+        :param password: The linecard password. If password not provided, it 
+            will prompt the user for it
+        :param use_ssh_keys: Whether or not to use SSH keys to authenticate.
+        """
+        self.ip = get_linecard_ip(linecard_name)
+
+        if not self.ip:
+            sys.exit(1)
+
+        self.linecard_name = linecard_name
+        self.username = username
+        self.password = password
+
+        self.connection = self._connect()
+
+
+    def _connect(self):
+        connection = paramiko.SSHClient()
+        # if ip address not in known_hosts, ignore known_hosts error
+        connection.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        try:
+            connection.connect(self.ip, username=self.username, password=self.password)
+        except paramiko.ssh_exception.NoValidConnectionsError as e:
+            connection = None
+            click.echo(e)
+        return connection
+
+    def _get_password(self):
+        """
+        Prompts the user for a password, and returns the password
+        
+        :param username: The username that we want to get the password for
+        :type username: str
+        :return: The password for the username.
+        """
+
+        return getpass(
+            "Password for username '{}': ".format(self.username),
+            # Pass in click stdout stream - this is similar to using click.echo
+            stream=click.get_text_stream('stdout')
+        )
+
+    def _set_tty_params(self):
+        tty.setraw(sys.stdin.fileno())
+        tty.setcbreak(sys.stdin.fileno())
+
+    def _is_data_to_read(self, read):
+        if self.channel in read:
+            return True
+        return False
+
+    def _is_data_to_write(self, read):
+        if sys.stdin in read:
+            return True
+        return False
+
+    def _write_to_terminal(self, data):
+        # Write channel output to terminal
+        sys.stdout.write(data)
+        sys.stdout.flush() 
+
+    def _start_interactive_shell(self):
+        oldtty = termios.tcgetattr(sys.stdin)
+        try:
+            self._set_tty_params()
+            self.channel.settimeout(0.0)
+
+            while True:
+                #Continuously wait for commands and execute them
+                read, write, ex = select.select([self.channel, sys.stdin], [], [])
+                if self._is_data_to_read(read):
+                    try:
+                        # Get output from channel
+                        x = u(self.channel.recv(1024))
+                        if len(x) == 0:
+                            # logout message will be displayed
+                            break
+                        self._write_to_terminal(x)
+                    except socket.timeout as e:
+                        click.echo("Connection timed out")
+                        break
+                if self._is_data_to_write(read):
+                    # If we are able to send input, get the input from stdin
+                    x = sys.stdin.read(1)
+                    if len(x) == 0:
+                        break
+                    # Send the input to the channel
+                    self.channel.send(x)
+        finally:
+            # Now that the channel has been exited, return to the previously-saved old tty
+            termios.tcsetattr(sys.stdin, termios.TCSADRAIN, oldtty)
+            pass
+
+
+    def start_shell(self) -> None:
+            """
+            Opens a session, gets a pseudo-terminal, invokes a shell, and then 
+            attaches the host shell to the remote shell.
+            """
+            # Create shell session
+            self.channel = self.connection.get_transport().open_session()
+            self.channel.get_pty()
+            self.channel.invoke_shell()
+            # Use Paramiko Interactive script to connect to the shell
+            self._start_interactive_shell()
+            # After user exits interactive shell, close the connection
+            self.connection.close()
+
+
+    def execute_cmd(self, command) -> str:
+        """
+        Takes a command as an argument, executes it on the remote shell, and returns the output
+        
+        :param command: The command to execute on the remote shell
+        :return: The output of the command.
+        """
+        # Execute the command and gather errors and output
+        _, stdout, stderr = self.connection.exec_command(command + "\n")
+        output = stdout.read().decode('utf-8')
+
+        if stderr:
+            # Error was present, add message to output
+            output += stderr.read().decode('utf-8')
+
+        # Close connection and return output
+        self.connection.close()
+        return output

rcli/rexec.py

@@ -0,0 +1,44 @@
+import os
+import click
+import paramiko
+import sys
+
+from .linecard import Linecard
+from rcli import utils as rcli_utils
+from sonic_py_common import device_info
+
+@click.command()
+@click.argument('linecard_names', nargs=-1, type=str, required=True)
+@click.option('-c', '--command', type=str, required=True)
+def cli(linecard_names, command):
+    """
+    Executes a command on one or many linecards
+    
+    :param linecard_names: A list of linecard names to execute the command on, 
+        use `all` to execute on all linecards.
+    :param command: The command to execute on the linecard(s)
+    """
+    if not device_info.is_chassis():
+        click.echo("This commmand is only supported Chassis")
+        sys.exit(1)
+
+    username = os.getlogin()
+    password = rcli_utils.get_password(username)
+
+    if list(linecard_names) == ["all"]:
+        # Get all linecard names using autocompletion helper
+        linecard_names = rcli_utils.get_all_linecards(None, None, "")
+
+    # Iterate through each linecard, execute command, and gather output
+    for linecard_name in linecard_names:
+        try:
+            lc = Linecard(linecard_name, username, password)
+            if lc.connection:
+                # If connection was created, connection exists. Otherwise, user will see an error message.
+                click.echo("======== {} output: ========".format(lc.linecard_name))
+                click.echo(lc.execute_cmd(command))
+        except paramiko.ssh_exception.AuthenticationException:
+            click.echo("Login failed on '{}' with username '{}'".format(linecard_name, lc.username))
+
+if __name__=="__main__":
+    cli(prog_name='rexec')

rcli/rshell.py

@@ -0,0 +1,38 @@
+import os
+import click
+import paramiko
+import sys
+
+from .linecard import Linecard
+from sonic_py_common import device_info
+from rcli import utils as rcli_utils
+
+
+@click.command()
+@click.argument('linecard_name', type=str, autocompletion=rcli_utils.get_all_linecards)
+def cli(linecard_name):
+    """
+    Open interactive shell for one linecard
+    
+    :param linecard_name: The name of the linecard to connect to
+    """
+    if not device_info.is_chassis():
+        click.echo("This commmand is only supported Chassis")
+        sys.exit(1)
+
+    username = os.getlogin()
+    password = rcli_utils.get_password(username)
+
+    try:
+        lc =Linecard(linecard_name, username, password)
+        if lc.connection:
+            click.echo("Connecting to {}".format(lc.linecard_name))
+            # If connection was created, connection exists. Otherwise, user will see an error message.
+            lc.start_shell()
+            click.echo("Connection Closed")
+    except paramiko.ssh_exception.AuthenticationException:
+        click.echo("Login failed on '{}' with username '{}'".format(linecard_name, lc.username))
+
+
+if __name__=="__main__":
+    cli(prog_name='rshell')

rcli/utils.py

@@ -0,0 +1,149 @@
+import click
+from getpass import getpass
+import os
+import sys
+
+from swsscommon.swsscommon import SonicV2Connector
+
+CHASSIS_MODULE_INFO_TABLE = 'CHASSIS_MODULE_TABLE'
+CHASSIS_MODULE_INFO_KEY_TEMPLATE = 'CHASSIS_MODULE {}'
+CHASSIS_MODULE_INFO_DESC_FIELD = 'desc'
+CHASSIS_MODULE_INFO_SLOT_FIELD = 'slot'
+CHASSIS_MODULE_INFO_OPERSTATUS_FIELD = 'oper_status'
+CHASSIS_MODULE_INFO_ADMINSTATUS_FIELD = 'admin_status'
+
+CHASSIS_MIDPLANE_INFO_TABLE = 'CHASSIS_MIDPLANE_TABLE'
+CHASSIS_MIDPLANE_INFO_IP_FIELD = 'ip_address'
+CHASSIS_MIDPLANE_INFO_ACCESS_FIELD = 'access'
+
+CHASSIS_MODULE_HOSTNAME_TABLE = 'CHASSIS_MODULE_HOSTNAME_TABLE'
+CHASSIS_MODULE_HOSTNAME = 'module_hostname'
+
+def connect_to_chassis_state_db():
+    chassis_state_db = SonicV2Connector(host="127.0.0.1")
+    chassis_state_db.connect(chassis_state_db.CHASSIS_STATE_DB)
+    return chassis_state_db
+
+
+def connect_state_db():
+    state_db = SonicV2Connector(host="127.0.0.1")
+    state_db.connect(state_db.STATE_DB)
+    return state_db
+
+
+
+def get_linecard_module_name_from_hostname(linecard_name: str):
+
+    chassis_state_db = connect_to_chassis_state_db()
+
+    keys = chassis_state_db.keys(chassis_state_db.CHASSIS_STATE_DB , '{}|{}'.format(CHASSIS_MODULE_HOSTNAME_TABLE, '*'))
+    for key in keys:
+        module_name = key.split('|')[1]
+        hostname = chassis_state_db.get(chassis_state_db.CHASSIS_STATE_DB, key, CHASSIS_MODULE_HOSTNAME)
+        if hostname.replace('-', '').lower() == linecard_name.replace('-', '').lower():
+             return module_name
+
+    return None
+
+def get_linecard_ip(linecard_name: str):
+    """
+    Given a linecard name, lookup its IP address in the midplane table
+    
+    :param linecard_name: The name of the linecard you want to connect to
+    :type linecard_name: str
+    :return: IP address of the linecard
+    """
+    # Adapted from `show chassis modules midplane-status` command logic:
+    # https://github.com/sonic-net/sonic-utilities/blob/master/show/chassis_modules.py
+
+    # if the user passes linecard hostname, then try to get the module name for that linecard
+    module_name = get_linecard_module_name_from_hostname(linecard_name)
+    # if the module name cannot be found from host, assume the user has passed module name
+    if module_name is None:
+        module_name = linecard_name
+    module_ip, module_access = get_module_ip_and_access_from_state_db(module_name)
+
+    if not module_ip:
+        click.echo('Linecard {} not found'.format(linecard_name))
+        return None
+
+    if module_access != 'True':
+        click.echo('Linecard {} not accessible'.format(linecard_name))
+        return None
+
+
+    return module_ip
+
+def get_module_ip_and_access_from_state_db(module_name):
+    state_db = connect_state_db()
+    data_dict = state_db.get_all(
+        state_db.STATE_DB, '{}|{}'.format(CHASSIS_MIDPLANE_INFO_TABLE,module_name ))
+    if data_dict is None:
+        return None, None
+
+    linecard_ip = data_dict.get(CHASSIS_MIDPLANE_INFO_IP_FIELD, None)
+    access = data_dict.get(CHASSIS_MIDPLANE_INFO_ACCESS_FIELD, None)
+
+    return linecard_ip, access
+
+
+def get_all_linecards(ctx, args, incomplete) -> list:
+    """
+    Return a list of all accessible linecard names. This function is used to 
+    autocomplete linecard names in the CLI.
+    
+    :param ctx: The Click context object that is passed to the command function
+    :param args: The arguments passed to the Click command
+    :param incomplete: The string that the user has typed so far
+    :return: A list of all accessible linecard names.
+    """
+    # Adapted from `show chassis modules midplane-status` command logic:
+    # https://github.com/sonic-net/sonic-utilities/blob/master/show/chassis_modules.py
+
+
+    chassis_state_db = connect_to_chassis_state_db()
+    state_db = connect_state_db()
+
+    linecards = []
+    keys = state_db.keys(state_db.STATE_DB,'{}|*'.format(CHASSIS_MIDPLANE_INFO_TABLE))
+    for key in keys:
+        key_list = key.split('|')
+        if len(key_list) != 2:  # error data in DB, log it and ignore
+            click.echo('Warn: Invalid Key {} in {} table'.format(key, CHASSIS_MIDPLANE_INFO_TABLE ))
+            continue
+        module_name = key_list[1]
+        linecard_ip, access = get_module_ip_and_access_from_state_db(module_name)
+        if linecard_ip is None:
+            continue
+
+        if access != "True" :
+            continue
+
+        # get the hostname for this module
+        hostname = chassis_state_db.get(chassis_state_db.CHASSIS_STATE_DB, '{}|{}'.format(CHASSIS_MODULE_HOSTNAME_TABLE, module_name), CHASSIS_MODULE_HOSTNAME)
+        if hostname:
+            linecards.append(hostname)
+        else:
+            linecards.append(module_name)
+
+    # Return a list of all matched linecards
+    return [lc for lc in linecards if incomplete in lc]
+
+
+def get_password(username=None):
+    """
+    Prompts the user for a password, and returns the password
+    
+    :param username: The username that we want to get the password for
+    :type username: str
+    :return: The password for the username.
+    """
+
+    if username is None:
+        username =os.getlogin()
+
+    return getpass(
+        "Password for username '{}': ".format(username),
+        # Pass in click stdout stream - this is similar to using click.echo
+        stream=click.get_text_stream('stdout')
+    )