Enable delfin installer to run multiple instances of delfin components

docker-compose.yml

@@ -35,6 +35,14 @@
 # 4. When finished using delfin project, bring down containers using following command
 #
 #    $ docker-compose down
+#
+# 5. To bring up delfin project with multiple service instances
+#
+#    $ docker-compose up -d --scale <<service-name>>=<<number of instances>>
+#
+#    example: Deploy delfin with 3 delfin-task and 2 delfin-alert instances
+#       $ docker-compose up -d --scale delfin-task=3 --scale delfin-alert=2
+#    Note: Multiple instances of delfin-api are not allowed
 
 version: '3.3'
 

installer/README.md

@@ -115,6 +115,22 @@ installer/install
 root@root1:~/delfin-demo/delfin$ installer/install
 ```
 
+#### Configure multiple instances of delfin components
+Respective environment variable required to set for running multiple instances 
+of delfin component before executing install command
+
+```sh
+$ export DELFIN_<<delfin component name>>_INSTANCES=<<number of instances>>
+$ installer/install
+
+# Example: Deploy delfin with 3 task and 2 alert instances 
+  $ export DELFIN_TASK_INSTANCES=3
+  $ export DELFIN_ALERT_INSTANCES=2
+  $ installer/install
+```
+
+Note: Multiple instances of exporter and api is not allowed currently.
+
 # Uninstall
 Running the uninstall script will stop all delfin processes and do cleanup
 ```sh

installer/install_delfin.py

@@ -16,8 +16,9 @@
 
 import os
 import subprocess
-from subprocess import CalledProcessError
 import traceback as tb
+from subprocess import CalledProcessError
+
 from installer.helper import copy_files, create_dir, \
     logger, logfile, delfin_log_dir, create_file
 
@@ -53,42 +54,45 @@ def create_delfin_db():
 
 
 def start_processes():
-    # start api process
-    proc_path = os.path.join(delfin_source_path, 'delfin', 'cmd', 'api.py')
-    command = 'python3 ' + proc_path + ' --config-file ' +\
-              conf_file + ' >' + DEVNULL + ' 2>&1 &'
-    # >/dev/null 2>&1
-    logger.info("Executing command [%s]", command)
-    os.system(command)
-    logger.info("API process_started")
-
-    # Start task process
-    proc_path = os.path.join(delfin_source_path, 'delfin', 'cmd', 'task.py')
-    command = 'python3 ' + proc_path + ' --config-file ' +\
-              conf_file + ' >' + DEVNULL + ' 2>&1 &'
-    logger.info("Executing command [%s]", command)
-    os.system(command)
-
-    logger.info("TASK process_started")
-
-    # Start alert process
-    proc_path = os.path.join(delfin_source_path, 'delfin', 'cmd', 'alert.py')
-    command = 'python3 ' + proc_path + ' --config-file ' +\
-              conf_file + ' >' + DEVNULL + ' 2>&1 &'
-    logger.info("Executing command [%s]", command)
-    os.system(command)
-    logger.info("ALERT process_started")
+    processes = ['api', 'task', 'alert']
+    # Start cmd processes
+    for process in processes:
+        env_var = 'DELFIN_' + process.upper() + '_INSTANCES'
+        try:
+            instances = os.environ.get(env_var)
+            # Ignore multiple instance of api
+            if not instances or process == 'api':
+                instances = '1'
+            start_process(process, int(instances))
+        except CalledProcessError as cpe:
+            logger.error("Got CPE error [%s]:[%s]" % (cpe, tb.print_exc()))
+            return
+        except ValueError as e:
+            logger.error(
+                "Got invalid [%s] environment variable:[%s]" % (env_var, e))
+            return
 
     # Start exporter server process
     proc_path = os.path.join(delfin_source_path, 'delfin', 'exporter',
                              'prometheus', 'exporter_server.py')
-    command = 'python3 ' + proc_path + ' --config-file ' +\
+    command = 'python3 ' + proc_path + ' --config-file ' + \
               conf_file + ' >' + DEVNULL + ' 2>&1 &'
     logger.info("Executing command [%s]", command)
     os.system(command)
     logger.info("Exporter process_started")
 
 
+def start_process(process, instances=1):
+    for instance in range(0, instances):
+        proc_path = os.path.join(delfin_source_path, 'delfin', 'cmd',
+                                 process + '.py')
+        command = 'python3 ' + proc_path + ' --config-file ' + \
+                  conf_file + ' >' + DEVNULL + ' 2>&1 &'
+        logger.info("Executing command [%s]", command)
+        os.system(command)
+        logger.info("[%s] process_started", process)
+
+
 def install_delfin():
     python_setup_comm = ['build', 'install']
     req_logs = os.path.join(delfin_log_dir, 'requirements.log')
@@ -99,7 +103,7 @@ def install_delfin():
     setup_file = os.path.join(delfin_source_path, 'setup.py')
     for command in python_setup_comm:
         try:
-            command = 'python3 ' + setup_file + ' ' +\
+            command = 'python3 ' + setup_file + ' ' + \
                       command + ' >>' + logfile
             logger.info("Executing [%s]", command)
             os.system(command)