working on idaholab#266, Malcolm ISO should format bigger drives for …

…index and artifact storage, NOT DONE YET

docs/malcolm-hedgehog-e2e-iso-install.md

@@ -169,7 +169,7 @@ The [configuration and tuning](malcolm-config.md#ConfigAndTuning) wizard's quest
     - This question allows users to specify Microsoft Active Directory compatibility (**winldap**) or generic LDAP compatibility (**openldap**, for OpenLDAP, glauth, etc.) when using [LDAP authentication](authsetup.md#AuthLDAP)
 * **Use StartTLS (rather than LDAPS) for LDAP connection security?**
     - When using LDAP authentication, this question allows users to configure [LDAP connection security](authsetup.md#AuthLDAPSecurity)
-* **Store PCAP, log and index files locally under /home/user/Malcolm?**
+* **Store PCAP, log and index files in /home/user/Malcolm?**
     - Malcolm generates a number of large file sets during normal operation: PCAP files, Zeek or Suricata logs, OpenSearch indices, etc. By default all of these are stored in subdirectories in the Malcolm installation directory. This question allows users to specify alternative storage location(s) (for example, a separate dedicated drive or RAID volume) for these artifacts.
 * **Enable index management policies (ILM/ISM) in Arkime?**
     - Choose **Y** to proceed to the following related questions about [using ILM/ISM with Arkime](index-management.md#ArkimeIndexPolicies).

docs/ubuntu-install-example.md

@@ -130,7 +130,7 @@ Specify external Docker network name (or leave blank for default networking) ():
 3: None
 Select authentication method (Basic): 1
 
-Store PCAP, log and index files locally under /home/user/Malcolm? (Y / n): y
+Store PCAP, log and index files in /home/user/Malcolm? (Y / n): y
 
 Enable index management policies (ILM/ISM) in Arkime? (y / N): n
 

scripts/install.py

@@ -71,9 +71,13 @@
     DatabaseMode,
     DATABASE_MODE_LABELS,
     DATABASE_MODE_ENUMS,
+    MALCOLM_DB_DIR,
+    MALCOLM_PCAP_DIR,
+    MALCOLM_LOGS_DIR,
     deep_get,
     eprint,
     flatten,
+    LoadFileIfJson,
     run_process,
     same_file_or_dir,
     str2bool,
@@ -857,40 +861,92 @@ def tweak_malcolm_runtime(self, malcolm_install_path):
                 pass
 
         # directories for data volume mounts (PCAP storage, Zeek log storage, OpenSearch indexes, etc.)
-        indexDir = './opensearch'
-        indexDirDefault = os.path.join(malcolm_install_path, indexDir)
+
+        # if the file .os_disk_config_defaults was created by the environment (os-disk-config.py)
+        #   we'll use those as defaults, otherwise base things underneath the malcolm_install_path
+        diskFormatInfo = {}
+        try:
+            diskFormatInfoFile = os.path.join(
+                os.path.realpath(os.path.join(ScriptPath, "..")), ".os_disk_config_defaults"
+            )
+            if os.path.isfile(diskFormatInfoFile):
+                with open(diskFormatInfoFile) as f:
+                    diskFormatInfo = LoadFileIfJson(f)
+        except Exception:
+            pass
+        diskFormatInfo = {k: v for k, v in diskFormatInfo.iteritems() if os.path.isdir(v)}
+
+        if MALCOLM_DB_DIR in diskFormatInfo:
+            for subDir in ['opensearch', 'opensearch-backup']: do
+                pathlib.Path(os.path.join(diskFormatInfo[MALCOLM_DB_DIR], subDir)).mkdir(parents=False, exist_ok=True)
+        if MALCOLM_LOGS_DIR in diskFormatInfo:
+            for subDir in ['zeek-logs', 'suricata-logs']: do
+                pathlib.Path(os.path.join(diskFormatInfo[MALCOLM_LOGS_DIR], subDir)).mkdir(parents=False, exist_ok=True)
+
+        if args.indexDir:
+            indexDirDefault = args.indexDir
+        else:
+            indexDir = 'opensearch'
+            if (MALCOLM_DB_DIR in diskFormatInfo) and os.path.isdir(os.path.join(diskFormatInfo[MALCOLM_DB_DIR], indexDir)):
+                indexDirDefault = os.path.join(diskFormatInfo[MALCOLM_DB_DIR], indexDir)
+            else:
+                indexDirDefault = os.path.join(malcolm_install_path, indexDir)
         indexDirFull = os.path.realpath(indexDirDefault)
 
         indexSnapshotCompressed = False
-        indexSnapshotDir = './opensearch-backup'
-        indexSnapshotDirDefault = os.path.join(malcolm_install_path, indexSnapshotDir)
+        if args.indexSnapshotDir:
+            indexSnapshotDirDefault = args.indexSnapshotDir
+        else
+            indexSnapshotDir = 'opensearch-backup'
+            if (MALCOLM_DB_DIR in diskFormatInfo) and os.path.isdir(os.path.join(diskFormatInfo[MALCOLM_DB_DIR], indexSnapshotDir)):
+                indexSnapshotDirDefault = os.path.join(diskFormatInfo[MALCOLM_DB_DIR], indexSnapshotDir)
+            else:
+                indexSnapshotDirDefault = os.path.join(malcolm_install_path, indexSnapshotDir)
         indexSnapshotDirFull = os.path.realpath(indexSnapshotDirDefault)
 
-        pcapDir = './pcap'
-        pcapDirDefault = os.path.join(malcolm_install_path, pcapDir)
+        if args.pcapDir:
+            pcapDirDefault = args.pcapDir
+        else:
+            pcapDir = 'pcap'
+            if (MALCOLM_PCAP_DIR in diskFormatInfo):
+                pcapDirDefault = diskFormatInfo[MALCOLM_PCAP_DIR]
+            else:
+                pcapDirDefault = os.path.join(malcolm_install_path, pcapDir)
         pcapDirFull = os.path.realpath(pcapDirDefault)
 
-        suricataLogDir = './suricata-logs'
-        suricataLogDirDefault = os.path.join(malcolm_install_path, suricataLogDir)
+        if args.suricataLogDir:
+            suricataLogDirDefault = args.suricataLogDir
+        else:
+            suricataLogDir = 'suricata-logs'
+            if (MALCOLM_LOGS_DIR in diskFormatInfo) and os.path.isdir(os.path.join(diskFormatInfo[MALCOLM_LOGS_DIR], suricataLogDir)):
+                suricataLogDirDefault = os.path.join(diskFormatInfo[MALCOLM_LOGS_DIR], suricataLogDir)
+            else:
+                suricataLogDirDefault = os.path.join(malcolm_install_path, suricataLogDir)
         suricataLogDirFull = os.path.realpath(suricataLogDirDefault)
 
-        zeekLogDir = './zeek-logs'
-        zeekLogDirDefault = os.path.join(malcolm_install_path, zeekLogDir)
+        if args.zeekLogDir:
+            zeekLogDirDefault = args.zeekLogDir
+        else:
+            zeekLogDir = 'zeek-logs'
+            if (MALCOLM_LOGS_DIR in diskFormatInfo) and os.path.isdir(os.path.join(diskFormatInfo[MALCOLM_LOGS_DIR], zeekLogDir)):
+                zeekLogDirDefault = os.path.join(diskFormatInfo[MALCOLM_LOGS_DIR], zeekLogDir)
+            else:
+                zeekLogDirDefault = os.path.join(malcolm_install_path, zeekLogDir)
         zeekLogDirFull = os.path.realpath(zeekLogDirDefault)
 
         if self.orchMode is OrchestrationFramework.DOCKER_COMPOSE:
-            if not InstallerYesOrNo(
-                f'Store {"PCAP, log and index" if (malcolmProfile == PROFILE_MALCOLM) else "PCAP and log"} files locally under {malcolm_install_path}?',
+            if (not diskFormatInfo) or not InstallerYesOrNo(
+                f'Store {"PCAP, log and index" if (malcolmProfile == PROFILE_MALCOLM) else "PCAP and log"} files in {malcolm_install_path}?',
                 default=not args.acceptDefaultsNonInteractive,
             ):
                 # PCAP directory
                 if not InstallerYesOrNo(
-                    'Store PCAP files locally in {}?'.format(pcapDirDefault),
+                    'Store PCAP files in {}?'.format(pcapDirDefault),
                     default=not bool(args.pcapDir),
                 ):
                     loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid PCAP directory')
                     while loopBreaker.increment():
-                        pcapDir = InstallerAskForString('Enter PCAP directory', default=args.pcapDir)
+                        pcapDir = InstallerAskForString('Enter PCAP directory', default=pcapDirDefault)
                         if (len(pcapDir) > 1) and os.path.isdir(pcapDir):
                             pcapDirFull = os.path.realpath(pcapDir)
                             pcapDir = (
@@ -902,12 +958,12 @@ def tweak_malcolm_runtime(self, malcolm_install_path):
 
                 # Zeek log directory
                 if not InstallerYesOrNo(
-                    'Store Zeek logs locally in {}?'.format(zeekLogDirDefault),
+                    'Store Zeek logs in {}?'.format(zeekLogDirDefault),
                     default=not bool(args.zeekLogDir),
                 ):
                     loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid Zeek directory')
                     while loopBreaker.increment():
-                        zeekLogDir = InstallerAskForString('Enter Zeek log directory', default=args.zeekLogDir)
+                        zeekLogDir = InstallerAskForString('Enter Zeek log directory', default=zeekLogDirDefault)
                         if (len(zeekLogDir) > 1) and os.path.isdir(zeekLogDir):
                             zeekLogDirFull = os.path.realpath(zeekLogDir)
                             zeekLogDir = (
@@ -919,13 +975,13 @@ def tweak_malcolm_runtime(self, malcolm_install_path):
 
                 # Suricata log directory
                 if not InstallerYesOrNo(
-                    'Store Suricata logs locally in {}?'.format(suricataLogDirDefault),
+                    'Store Suricata logs in {}?'.format(suricataLogDirDefault),
                     default=not bool(args.suricataLogDir),
                 ):
                     loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid Suricata directory')
                     while loopBreaker.increment():
                         suricataLogDir = InstallerAskForString(
-                            'Enter Suricata log directory', default=args.suricataLogDir
+                            'Enter Suricata log directory', default=suricataLogDirDefault
                         )
                         if (len(suricataLogDir) > 1) and os.path.isdir(suricataLogDir):
                             suricataLogDirFull = os.path.realpath(suricataLogDir)
@@ -939,12 +995,12 @@ def tweak_malcolm_runtime(self, malcolm_install_path):
                 if (malcolmProfile == PROFILE_MALCOLM) and (opensearchPrimaryMode == DatabaseMode.OpenSearchLocal):
                     # opensearch index directory
                     if not InstallerYesOrNo(
-                        'Store OpenSearch indices locally in {}?'.format(indexDirDefault),
+                        'Store OpenSearch indices in {}?'.format(indexDirDefault),
                         default=not bool(args.indexDir),
                     ):
                         loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid OpenSearch index directory')
                         while loopBreaker.increment():
-                            indexDir = InstallerAskForString('Enter OpenSearch index directory', default=args.indexDir)
+                            indexDir = InstallerAskForString('Enter OpenSearch index directory', default=indexDirDefault)
                             if (len(indexDir) > 1) and os.path.isdir(indexDir):
                                 indexDirFull = os.path.realpath(indexDir)
                                 indexDir = (
@@ -956,13 +1012,13 @@ def tweak_malcolm_runtime(self, malcolm_install_path):
 
                     # opensearch snapshot repository directory and compression
                     if not InstallerYesOrNo(
-                        'Store OpenSearch index snapshots locally in {}?'.format(indexSnapshotDirDefault),
+                        'Store OpenSearch index snapshots in {}?'.format(indexSnapshotDirDefault),
                         default=not bool(args.indexSnapshotDir),
                     ):
                         loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid OpenSearch snapshots directory')
                         while loopBreaker.increment():
                             indexSnapshotDir = InstallerAskForString(
-                                'Enter OpenSearch index snapshot directory', default=args.indexSnapshotDir
+                                'Enter OpenSearch index snapshot directory', default=indexSnapshotDirDefault
                             )
                             if (len(indexSnapshotDir) > 1) and os.path.isdir(indexSnapshotDir):
                                 indexSnapshotDirFull = os.path.realpath(indexSnapshotDir)

scripts/malcolm_utils.py

@@ -50,6 +50,15 @@ class DatabaseMode(enum.IntFlag):
 DATABASE_MODE_ENUMS['opensearch-remote'] = DatabaseMode.OpenSearchRemote
 DATABASE_MODE_ENUMS['elasticsearch-remote'] = DatabaseMode.ElasticsearchRemote
 
+OS_MODE_HEDGEHOG = 'hedgehog'
+OS_MODE_MALCOLM = 'malcolm'
+
+HEDGEHOG_PCAP_DIR = "pcap"
+HEDGEHOG_ZEEK_DIR = "zeek"
+MALCOLM_DB_DIR = "datastore"
+MALCOLM_PCAP_DIR = "pcap"
+MALCOLM_LOGS_DIR = "logs"
+
 
 def DatabaseModeEnumToStr(val):
     return DATABASE_MODE_LABELS[val]

shared/bin/os-disk-config.py

@@ -12,6 +12,7 @@
 ###################################################################################################
 
 import os
+import json
 import re
 import glob
 import sys
@@ -21,16 +22,20 @@
 from collections import defaultdict
 from fstab import Fstab
 
-from malcolm_utils import remove_prefix, str2bool, sizeof_fmt, run_subprocess, eprint
-
-OS_MODE_HEDGEHOG = 'hedgehog'
-OS_MODE_MALCOLM = 'malcolm'
+from malcolm_utils import (
+    remove_prefix,
+    str2bool,
+    sizeof_fmt,
+    run_subprocess,
+    eprint,
+    OS_MODE_HEDGEHOG,
+    OS_MODE_MALCOLM,
+    HEDGEHOG_ZEEK_DIR,
+    MALCOLM_DB_DIR,
+    MALCOLM_PCAP_DIR,
+    MALCOLM_LOGS_DIR,
+)
 
-HEDGEHOG_PCAP_DIR = "pcap"
-HEDGEHOG_ZEEK_DIR = "zeek"
-MALCOLM_DB_DIR = "datastore"
-MALCOLM_PCAP_DIR = "pcap"
-MALCOLM_LOGS_DIR = "logs"
 
 OS_PARAMS = defaultdict(lambda: None)
 OS_PARAMS[OS_MODE_HEDGEHOG] = defaultdict(lambda: None)
@@ -49,6 +54,7 @@
         SYSTEM_CONFIG_FILE: '/opt/sensor/sensor_ctl/control_vars.conf',
         CRYPT_KEYFILE: '/etc/capture_crypt.key',
         CRYPT_KEYFILE_PERMS: 0o600,
+        OTHER_FILE_PERMS: 0o600,
         CRYPT_DEV_PREFIX: 'capture_vault_',
     }
 )
@@ -65,6 +71,7 @@
         SUBDIR_PERMS: 0o770,
         CRYPT_KEYFILE: '/etc/capture_crypt.key',
         CRYPT_KEYFILE_PERMS: 0o600,
+        OTHER_FILE_PERMS: 0o600,
         CRYPT_DEV_PREFIX: 'malcolm_vault_',
     }
 )
@@ -577,7 +584,7 @@ def main():
         # get the GID of the group of the user(s) under which the processes will be run
         try:
             ecode, guidGetOut = run_subprocess(
-                f"getent group {OS_PARAMS[osMode][GROUP_OWNER]}", stdout=True, stderr=True
+                f"getent group {OS_PARAMS[osMode][GROUP_OWNER]}", stdout=True, stderr=False
             )
             if (ecode == 0) and (len(guidGetOut) > 0):
                 ownerGuid = int(guidGetOut[0].split(':')[2])
@@ -586,6 +593,16 @@ def main():
         except Exception:
             ownerGuid = -1
 
+        # get home directory for USER_UID
+        try:
+            ecode, getentOut = run_subprocess(f"getent passwd {OS_PARAMS[osMode][USER_UID]}", stdout=True, stderr=False)
+            if (ecode == 0) and (len(getentOut) > 0):
+                ownerHome = getentOut[0].split(':')[5]
+            else:
+                ownerHome = ''
+        except Exception:
+            ownerHome = ''
+
         # rmdir any mount directories that might be interfering from previous configurations
         if os.path.isdir(OS_PARAMS[osMode][MOUNT_ROOT_PATH]):
             for root, dirs, files in os.walk(OS_PARAMS[osMode][MOUNT_ROOT_PATH], topdown=False):
@@ -672,8 +689,8 @@ def main():
                             createdUserDirs[subDir] = userDir
                             break
 
-                # replace paths in-place in control_vars.conf
                 if (osMode == OS_MODE_HEDGEHOG) and os.path.isfile(OS_PARAMS[osMode][SYSTEM_CONFIG_FILE]):
+                    # replace paths in-place in control_vars.conf
                     capture_re = re.compile(r"\b(?P<key>PCAP_PATH|ZEEK_LOG_PATH)\s*=\s*.*?$")
                     with fileinput.FileInput(OS_PARAMS[osMode][SYSTEM_CONFIG_FILE], inplace=True, backup='.bak') as f:
                         for line in f:
@@ -693,6 +710,15 @@ def main():
                             else:
                                 print(line)
 
+                elif (osMode == OS_MODE_MALCOLM) and os.path.isdir(os.path.join(ownerHome, 'Malcolm')):
+                    # write .os_disk_config_defaults for to be picked up by install.py
+                    configFileFull = os.path.join(os.path.join(ownerHome, 'Malcolm'), '.os_disk_config_defaults')
+                    with open(configFileFull, 'w') as f:
+                        f.write(json.dumps(createdUserDirs), indent=4)
+                    if os.path.isfile(configFileFull):
+                        os.chown(configFileFull, OS_PARAMS[osMode][USER_UID], ownerGuid)
+                        os.chmod(configFileFull, OS_PARAMS[osMode][CRYPT_KEYFILE_PERMS])
+
             else:
                 eprint(f"Error {ecode} mounting {par.partition}")