diff --git a/Dockerfiles/file-monitor.Dockerfile b/Dockerfiles/file-monitor.Dockerfile index 722b361a6..ff53ef2e4 100644 --- a/Dockerfiles/file-monitor.Dockerfile +++ b/Dockerfiles/file-monitor.Dockerfile @@ -44,6 +44,11 @@ ARG EXTRACTED_FILE_ENABLE_YARA=false ARG EXTRACTED_FILE_YARA_CUSTOM_ONLY=false ARG EXTRACTED_FILE_ENABLE_CAPA=false ARG EXTRACTED_FILE_CAPA_VERBOSE=false +ARG EXTRACTED_FILE_HTTP_SERVER_DEBUG=false +ARG EXTRACTED_FILE_HTTP_SERVER_ENABLE=false +ARG EXTRACTED_FILE_HTTP_SERVER_ENCRYPT=false +ARG EXTRACTED_FILE_HTTP_SERVER_KEY=quarantined +ARG EXTRACTED_FILE_HTTP_SERVER_PORT=8440 ENV ZEEK_EXTRACTOR_PATH $ZEEK_EXTRACTOR_PATH ENV ZEEK_LOG_DIRECTORY $ZEEK_LOG_DIRECTORY @@ -78,6 +83,11 @@ ENV YARA_RULES_SRC_DIR "$SRC_BASE_DIR/signature-base" ENV CAPA_URL "https://github.com/fireeye/capa" ENV CAPA_RULES_URL "https://github.com/fireeye/capa-rules" ENV CAPA_RULES_DIR "/capa-rules" +ENV EXTRACTED_FILE_HTTP_SERVER_DEBUG $EXTRACTED_FILE_HTTP_SERVER_DEBUG +ENV EXTRACTED_FILE_HTTP_SERVER_ENABLE $EXTRACTED_FILE_HTTP_SERVER_ENABLE +ENV EXTRACTED_FILE_HTTP_SERVER_ENCRYPT $EXTRACTED_FILE_HTTP_SERVER_ENCRYPT +ENV EXTRACTED_FILE_HTTP_SERVER_KEY $EXTRACTED_FILE_HTTP_SERVER_KEY +ENV EXTRACTED_FILE_HTTP_SERVER_PORT $EXTRACTED_FILE_HTTP_SERVER_PORT ENV SUPERCRONIC_VERSION "0.1.12" ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64" @@ -123,7 +133,7 @@ RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list python3-pyinotify \ python3-requests \ python3-zmq && \ - pip3 install clamd supervisor yara-python python-magic psutil && \ + pip3 install clamd supervisor yara-python python-magic psutil pycryptodome && \ pip2 install flare-capa && \ curl -fsSLO "$SUPERCRONIC_URL" && \ echo "${SUPERCRONIC_SHA1SUM} ${SUPERCRONIC}" | sha1sum -c - && \ @@ -204,7 +214,7 @@ RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list echo "0 */6 * * * /bin/bash /usr/local/bin/capa-rules-update.sh\n0 */6 * * * /bin/bash /usr/local/bin/yara-rules-update.sh" > ${SUPERCRONIC_CRONTAB} ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/ -ADD shared/bin/zeek_carve_*.py /usr/local/bin/ +ADD shared/bin/zeek_carve*.py /usr/local/bin/ ADD shared/bin/malass_client.py /usr/local/bin/ ADD file-monitor/supervisord.conf /etc/supervisord.conf ADD file-monitor/docker-entrypoint.sh /docker-entrypoint.sh diff --git a/README.md b/README.md index c5fbd59af..2a01b3162 100644 --- a/README.md +++ b/README.md @@ -537,6 +537,12 @@ Various other environment variables inside of `docker-compose.yml` can be tweake * `EXTRACTED_FILE_UPDATE_RULES` – if set to `true`, file scanner engines (e.g., ClamAV, Capa, Yara) will periodically update their rule definitions +* `EXTRACTED_FILE_HTTP_SERVER_ENABLE` – if set to `true`, the directory containing [Zeek-extracted files](#ZeekFileExtraction) will be served over HTTP at `./extracted-files/` (e.g., [https://localhost/extracted-files/](https://localhost/extracted-files/) if you are connecting locally) + +* `EXTRACTED_FILE_HTTP_SERVER_ENCRYPT` – if set to `true`, those Zeek-extracted files will be AES-256-CBC-encrypted in an `openssl enc`-compatible format (e.g., `openssl enc -aes-256-cbc -d -in example.exe.encrypted -out example.exe`) + +* `EXTRACTED_FILE_HTTP_SERVER_KEY` – specifies the AES-256-CBC decryption password for encrypted Zeek-extracted files; used in conjunction with `EXTRACTED_FILE_HTTP_SERVER_ENCRYPT` + * `PCAP_ENABLE_NETSNIFF` – if set to `true`, Malcolm will capture network traffic on the local network interface(s) indicated in `PCAP_IFACE` using [netsniff-ng](http://netsniff-ng.org/) * `PCAP_ENABLE_TCPDUMP` – if set to `true`, Malcolm will capture network traffic on the local network interface(s) indicated in `PCAP_IFACE` using [tcpdump](https://www.tcpdump.org/); there is no reason to enable *both* `PCAP_ENABLE_NETSNIFF` and `PCAP_ENABLE_TCPDUMP` @@ -1291,6 +1297,8 @@ The `EXTRACTED_FILE_PRESERVATION` [environment variable in `docker-compose.yml`] * `all`: preserve flagged files in `./zeek-logs/extract_files/quarantine` and all other extracted files in `./zeek-logs/extract_files/preserved` * `none`: preserve no extracted files +The `EXTRACTED_FILE_HTTP_SERVER_...` [environment variables in `docker-compose.yml`](#DockerComposeYml) configure access to the Zeek-extracted files path through the means of a simple HTTPS directory server. Beware that Zeek-extracted files may contain malware. As such, the files may be optionally encrypted upon download. + ### Automatic host and subnet name assignment #### IP/MAC address to hostname mapping via `host-map.txt` diff --git a/docker-compose-standalone.yml b/docker-compose-standalone.yml index 6febd8452..a37911e5e 100644 --- a/docker-compose-standalone.yml +++ b/docker-compose-standalone.yml @@ -55,6 +55,9 @@ x-zeek-variables: &zeek-variables EXTRACTED_FILE_UPDATE_RULES : 'false' EXTRACTED_FILE_PIPELINE_DEBUG : 'false' EXTRACTED_FILE_PIPELINE_DEBUG_EXTRA : 'false' + EXTRACTED_FILE_HTTP_SERVER_ENABLE : 'false' + EXTRACTED_FILE_HTTP_SERVER_ENCRYPT : 'true' + EXTRACTED_FILE_HTTP_SERVER_KEY : 'quarantined' # environment variables for tweaking Zeek at runtime (see local.zeek) # set to a non-blank value to disable the corresponding feature ZEEK_DISABLE_MITRE_BZAR : '' @@ -540,6 +543,7 @@ services: - upload - htadmin - name-map-ui + - file-monitor ports: - "443:443" - "488:488" diff --git a/docker-compose.yml b/docker-compose.yml index ffba623ea..83adbf496 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -55,6 +55,9 @@ x-zeek-variables: &zeek-variables EXTRACTED_FILE_UPDATE_RULES : 'false' EXTRACTED_FILE_PIPELINE_DEBUG : 'false' EXTRACTED_FILE_PIPELINE_DEBUG_EXTRA : 'false' + EXTRACTED_FILE_HTTP_SERVER_ENABLE : 'false' + EXTRACTED_FILE_HTTP_SERVER_ENCRYPT : 'true' + EXTRACTED_FILE_HTTP_SERVER_KEY : 'quarantined' # environment variables for tweaking Zeek at runtime (see local.zeek) # set to a non-blank value to disable the corresponding feature ZEEK_DISABLE_MITRE_BZAR : '' @@ -597,6 +600,7 @@ services: - upload - htadmin - name-map-ui + - file-monitor ports: - "443:443" - "488:488" diff --git a/file-monitor/supervisord.conf b/file-monitor/supervisord.conf index 2429c0b52..78b32e88a 100644 --- a/file-monitor/supervisord.conf +++ b/file-monitor/supervisord.conf @@ -169,6 +169,22 @@ stdout_logfile=/dev/fd/1 stdout_logfile_maxbytes=0 redirect_stderr=true +[program:fileserve] +command=/usr/local/bin/zeek_carved_http_server.py + --port %(ENV_EXTRACTED_FILE_HTTP_SERVER_PORT)s + --encrypt %(ENV_EXTRACTED_FILE_HTTP_SERVER_ENCRYPT)s + --directory /data/zeek/extract_files +autostart=%(ENV_EXTRACTED_FILE_HTTP_SERVER_ENABLE)s +autorestart=true +startsecs=0 +startretries=0 +stopasgroup=true +killasgroup=true +directory=/data/zeek/extract_files +stdout_logfile=/dev/fd/1 +stdout_logfile_maxbytes=0 +redirect_stderr=true + [program:cron] autorestart=true command=/usr/local/bin/supercronic -json "%(ENV_SUPERCRONIC_CRONTAB)s" diff --git a/malcolm-iso/config/hooks/normal/0169-pip-installs.hook.chroot b/malcolm-iso/config/hooks/normal/0169-pip-installs.hook.chroot index feb7b9204..32c846ae4 100755 --- a/malcolm-iso/config/hooks/normal/0169-pip-installs.hook.chroot +++ b/malcolm-iso/config/hooks/normal/0169-pip-installs.hook.chroot @@ -11,5 +11,6 @@ pip3 install --no-compile --no-cache-dir --force-reinstall --upgrade \ docker-compose \ netifaces \ psutil \ + pycryptodome \ pythondialog \ requests[security] diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 7de9dc87b..25896a605 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -69,6 +69,10 @@ http { server name-map-ui:8080; } + upstream docker-extracted-file-http-server { + server filemon:8440; + } + # htadmin (htpasswd/user management) server { listen 488 ssl; @@ -195,6 +199,12 @@ http { proxy_cache off; } + location ~* ^/extracted-files\b(.*) { + proxy_pass http://docker-extracted-file-http-server$1; + proxy_redirect off; + proxy_set_header Host filemon.malcolm.local; + } + location = /favicon.ico { alias /etc/nginx/favicon.ico; } diff --git a/sensor-iso/config/hooks/normal/0169-pip-installs.hook.chroot b/sensor-iso/config/hooks/normal/0169-pip-installs.hook.chroot index 575c7f9a2..b18972b9a 100755 --- a/sensor-iso/config/hooks/normal/0169-pip-installs.hook.chroot +++ b/sensor-iso/config/hooks/normal/0169-pip-installs.hook.chroot @@ -17,6 +17,7 @@ pip3 install --no-compile --no-cache-dir --force-reinstall --upgrade \ ipaddress \ netifaces \ psutil \ + pycryptodome \ pyinotify \ python-magic \ pythondialog \ diff --git a/shared/bin/zeek_carved_http_server.py b/shared/bin/zeek_carved_http_server.py new file mode 100755 index 000000000..8a6ec4d8e --- /dev/null +++ b/shared/bin/zeek_carved_http_server.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Multithreaded simple HTTP directory server. +# +# The files can optionally be aes-256-cbc encrypted in a way that's compatible with: +# openssl enc -aes-256-cbc -d -in encrypted.data -out decrypted.data + +import argparse +import hashlib +import os +import sys +from threading import Thread +from socketserver import ThreadingMixIn +from http.server import HTTPServer, SimpleHTTPRequestHandler +from Crypto.Cipher import AES + +KEY_SIZE = 32 +OPENSSL_ENC_MAGIC = b'Salted__' +PKCS5_SALT_LEN = 8 + +################################################################################################### +args = None +debug = False +script_name = os.path.basename(__file__) +script_path = os.path.dirname(os.path.realpath(__file__)) +orig_path = os.getcwd() + +################################################################################################### +# print to stderr +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + sys.stderr.flush() + +################################################################################################### +# convenient boolean argument parsing +def str2bool(v): + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise argparse.ArgumentTypeError('Boolean value expected.') + +################################################################################################### +# EVP_BytesToKey +# +# reference: https://github.com/openssl/openssl/blob/6f0ac0e2f27d9240516edb9a23b7863e7ad02898/crypto/evp/evp_key.c#L74 +# https://gist.github.com/chrono-meter/d122cbefc6f6248a0af554995f072460 +def EVP_BytesToKey(key_length: int, iv_length: int, md, salt: bytes, data: bytes, count: int=1) -> (bytes, bytes): + assert data + assert salt == b'' or len(salt) == PKCS5_SALT_LEN + + md_buf = b'' + key = b'' + iv = b'' + addmd = 0 + + while key_length > len(key) or iv_length > len(iv): + c = md() + if addmd: + c.update(md_buf) + addmd += 1 + c.update(data) + c.update(salt) + md_buf = c.digest() + for i in range(1, count): + md_buf = md(md_buf) + + md_buf2 = md_buf + + if key_length > len(key): + key, md_buf2 = key + md_buf2[:key_length - len(key)], md_buf2[key_length - len(key):] + + if iv_length > len(iv): + iv = iv + md_buf2[:iv_length - len(iv)] + + return key, iv + +################################################################################################### +# +class HTTPHandler(SimpleHTTPRequestHandler): + + # return full path based on server base path and requested path + def translate_path(self, path): + path = SimpleHTTPRequestHandler.translate_path(self, path) + relpath = os.path.relpath(path, os.getcwd()) + fullpath = os.path.join(self.server.base_path, relpath) + return fullpath + + # override do_GET so that files are encrypted, if requested + def do_GET(self): + global debug + global args + + fullpath = self.translate_path(self.path) + + if (not args.encrypt) or os.path.isdir(fullpath): + # unencrypted, just use default implementation + SimpleHTTPRequestHandler.do_GET(self) + + else: + # encrypt file transfers + if os.path.isfile(fullpath) or os.path.islink(fullpath): + self.send_response(200) + self.send_header('Content-type', 'application/octet-stream') + self.send_header('Content-Disposition', f'attachment; filename={os.path.basename(fullpath)}.encrypted') + self.end_headers() + salt = os.urandom(PKCS5_SALT_LEN) + key, iv = EVP_BytesToKey(KEY_SIZE, AES.block_size, hashlib.sha256, salt, args.key.encode('utf-8')) + cipher = AES.new(key, AES.MODE_CBC, iv) + encrypted = b"" + encrypted += OPENSSL_ENC_MAGIC + encrypted += salt + self.wfile.write(encrypted) + with open(fullpath, 'rb') as f: + padding = b'' + while True: + chunk = f.read(cipher.block_size) + if len(chunk) < cipher.block_size: + remaining = cipher.block_size - len(chunk) + padding = bytes([remaining] * remaining) + self.wfile.write(cipher.encrypt(chunk + padding)) + if padding: + break + + else: + self.send_error(404, "Not Found") + +################################################################################################### +# +class ThreadingHTTPServer(ThreadingMixIn, HTTPServer): + def __init__(self, base_path, server_address, RequestHandlerClass=HTTPHandler): + self.base_path = base_path + HTTPServer.__init__(self, server_address, RequestHandlerClass) + +################################################################################################### +# +def serve_on_port(path : str, port : int): + server = ThreadingHTTPServer(path, ("", port)) + print(f"serving {path} at port {port}") + server.serve_forever() + +################################################################################################### +# main +def main(): + global args + global debug + global orig_path + + defaultDebug = os.getenv('EXTRACTED_FILE_HTTP_SERVER_DEBUG', 'false') + defaultEncrypt = os.getenv('EXTRACTED_FILE_HTTP_SERVER_ENCRYPT', 'false') + defaultPort = int(os.getenv('EXTRACTED_FILE_HTTP_SERVER_PORT', 8440)) + defaultKey = os.getenv('EXTRACTED_FILE_HTTP_SERVER_KEY', 'quarantined') + defaultDir = os.getenv('EXTRACTED_FILE_HTTP_SERVER_PATH', orig_path) + + parser = argparse.ArgumentParser(description=script_name, add_help=False, usage='{} '.format(script_name)) + parser.add_argument('-v', '--verbose', dest='debug', type=str2bool, nargs='?', const=True, default=defaultDebug, metavar='true|false', help=f"Verbose/debug output ({defaultDebug})") + parser.add_argument('-p', '--port', dest='port', help=f"Server port ({defaultPort})", metavar='', type=int, default=defaultPort) + parser.add_argument('-d', '--directory', dest='serveDir', help=f'Directory to serve ({defaultDir})', metavar='', type=str, default=defaultDir) + parser.add_argument('-e', '--encrypt', dest='encrypt', type=str2bool, nargs='?', const=True, default=defaultEncrypt, metavar='true|false', help=f"Encrypt files with aes-256-cbc ({defaultEncrypt})") + parser.add_argument('-k', '--key', dest='key', help=f"File encryption key", metavar='', type=str, default=defaultKey) + try: + parser.error = parser.exit + args = parser.parse_args() + except SystemExit: + parser.print_help() + exit(2) + + debug = args.debug + if debug: + eprint(os.path.join(script_path, script_name)) + eprint("Arguments: {}".format(sys.argv[1:])) + eprint("Arguments: {}".format(args)) + else: + sys.tracebacklimit = 0 + + Thread(target=serve_on_port, args=[args.serveDir, args.port]).start() + +################################################################################################### +if __name__ == '__main__': + main()