diff --git a/Dockerfiles/file-monitor.Dockerfile b/Dockerfiles/file-monitor.Dockerfile
index 722b361a6..ff53ef2e4 100644
--- a/Dockerfiles/file-monitor.Dockerfile
+++ b/Dockerfiles/file-monitor.Dockerfile
@@ -44,6 +44,11 @@ ARG EXTRACTED_FILE_ENABLE_YARA=false
ARG EXTRACTED_FILE_YARA_CUSTOM_ONLY=false
ARG EXTRACTED_FILE_ENABLE_CAPA=false
ARG EXTRACTED_FILE_CAPA_VERBOSE=false
+ARG EXTRACTED_FILE_HTTP_SERVER_DEBUG=false
+ARG EXTRACTED_FILE_HTTP_SERVER_ENABLE=false
+ARG EXTRACTED_FILE_HTTP_SERVER_ENCRYPT=false
+ARG EXTRACTED_FILE_HTTP_SERVER_KEY=quarantined
+ARG EXTRACTED_FILE_HTTP_SERVER_PORT=8440
ENV ZEEK_EXTRACTOR_PATH $ZEEK_EXTRACTOR_PATH
ENV ZEEK_LOG_DIRECTORY $ZEEK_LOG_DIRECTORY
@@ -78,6 +83,11 @@ ENV YARA_RULES_SRC_DIR "$SRC_BASE_DIR/signature-base"
ENV CAPA_URL "https://github.com/fireeye/capa"
ENV CAPA_RULES_URL "https://github.com/fireeye/capa-rules"
ENV CAPA_RULES_DIR "/capa-rules"
+ENV EXTRACTED_FILE_HTTP_SERVER_DEBUG $EXTRACTED_FILE_HTTP_SERVER_DEBUG
+ENV EXTRACTED_FILE_HTTP_SERVER_ENABLE $EXTRACTED_FILE_HTTP_SERVER_ENABLE
+ENV EXTRACTED_FILE_HTTP_SERVER_ENCRYPT $EXTRACTED_FILE_HTTP_SERVER_ENCRYPT
+ENV EXTRACTED_FILE_HTTP_SERVER_KEY $EXTRACTED_FILE_HTTP_SERVER_KEY
+ENV EXTRACTED_FILE_HTTP_SERVER_PORT $EXTRACTED_FILE_HTTP_SERVER_PORT
ENV SUPERCRONIC_VERSION "0.1.12"
ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64"
@@ -123,7 +133,7 @@ RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list
python3-pyinotify \
python3-requests \
python3-zmq && \
- pip3 install clamd supervisor yara-python python-magic psutil && \
+ pip3 install clamd supervisor yara-python python-magic psutil pycryptodome && \
pip2 install flare-capa && \
curl -fsSLO "$SUPERCRONIC_URL" && \
echo "${SUPERCRONIC_SHA1SUM} ${SUPERCRONIC}" | sha1sum -c - && \
@@ -204,7 +214,7 @@ RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list
echo "0 */6 * * * /bin/bash /usr/local/bin/capa-rules-update.sh\n0 */6 * * * /bin/bash /usr/local/bin/yara-rules-update.sh" > ${SUPERCRONIC_CRONTAB}
ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
-ADD shared/bin/zeek_carve_*.py /usr/local/bin/
+ADD shared/bin/zeek_carve*.py /usr/local/bin/
ADD shared/bin/malass_client.py /usr/local/bin/
ADD file-monitor/supervisord.conf /etc/supervisord.conf
ADD file-monitor/docker-entrypoint.sh /docker-entrypoint.sh
diff --git a/README.md b/README.md
index c5fbd59af..2a01b3162 100644
--- a/README.md
+++ b/README.md
@@ -537,6 +537,12 @@ Various other environment variables inside of `docker-compose.yml` can be tweake
* `EXTRACTED_FILE_UPDATE_RULES` – if set to `true`, file scanner engines (e.g., ClamAV, Capa, Yara) will periodically update their rule definitions
+* `EXTRACTED_FILE_HTTP_SERVER_ENABLE` – if set to `true`, the directory containing [Zeek-extracted files](#ZeekFileExtraction) will be served over HTTP at `./extracted-files/` (e.g., [https://localhost/extracted-files/](https://localhost/extracted-files/) if you are connecting locally)
+
+* `EXTRACTED_FILE_HTTP_SERVER_ENCRYPT` – if set to `true`, those Zeek-extracted files will be AES-256-CBC-encrypted in an `openssl enc`-compatible format (e.g., `openssl enc -aes-256-cbc -d -in example.exe.encrypted -out example.exe`)
+
+* `EXTRACTED_FILE_HTTP_SERVER_KEY` – specifies the AES-256-CBC decryption password for encrypted Zeek-extracted files; used in conjunction with `EXTRACTED_FILE_HTTP_SERVER_ENCRYPT`
+
* `PCAP_ENABLE_NETSNIFF` – if set to `true`, Malcolm will capture network traffic on the local network interface(s) indicated in `PCAP_IFACE` using [netsniff-ng](http://netsniff-ng.org/)
* `PCAP_ENABLE_TCPDUMP` – if set to `true`, Malcolm will capture network traffic on the local network interface(s) indicated in `PCAP_IFACE` using [tcpdump](https://www.tcpdump.org/); there is no reason to enable *both* `PCAP_ENABLE_NETSNIFF` and `PCAP_ENABLE_TCPDUMP`
@@ -1291,6 +1297,8 @@ The `EXTRACTED_FILE_PRESERVATION` [environment variable in `docker-compose.yml`]
* `all`: preserve flagged files in `./zeek-logs/extract_files/quarantine` and all other extracted files in `./zeek-logs/extract_files/preserved`
* `none`: preserve no extracted files
+The `EXTRACTED_FILE_HTTP_SERVER_...` [environment variables in `docker-compose.yml`](#DockerComposeYml) configure access to the Zeek-extracted files path through the means of a simple HTTPS directory server. Beware that Zeek-extracted files may contain malware. As such, the files may be optionally encrypted upon download.
+
### Automatic host and subnet name assignment
#### IP/MAC address to hostname mapping via `host-map.txt`
diff --git a/docker-compose-standalone.yml b/docker-compose-standalone.yml
index 6febd8452..a37911e5e 100644
--- a/docker-compose-standalone.yml
+++ b/docker-compose-standalone.yml
@@ -55,6 +55,9 @@ x-zeek-variables: &zeek-variables
EXTRACTED_FILE_UPDATE_RULES : 'false'
EXTRACTED_FILE_PIPELINE_DEBUG : 'false'
EXTRACTED_FILE_PIPELINE_DEBUG_EXTRA : 'false'
+ EXTRACTED_FILE_HTTP_SERVER_ENABLE : 'false'
+ EXTRACTED_FILE_HTTP_SERVER_ENCRYPT : 'true'
+ EXTRACTED_FILE_HTTP_SERVER_KEY : 'quarantined'
# environment variables for tweaking Zeek at runtime (see local.zeek)
# set to a non-blank value to disable the corresponding feature
ZEEK_DISABLE_MITRE_BZAR : ''
@@ -540,6 +543,7 @@ services:
- upload
- htadmin
- name-map-ui
+ - file-monitor
ports:
- "443:443"
- "488:488"
diff --git a/docker-compose.yml b/docker-compose.yml
index ffba623ea..83adbf496 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -55,6 +55,9 @@ x-zeek-variables: &zeek-variables
EXTRACTED_FILE_UPDATE_RULES : 'false'
EXTRACTED_FILE_PIPELINE_DEBUG : 'false'
EXTRACTED_FILE_PIPELINE_DEBUG_EXTRA : 'false'
+ EXTRACTED_FILE_HTTP_SERVER_ENABLE : 'false'
+ EXTRACTED_FILE_HTTP_SERVER_ENCRYPT : 'true'
+ EXTRACTED_FILE_HTTP_SERVER_KEY : 'quarantined'
# environment variables for tweaking Zeek at runtime (see local.zeek)
# set to a non-blank value to disable the corresponding feature
ZEEK_DISABLE_MITRE_BZAR : ''
@@ -597,6 +600,7 @@ services:
- upload
- htadmin
- name-map-ui
+ - file-monitor
ports:
- "443:443"
- "488:488"
diff --git a/file-monitor/supervisord.conf b/file-monitor/supervisord.conf
index 2429c0b52..78b32e88a 100644
--- a/file-monitor/supervisord.conf
+++ b/file-monitor/supervisord.conf
@@ -169,6 +169,22 @@ stdout_logfile=/dev/fd/1
stdout_logfile_maxbytes=0
redirect_stderr=true
+[program:fileserve]
+command=/usr/local/bin/zeek_carved_http_server.py
+ --port %(ENV_EXTRACTED_FILE_HTTP_SERVER_PORT)s
+ --encrypt %(ENV_EXTRACTED_FILE_HTTP_SERVER_ENCRYPT)s
+ --directory /data/zeek/extract_files
+autostart=%(ENV_EXTRACTED_FILE_HTTP_SERVER_ENABLE)s
+autorestart=true
+startsecs=0
+startretries=0
+stopasgroup=true
+killasgroup=true
+directory=/data/zeek/extract_files
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+redirect_stderr=true
+
[program:cron]
autorestart=true
command=/usr/local/bin/supercronic -json "%(ENV_SUPERCRONIC_CRONTAB)s"
diff --git a/malcolm-iso/config/hooks/normal/0169-pip-installs.hook.chroot b/malcolm-iso/config/hooks/normal/0169-pip-installs.hook.chroot
index feb7b9204..32c846ae4 100755
--- a/malcolm-iso/config/hooks/normal/0169-pip-installs.hook.chroot
+++ b/malcolm-iso/config/hooks/normal/0169-pip-installs.hook.chroot
@@ -11,5 +11,6 @@ pip3 install --no-compile --no-cache-dir --force-reinstall --upgrade \
docker-compose \
netifaces \
psutil \
+ pycryptodome \
pythondialog \
requests[security]
diff --git a/nginx/nginx.conf b/nginx/nginx.conf
index 7de9dc87b..25896a605 100644
--- a/nginx/nginx.conf
+++ b/nginx/nginx.conf
@@ -69,6 +69,10 @@ http {
server name-map-ui:8080;
}
+ upstream docker-extracted-file-http-server {
+ server filemon:8440;
+ }
+
# htadmin (htpasswd/user management)
server {
listen 488 ssl;
@@ -195,6 +199,12 @@ http {
proxy_cache off;
}
+ location ~* ^/extracted-files\b(.*) {
+ proxy_pass http://docker-extracted-file-http-server$1;
+ proxy_redirect off;
+ proxy_set_header Host filemon.malcolm.local;
+ }
+
location = /favicon.ico {
alias /etc/nginx/favicon.ico;
}
diff --git a/sensor-iso/config/hooks/normal/0169-pip-installs.hook.chroot b/sensor-iso/config/hooks/normal/0169-pip-installs.hook.chroot
index 575c7f9a2..b18972b9a 100755
--- a/sensor-iso/config/hooks/normal/0169-pip-installs.hook.chroot
+++ b/sensor-iso/config/hooks/normal/0169-pip-installs.hook.chroot
@@ -17,6 +17,7 @@ pip3 install --no-compile --no-cache-dir --force-reinstall --upgrade \
ipaddress \
netifaces \
psutil \
+ pycryptodome \
pyinotify \
python-magic \
pythondialog \
diff --git a/shared/bin/zeek_carved_http_server.py b/shared/bin/zeek_carved_http_server.py
new file mode 100755
index 000000000..8a6ec4d8e
--- /dev/null
+++ b/shared/bin/zeek_carved_http_server.py
@@ -0,0 +1,182 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+# Multithreaded simple HTTP directory server.
+#
+# The files can optionally be aes-256-cbc encrypted in a way that's compatible with:
+# openssl enc -aes-256-cbc -d -in encrypted.data -out decrypted.data
+
+import argparse
+import hashlib
+import os
+import sys
+from threading import Thread
+from socketserver import ThreadingMixIn
+from http.server import HTTPServer, SimpleHTTPRequestHandler
+from Crypto.Cipher import AES
+
+KEY_SIZE = 32
+OPENSSL_ENC_MAGIC = b'Salted__'
+PKCS5_SALT_LEN = 8
+
+###################################################################################################
+args = None
+debug = False
+script_name = os.path.basename(__file__)
+script_path = os.path.dirname(os.path.realpath(__file__))
+orig_path = os.getcwd()
+
+###################################################################################################
+# print to stderr
+def eprint(*args, **kwargs):
+ print(*args, file=sys.stderr, **kwargs)
+ sys.stderr.flush()
+
+###################################################################################################
+# convenient boolean argument parsing
+def str2bool(v):
+ if v.lower() in ('yes', 'true', 't', 'y', '1'):
+ return True
+ elif v.lower() in ('no', 'false', 'f', 'n', '0'):
+ return False
+ else:
+ raise argparse.ArgumentTypeError('Boolean value expected.')
+
+###################################################################################################
+# EVP_BytesToKey
+#
+# reference: https://github.com/openssl/openssl/blob/6f0ac0e2f27d9240516edb9a23b7863e7ad02898/crypto/evp/evp_key.c#L74
+# https://gist.github.com/chrono-meter/d122cbefc6f6248a0af554995f072460
+def EVP_BytesToKey(key_length: int, iv_length: int, md, salt: bytes, data: bytes, count: int=1) -> (bytes, bytes):
+ assert data
+ assert salt == b'' or len(salt) == PKCS5_SALT_LEN
+
+ md_buf = b''
+ key = b''
+ iv = b''
+ addmd = 0
+
+ while key_length > len(key) or iv_length > len(iv):
+ c = md()
+ if addmd:
+ c.update(md_buf)
+ addmd += 1
+ c.update(data)
+ c.update(salt)
+ md_buf = c.digest()
+ for i in range(1, count):
+ md_buf = md(md_buf)
+
+ md_buf2 = md_buf
+
+ if key_length > len(key):
+ key, md_buf2 = key + md_buf2[:key_length - len(key)], md_buf2[key_length - len(key):]
+
+ if iv_length > len(iv):
+ iv = iv + md_buf2[:iv_length - len(iv)]
+
+ return key, iv
+
+###################################################################################################
+#
+class HTTPHandler(SimpleHTTPRequestHandler):
+
+ # return full path based on server base path and requested path
+ def translate_path(self, path):
+ path = SimpleHTTPRequestHandler.translate_path(self, path)
+ relpath = os.path.relpath(path, os.getcwd())
+ fullpath = os.path.join(self.server.base_path, relpath)
+ return fullpath
+
+ # override do_GET so that files are encrypted, if requested
+ def do_GET(self):
+ global debug
+ global args
+
+ fullpath = self.translate_path(self.path)
+
+ if (not args.encrypt) or os.path.isdir(fullpath):
+ # unencrypted, just use default implementation
+ SimpleHTTPRequestHandler.do_GET(self)
+
+ else:
+ # encrypt file transfers
+ if os.path.isfile(fullpath) or os.path.islink(fullpath):
+ self.send_response(200)
+ self.send_header('Content-type', 'application/octet-stream')
+ self.send_header('Content-Disposition', f'attachment; filename={os.path.basename(fullpath)}.encrypted')
+ self.end_headers()
+ salt = os.urandom(PKCS5_SALT_LEN)
+ key, iv = EVP_BytesToKey(KEY_SIZE, AES.block_size, hashlib.sha256, salt, args.key.encode('utf-8'))
+ cipher = AES.new(key, AES.MODE_CBC, iv)
+ encrypted = b""
+ encrypted += OPENSSL_ENC_MAGIC
+ encrypted += salt
+ self.wfile.write(encrypted)
+ with open(fullpath, 'rb') as f:
+ padding = b''
+ while True:
+ chunk = f.read(cipher.block_size)
+ if len(chunk) < cipher.block_size:
+ remaining = cipher.block_size - len(chunk)
+ padding = bytes([remaining] * remaining)
+ self.wfile.write(cipher.encrypt(chunk + padding))
+ if padding:
+ break
+
+ else:
+ self.send_error(404, "Not Found")
+
+###################################################################################################
+#
+class ThreadingHTTPServer(ThreadingMixIn, HTTPServer):
+ def __init__(self, base_path, server_address, RequestHandlerClass=HTTPHandler):
+ self.base_path = base_path
+ HTTPServer.__init__(self, server_address, RequestHandlerClass)
+
+###################################################################################################
+#
+def serve_on_port(path : str, port : int):
+ server = ThreadingHTTPServer(path, ("", port))
+ print(f"serving {path} at port {port}")
+ server.serve_forever()
+
+###################################################################################################
+# main
+def main():
+ global args
+ global debug
+ global orig_path
+
+ defaultDebug = os.getenv('EXTRACTED_FILE_HTTP_SERVER_DEBUG', 'false')
+ defaultEncrypt = os.getenv('EXTRACTED_FILE_HTTP_SERVER_ENCRYPT', 'false')
+ defaultPort = int(os.getenv('EXTRACTED_FILE_HTTP_SERVER_PORT', 8440))
+ defaultKey = os.getenv('EXTRACTED_FILE_HTTP_SERVER_KEY', 'quarantined')
+ defaultDir = os.getenv('EXTRACTED_FILE_HTTP_SERVER_PATH', orig_path)
+
+ parser = argparse.ArgumentParser(description=script_name, add_help=False, usage='{} '.format(script_name))
+ parser.add_argument('-v', '--verbose', dest='debug', type=str2bool, nargs='?', const=True, default=defaultDebug, metavar='true|false', help=f"Verbose/debug output ({defaultDebug})")
+ parser.add_argument('-p', '--port', dest='port', help=f"Server port ({defaultPort})", metavar='', type=int, default=defaultPort)
+ parser.add_argument('-d', '--directory', dest='serveDir', help=f'Directory to serve ({defaultDir})', metavar='', type=str, default=defaultDir)
+ parser.add_argument('-e', '--encrypt', dest='encrypt', type=str2bool, nargs='?', const=True, default=defaultEncrypt, metavar='true|false', help=f"Encrypt files with aes-256-cbc ({defaultEncrypt})")
+ parser.add_argument('-k', '--key', dest='key', help=f"File encryption key", metavar='', type=str, default=defaultKey)
+ try:
+ parser.error = parser.exit
+ args = parser.parse_args()
+ except SystemExit:
+ parser.print_help()
+ exit(2)
+
+ debug = args.debug
+ if debug:
+ eprint(os.path.join(script_path, script_name))
+ eprint("Arguments: {}".format(sys.argv[1:]))
+ eprint("Arguments: {}".format(args))
+ else:
+ sys.tracebacklimit = 0
+
+ Thread(target=serve_on_port, args=[args.serveDir, args.port]).start()
+
+###################################################################################################
+if __name__ == '__main__':
+ main()