Topic/2.6.1 merge (#159)

Malcolm v2.6.1 contains the following changes: v2.6.0...v2.6.1 * Added [TFTP](https://en.wikipedia.org/wiki/Trivial_File_Transfer_Protocol) [Zeek parser](https://github.com/zeek/spicy-tftp) and corresponding Logstash parsing, Arkime WISE support and Kibana dashboards * Provide browser-based access to zeek/extracted-files directory (idaholab#34) * Fix LDAP analyzer not parsing all events (idaholab#35) * Provide more fine-tuned controls for Zeek's node.cfg in Hedgehog sensor (idaholab#36, /pull/158) * set zeek.uid to conn_uids for files.log entries (idaholab#33) * Modify Zeek build chain to use default GCC compilers instead of LLVM/clang,which reduces build dependencies * Use Firefox instead of Chromium for browser in ISO-installed versions of Malcolm and in Hedgehog Linux * Updated copyright notices in text from "2020" to "2021" (which is the bulk of the changed files in this commit) * Version bumps * Yara to 4.0.4
cisagov · Feb 5, 2021 · 8d5e416 · 8d5e416
1 parent ad3e65a
commit 8d5e416
Show file tree

Hide file tree

Showing 256 changed files with 1,544 additions and 713 deletions.
diff --git a/Dockerfiles/arkime.Dockerfile b/Dockerfiles/arkime.Dockerfile
@@ -1,6 +1,6 @@
 FROM debian:buster-slim AS build
 
-# Copyright (c) 2020 Battelle Energy Alliance, LLC.  All rights reserved.
+# Copyright (c) 2021 Battelle Energy Alliance, LLC.  All rights reserved.
 
 ENV DEBIAN_FRONTEND noninteractive
 

diff --git a/Dockerfiles/curator.Dockerfile b/Dockerfiles/curator.Dockerfile
@@ -1,6 +1,6 @@
 FROM debian:buster-slim
 
-# Copyright (c) 2020 Battelle Energy Alliance, LLC.  All rights reserved.
+# Copyright (c) 2021 Battelle Energy Alliance, LLC.  All rights reserved.
 LABEL maintainer="malcolm.netsec@gmail.com"
 LABEL org.opencontainers.image.authors='malcolm.netsec@gmail.com'
 LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm'

diff --git a/Dockerfiles/elastalert.Dockerfile b/Dockerfiles/elastalert.Dockerfile
@@ -1,6 +1,6 @@
 FROM mmguero/elastalert:2.0.5
 
-# Copyright (c) 2020 Battelle Energy Alliance, LLC.  All rights reserved.
+# Copyright (c) 2021 Battelle Energy Alliance, LLC.  All rights reserved.
 LABEL maintainer="malcolm.netsec@gmail.com"
 LABEL org.opencontainers.image.authors='malcolm.netsec@gmail.com'
 LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm'

diff --git a/Dockerfiles/elasticsearch.Dockerfile b/Dockerfiles/elasticsearch.Dockerfile
@@ -1,6 +1,6 @@
 FROM docker.elastic.co/elasticsearch/elasticsearch-oss:7.6.2
 
-# Copyright (c) 2020 Battelle Energy Alliance, LLC.  All rights reserved.
+# Copyright (c) 2021 Battelle Energy Alliance, LLC.  All rights reserved.
 LABEL maintainer="malcolm.netsec@gmail.com"
 LABEL org.opencontainers.image.authors='malcolm.netsec@gmail.com'
 LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm'

diff --git a/Dockerfiles/file-monitor.Dockerfile b/Dockerfiles/file-monitor.Dockerfile
@@ -1,6 +1,6 @@
 FROM debian:buster-slim
 
-# Copyright (c) 2020 Battelle Energy Alliance, LLC.  All rights reserved.
+# Copyright (c) 2021 Battelle Energy Alliance, LLC.  All rights reserved.
 LABEL maintainer="malcolm.netsec@gmail.com"
 LABEL org.opencontainers.image.authors='malcolm.netsec@gmail.com'
 LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm'
@@ -44,6 +44,11 @@ ARG EXTRACTED_FILE_ENABLE_YARA=false
 ARG EXTRACTED_FILE_YARA_CUSTOM_ONLY=false
 ARG EXTRACTED_FILE_ENABLE_CAPA=false
 ARG EXTRACTED_FILE_CAPA_VERBOSE=false
+ARG EXTRACTED_FILE_HTTP_SERVER_DEBUG=false
+ARG EXTRACTED_FILE_HTTP_SERVER_ENABLE=false
+ARG EXTRACTED_FILE_HTTP_SERVER_ENCRYPT=false
+ARG EXTRACTED_FILE_HTTP_SERVER_KEY=quarantined
+ARG EXTRACTED_FILE_HTTP_SERVER_PORT=8440
 
 ENV ZEEK_EXTRACTOR_PATH $ZEEK_EXTRACTOR_PATH
 ENV ZEEK_LOG_DIRECTORY $ZEEK_LOG_DIRECTORY
@@ -70,14 +75,19 @@ ENV EXTRACTED_FILE_ENABLE_CAPA $EXTRACTED_FILE_ENABLE_CAPA
 ENV EXTRACTED_FILE_CAPA_VERBOSE $EXTRACTED_FILE_CAPA_VERBOSE
 ENV SRC_BASE_DIR "/usr/local/src"
 ENV CLAMAV_RULES_DIR "/var/lib/clamav"
-ENV YARA_VERSION "4.0.2"
+ENV YARA_VERSION "4.0.4"
 ENV YARA_URL "https://github.com/VirusTotal/yara/archive/v${YARA_VERSION}.tar.gz"
 ENV YARA_RULES_URL "https://github.com/Neo23x0/signature-base"
 ENV YARA_RULES_DIR "/yara-rules"
 ENV YARA_RULES_SRC_DIR "$SRC_BASE_DIR/signature-base"
 ENV CAPA_URL "https://github.com/fireeye/capa"
 ENV CAPA_RULES_URL "https://github.com/fireeye/capa-rules"
 ENV CAPA_RULES_DIR "/capa-rules"
+ENV EXTRACTED_FILE_HTTP_SERVER_DEBUG $EXTRACTED_FILE_HTTP_SERVER_DEBUG
+ENV EXTRACTED_FILE_HTTP_SERVER_ENABLE $EXTRACTED_FILE_HTTP_SERVER_ENABLE
+ENV EXTRACTED_FILE_HTTP_SERVER_ENCRYPT $EXTRACTED_FILE_HTTP_SERVER_ENCRYPT
+ENV EXTRACTED_FILE_HTTP_SERVER_KEY $EXTRACTED_FILE_HTTP_SERVER_KEY
+ENV EXTRACTED_FILE_HTTP_SERVER_PORT $EXTRACTED_FILE_HTTP_SERVER_PORT
 
 ENV SUPERCRONIC_VERSION "0.1.12"
 ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64"
@@ -123,7 +133,7 @@ RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list
       python3-pyinotify \
       python3-requests \
       python3-zmq && \
-    pip3 install clamd supervisor yara-python python-magic psutil && \
+    pip3 install clamd supervisor yara-python python-magic psutil pycryptodome && \
     pip2 install flare-capa && \
     curl -fsSLO "$SUPERCRONIC_URL" && \
       echo "${SUPERCRONIC_SHA1SUM}  ${SUPERCRONIC}" | sha1sum -c - && \
@@ -204,7 +214,7 @@ RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list
       echo "0 */6 * * * /bin/bash /usr/local/bin/capa-rules-update.sh\n0 */6 * * * /bin/bash /usr/local/bin/yara-rules-update.sh" > ${SUPERCRONIC_CRONTAB}
 
 ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
-ADD shared/bin/zeek_carve_*.py /usr/local/bin/
+ADD shared/bin/zeek_carve*.py /usr/local/bin/
 ADD shared/bin/malass_client.py /usr/local/bin/
 ADD file-monitor/supervisord.conf /etc/supervisord.conf
 ADD file-monitor/docker-entrypoint.sh /docker-entrypoint.sh

diff --git a/Dockerfiles/file-upload.Dockerfile b/Dockerfiles/file-upload.Dockerfile
@@ -1,6 +1,6 @@
 FROM debian:buster-slim AS build
 
-# Copyright (c) 2020 Battelle Energy Alliance, LLC.  All rights reserved.
+# Copyright (c) 2021 Battelle Energy Alliance, LLC.  All rights reserved.
 
 ENV DEBIAN_FRONTEND noninteractive
 

diff --git a/Dockerfiles/filebeat.Dockerfile b/Dockerfiles/filebeat.Dockerfile
@@ -1,6 +1,6 @@
 FROM docker.elastic.co/beats/filebeat-oss:7.6.2
 
-# Copyright (c) 2020 Battelle Energy Alliance, LLC.  All rights reserved.
+# Copyright (c) 2021 Battelle Energy Alliance, LLC.  All rights reserved.
 LABEL maintainer="malcolm.netsec@gmail.com"
 LABEL org.opencontainers.image.authors='malcolm.netsec@gmail.com'
 LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm'

diff --git a/Dockerfiles/freq.Dockerfile b/Dockerfiles/freq.Dockerfile
@@ -1,6 +1,6 @@
 FROM debian:buster-slim
 
-# Copyright (c) 2020 Battelle Energy Alliance, LLC.  All rights reserved.
+# Copyright (c) 2021 Battelle Energy Alliance, LLC.  All rights reserved.
 LABEL maintainer="malcolm.netsec@gmail.com"
 LABEL org.opencontainers.image.authors='malcolm.netsec@gmail.com'
 LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm'

diff --git a/Dockerfiles/htadmin.Dockerfile b/Dockerfiles/htadmin.Dockerfile
@@ -1,6 +1,6 @@
 FROM debian:buster-slim
 
-# Copyright (c) 2020 Battelle Energy Alliance, LLC.  All rights reserved.
+# Copyright (c) 2021 Battelle Energy Alliance, LLC.  All rights reserved.
 LABEL maintainer="malcolm.netsec@gmail.com"
 LABEL org.opencontainers.image.authors='malcolm.netsec@gmail.com'
 LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm'

diff --git a/Dockerfiles/kibana.Dockerfile b/Dockerfiles/kibana.Dockerfile
@@ -1,6 +1,6 @@
 FROM docker.elastic.co/kibana/kibana-oss:7.6.2
 
-# Copyright (c) 2020 Battelle Energy Alliance, LLC.  All rights reserved.
+# Copyright (c) 2021 Battelle Energy Alliance, LLC.  All rights reserved.
 LABEL maintainer="malcolm.netsec@gmail.com"
 LABEL org.opencontainers.image.authors='malcolm.netsec@gmail.com'
 LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm'

diff --git a/Dockerfiles/logstash.Dockerfile b/Dockerfiles/logstash.Dockerfile
@@ -1,6 +1,6 @@
 FROM centos:7 AS build
 
-# Copyright (c) 2020 Battelle Energy Alliance, LLC.  All rights reserved.
+# Copyright (c) 2021 Battelle Energy Alliance, LLC.  All rights reserved.
 
 RUN yum install -y epel-release && \
     yum update -y && \

diff --git a/Dockerfiles/name-map-ui.Dockerfile b/Dockerfiles/name-map-ui.Dockerfile
@@ -1,6 +1,6 @@
 FROM alpine:3.12
 
-# Copyright (c) 2020 Battelle Energy Alliance, LLC.  All rights reserved.
+# Copyright (c) 2021 Battelle Energy Alliance, LLC.  All rights reserved.
 LABEL maintainer="malcolm.netsec@gmail.com"
 LABEL org.opencontainers.image.authors='malcolm.netsec@gmail.com'
 LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm'

diff --git a/Dockerfiles/nginx.Dockerfile b/Dockerfiles/nginx.Dockerfile
@@ -1,4 +1,4 @@
-# Copyright (c) 2020 Battelle Energy Alliance, LLC.  All rights reserved.
+# Copyright (c) 2021 Battelle Energy Alliance, LLC.  All rights reserved.
 
 ####################################################################################
 # thanks to:  nginx                       -  https://github.com/nginxinc/docker-nginx/blob/master/mainline/alpine/Dockerfile

diff --git a/Dockerfiles/pcap-capture.Dockerfile b/Dockerfiles/pcap-capture.Dockerfile
@@ -1,6 +1,6 @@
 FROM debian:buster-slim
 
-# Copyright (c) 2020 Battelle Energy Alliance, LLC.  All rights reserved.
+# Copyright (c) 2021 Battelle Energy Alliance, LLC.  All rights reserved.
 LABEL maintainer="malcolm.netsec@gmail.com"
 LABEL org.opencontainers.image.authors='malcolm.netsec@gmail.com'
 LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm'

diff --git a/Dockerfiles/pcap-monitor.Dockerfile b/Dockerfiles/pcap-monitor.Dockerfile
@@ -1,6 +1,6 @@
 FROM debian:buster-slim
 
-# Copyright (c) 2020 Battelle Energy Alliance, LLC.  All rights reserved.
+# Copyright (c) 2021 Battelle Energy Alliance, LLC.  All rights reserved.
 LABEL maintainer="malcolm.netsec@gmail.com"
 LABEL org.opencontainers.image.authors='malcolm.netsec@gmail.com'
 LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm'

diff --git a/Dockerfiles/zeek.Dockerfile b/Dockerfiles/zeek.Dockerfile
@@ -1,79 +1,63 @@
 FROM debian:buster-slim AS build
 
-# Copyright (c) 2020 Battelle Energy Alliance, LLC.  All rights reserved.
+# Copyright (c) 2021 Battelle Energy Alliance, LLC.  All rights reserved.
 
 ENV DEBIAN_FRONTEND noninteractive
 
 # build zeek and plugins (spicy, additional protocol parsers, etc.)
 
-ENV BISON_VERSION "3.7.4"
 ENV CCACHE_DIR "/var/spool/ccache"
 ENV CCACHE_COMPRESS 1
-ENV CMAKE_DIR "/opt/cmake"
-ENV CMAKE_VERSION "3.19.3"
 ENV SPICY_DIR "/opt/spicy"
 ENV SRC_BASE_DIR "/usr/local/src"
 ENV ZEEK_DIR "/opt/zeek"
 ENV ZEEK_PATCH_DIR "${SRC_BASE_DIR}/zeek-patches"
 ENV ZEEK_SRC_DIR "${SRC_BASE_DIR}/zeek-${ZEEK_VERSION}"
 ENV ZEEK_VERSION "3.0.12"
 
-# using clang now instead of gcc because Spicy depends on it
-ENV LLVM_VERSION "11"
-ENV CC "clang-${LLVM_VERSION}"
-ENV CXX "clang++-${LLVM_VERSION}"
-ENV ASM "clang-${LLVM_VERSION}"
-
-ENV PATH "${ZEEK_DIR}/bin:${CMAKE_DIR}/bin:${PATH}"
+ENV PATH "${ZEEK_DIR}/bin:${PATH}"
 
 ADD shared/bin/zeek_install_plugins.sh /usr/local/bin/
 # empty for now...
 # ADD zeek/patches ${ZEEK_PATCH_DIR}
 
-RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list && \
-      echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list && \
+RUN echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list && \
+    echo "deb http://deb.debian.org/debian testing main" >> /etc/apt/sources.list && \
+    echo "Package: bison,build-essential,cmake,python3,python3-dev,python3-pip,python3-setuptools,python3-wheel\\nPin: release a=testing\\nPin-Priority: 800\\n\\nPackage: *\\nPin: release a=stable\\nPin-Priority: 700\\n\\nPackage: *\\nPin: release a=buster-backports\\nPin-Priority: 650\\n\\n\\nPackage: *\\nPin: release a=testing\\nPin-Priority: 600\\n" > /etc/apt/preferences.d/pin && \
+    echo 'APT::Default-Release "buster";' >> /etc/apt/apt.conf && \
       apt-get -q update && \
       apt-get install -q -y --no-install-recommends gnupg2 curl ca-certificates && \
-      bash -c "curl -sSL https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -" && \
-      echo "deb http://apt.llvm.org/buster/ llvm-toolchain-buster-${LLVM_VERSION} main" >> /etc/apt/sources.list && \
     apt-get -q update && \
     apt-get install -q -y -t buster-backports --no-install-recommends \
         binutils \
         ccache \
-        clang-${LLVM_VERSION} \
         file \
         flex \
         git \
         google-perftools \
         jq \
-        libclang-${LLVM_VERSION}-dev \
         libfl-dev \
         libgoogle-perftools-dev \
         libkrb5-dev \
         libmaxminddb-dev \
         libpcap0.8-dev \
         libssl-dev \
-        llvm-${LLVM_VERSION}-dev \
         locales-all \
         make \
         ninja-build \
         patch \
-        python3 \
-        python3-dev \
-        python3-pip \
-        python3-setuptools \
-        python3-wheel \
         swig \
         zlib1g-dev && \
+  apt-get install -q -y -t testing --no-install-recommends \
+    bison \
+    build-essential \
+    cmake \
+    python3 \
+    python3-dev \
+    python3-pip \
+    python3-setuptools \
+    python3-wheel && \
   pip3 install --no-cache-dir zkg btest pre-commit && \
-  mkdir -p "${CMAKE_DIR}" && \
-    curl -sSL "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz" | tar xzf - -C "${CMAKE_DIR}" --strip-components 1 && \
-  cd "${SRC_BASE_DIR}" && \
-    curl -sSL "https://ftp.gnu.org/gnu/bison/bison-${BISON_VERSION}.tar.gz" | tar xzf - -C "${SRC_BASE_DIR}" && \
-    cd "./bison-${BISON_VERSION}" && \
-    ./configure --prefix=/usr && \
-    make && \
-    make install && \
   cd "${SRC_BASE_DIR}" && \
     curl -sSL "https://old.zeek.org/downloads/zeek-${ZEEK_VERSION}.tar.gz" | tar xzf - -C "${SRC_BASE_DIR}" && \
     cd "./zeek-${ZEEK_VERSION}" && \
@@ -109,29 +93,24 @@ ENV PUSER_PRIV_DROP true
 ENV DEBIAN_FRONTEND noninteractive
 ENV TERM xterm
 
-ENV LLVM_VERSION "11"
 ENV ZEEK_DIR "/opt/zeek"
 ENV SPICY_DIR "/opt/spicy"
 
 COPY --from=build ${ZEEK_DIR} ${ZEEK_DIR}
 COPY --from=build ${SPICY_DIR} ${SPICY_DIR}
 
-RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list && \
-      echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list && \
+RUN echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list && \
+    echo "deb http://deb.debian.org/debian testing main" >> /etc/apt/sources.list && \
+    echo "Package: bison,build-essential,cmake,python3,python3-dev,python3-pip,python3-setuptools,python3-wheel\\nPin: release a=testing\\nPin-Priority: 800\\n\\nPackage: *\\nPin: release a=stable\\nPin-Priority: 700\\n\\nPackage: *\\nPin: release a=buster-backports\\nPin-Priority: 650\\n\\n\\nPackage: *\\nPin: release a=testing\\nPin-Priority: 600\\n" > /etc/apt/preferences.d/pin && \
+    echo 'APT::Default-Release "buster";' >> /etc/apt/apt.conf && \
       apt-get -q update && \
       apt-get install -q -y --no-install-recommends gnupg2 curl ca-certificates && \
-      bash -c "curl -sSL https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -" && \
-      echo "deb http://apt.llvm.org/buster/ llvm-toolchain-buster-${LLVM_VERSION} main" >> /etc/apt/sources.list && \
     apt-get -q update && \
     apt-get install -q -y -t buster-backports --no-install-recommends \
       binutils \
       file \
       git \
       libatomic1 \
-      libclang-${LLVM_VERSION}-dev \
-      libclang-cpp${LLVM_VERSION} \
-      libclang-cpp${LLVM_VERSION}-dev \
-      libclang1-${LLVM_VERSION} \
       libgoogle-perftools4 \
       libkrb5-3 \
       libmaxminddb0 \
@@ -141,16 +120,19 @@ RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list
       libtcmalloc-minimal4 \
       libunwind8 \
       libzmq5 \
-      llvm-${LLVM_VERSION} \
       procps \
       psmisc \
       python \
+      supervisor \
+      vim-tiny && \
+    apt-get install -q -y -t testing --no-install-recommends \
+      bison \
+      build-essential \
+      cmake \
       python3 \
       python3-pip \
       python3-setuptools \
-      python3-wheel \
-      supervisor \
-      vim-tiny && \
+      python3-wheel && \
     pip3 install --no-cache-dir pyzmq && \
     bash -c "( find /opt/zeek/ -type l ! -exec test -r {} \; -print | xargs -r -l rm -vf ) || true" && \
     apt-get -q -y --purge remove libssl-dev && \

diff --git a/License.txt b/License.txt
@@ -2,7 +2,7 @@ Software License Agreement
 
 For more information, please see DHS_TOU.pdf.
 
-Malcolm is Copyright (c) 2020 Battelle Energy Alliance, LLC, and is developed
+Malcolm is Copyright (c) 2021 Battelle Energy Alliance, LLC, and is developed
 and released through the cooperation of the Cybersecurity and Infrastructure
 Security Agency of the U.S. Department of Homeland Security. All rights reserved.
 
@@ -22,7 +22,7 @@ means, and for any purpose whatsoever.
 https://github.com/cisagov/Malcolm
 
 
-Copyright 2020 Battelle Energy Alliance, LLC
+Copyright 2021 Battelle Energy Alliance, LLC
 
 
 ALL RIGHTS RESERVED

diff --git a/Notice.txt b/Notice.txt
@@ -4,7 +4,7 @@ https://github.com/cisagov/Malcolm
 
 See License.txt for license terms.
 
-Malcolm is Copyright (c) 2020 Battelle Energy Alliance, LLC, and is developed
+Malcolm is Copyright (c) 2021 Battelle Energy Alliance, LLC, and is developed
 and released through the cooperation of the Cybersecurity and Infrastructure
 Security Agency of the U.S. Department of Homeland Security. All rights reserved.