Merge pull request #300 from cisagov/v24.03.0_merge_cisagov

Malcolm v24.03.0

.dockerignore

@@ -28,8 +28,8 @@ arkime-logs
 arkime-raw
 kubernetes
 malcolm-iso
-sensor-iso
-sensor-raspi
+hedgehog-iso
+hedgehog-raspi
 nginx/nginx_ldap*.conf
 pcap
 _site

.github/workflows/sensor-iso-build-docker-wrap-push-ghcr.yml → .github/workflows/hedgehog-iso-build-docker-wrap-push-ghcr.yml

@@ -1,15 +1,15 @@
-name: sensor-iso-build-docker-wrap-push-ghcr
+name: hedgehog-iso-build-docker-wrap-push-ghcr
 
 on:
   push:
     branches:
       - main
       - development
     paths:
-      - 'sensor-iso/**'
+      - 'hedgehog-iso/**'
       - 'shared/bin/*'
       - '.trigger_iso_workflow_build'
-      - '.github/workflows/sensor-iso-build-docker-wrap-push-ghcr.yml'
+      - '.github/workflows/hedgehog-iso-build-docker-wrap-push-ghcr.yml'
   workflow_dispatch:
   repository_dispatch:
 
@@ -93,13 +93,13 @@ jobs:
       -
         name: Build image
         run: |
-          cp -r ./shared ./docs ./_config.yml ./_includes ./_layouts ./Gemfile ./README.md ./malcolm-iso/htpdate ./sensor-iso
-          cp ./scripts/malcolm_utils.py ./sensor-iso/shared/bin/
-          cp ./scripts/documentation_build.sh ./sensor-iso/docs/
-          cp -r ./arkime/patch ./sensor-iso/shared/arkime_patch
-          mkdir -p ./sensor-iso/suricata
-          cp -r ./suricata/rules-default ./sensor-iso/suricata/
-          pushd ./sensor-iso
+          cp -r ./shared ./docs ./_config.yml ./_includes ./_layouts ./Gemfile ./README.md ./malcolm-iso/htpdate ./hedgehog-iso
+          cp ./scripts/malcolm_utils.py ./hedgehog-iso/shared/bin/
+          cp ./scripts/documentation_build.sh ./hedgehog-iso/docs/
+          cp -r ./arkime/patch ./hedgehog-iso/shared/arkime_patch
+          mkdir -p ./hedgehog-iso/suricata
+          cp -r ./suricata/rules-default ./hedgehog-iso/suricata/
+          pushd ./hedgehog-iso
           echo "${{ steps.extract_malcolm_version.outputs.mversion }}" > ./shared/version.txt
           echo "${{ secrets.MAXMIND_GEOIP_DB_LICENSE_KEY }}" > ./shared/maxmind_license.txt
           echo "GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}" > ./shared/environment.chroot
@@ -115,7 +115,7 @@ jobs:
         uses: aquasecurity/trivy-action@master
         with:
           scan-type: 'fs'
-          scan-ref: './sensor-iso'
+          scan-ref: './hedgehog-iso'
           format: 'sarif'
           output: 'trivy-results.sarif'
           severity: 'HIGH,CRITICAL'
@@ -140,6 +140,6 @@ jobs:
         name: Build and push ISO image
         uses: docker/build-push-action@v5
         with:
-          context: ./sensor-iso
+          context: ./hedgehog-iso
           push: true
           tags: ghcr.io/${{ github.repository_owner }}/malcolm/hedgehog:${{ steps.extract_branch.outputs.branch }}

.github/workflows/sensor-raspi-build-docker-wrap-push-ghcr.yml → .github/workflows/hedgehog-raspi-build-docker-wrap-push-ghcr.yml

@@ -1,4 +1,4 @@
-name: sensor-raspi-build-docker-wrap-push-ghcr
+name: hedgehog-raspi-build-docker-wrap-push-ghcr
 
 on:
   # push:
@@ -76,7 +76,7 @@ jobs:
       -
         name: Build image
         run: |
-          pushd ./sensor-raspi
+          pushd ./hedgehog-raspi
           mkdir -p ./shared
           echo "${{ steps.extract_malcolm_version.outputs.mversion }}" > ./shared/version.txt
           echo "${{ secrets.MAXMIND_GEOIP_DB_LICENSE_KEY }}" > ./shared/maxmind_license.txt
@@ -97,6 +97,6 @@ jobs:
         name: Build and push IMG image
         uses: docker/build-push-action@v5
         with:
-          context: ./sensor-raspi
+          context: ./hedgehog-raspi
           push: true
           tags: ghcr.io/${{ github.repository_owner }}/malcolm/hedgehog-raspi:${{ steps.extract_branch.outputs.branch }}

.gitignore

@@ -37,13 +37,13 @@ malcolm_netbox_backup_*.gz
 *-build.log
 Gemfile.lock
 _site
-sensor-iso/_config.yml
-sensor-iso/_includes
-sensor-iso/_layouts
-sensor-iso/_site
-sensor-iso/docs
-sensor-iso/Gemfile
-sensor-iso/README.md
+hedgehog-iso/_config.yml
+hedgehog-iso/_includes
+hedgehog-iso/_layouts
+hedgehog-iso/_site
+hedgehog-iso/docs
+hedgehog-iso/Gemfile
+hedgehog-iso/README.md
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

Dockerfiles/arkime.Dockerfile

@@ -7,7 +7,7 @@ ENV TERM xterm
 ENV PYTHONDONTWRITEBYTECODE 1
 ENV PYTHONUNBUFFERED 1
 
-ENV ARKIME_VERSION "v5.0.0"
+ENV ARKIME_VERSION "v5.0.1"
 ENV ARKIME_DIR "/opt/arkime"
 ENV ARKIME_URL "https://github.com/arkime/arkime.git"
 ENV ARKIME_LOCALELASTICSEARCH no

Dockerfiles/dashboards.Dockerfile

@@ -1,4 +1,4 @@
-FROM opensearchproject/opensearch-dashboards:2.11.1
+FROM opensearchproject/opensearch-dashboards:2.12.0
 
 LABEL maintainer="malcolm@inl.gov"
 LABEL org.opencontainers.image.authors='malcolm@inl.gov'
@@ -20,7 +20,7 @@ ENV PUSER_PRIV_DROP true
 ENV TERM xterm
 
 ENV TINI_VERSION v0.19.0
-ENV OSD_TRANSFORM_VIS_VERSION 2.11.0
+ENV OSD_TRANSFORM_VIS_VERSION 2.12.0
 
 ARG NODE_OPTIONS="--max_old_space_size=4096"
 ENV NODE_OPTIONS $NODE_OPTIONS
@@ -39,10 +39,10 @@ RUN yum upgrade -y && \
     # Malcolm manages authentication and encryption via NGINX reverse proxy
     /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin remove securityDashboards --allow-root && \
     cd /tmp && \
-        unzip transformVis.zip opensearch-dashboards/transformVis/opensearch_dashboards.json opensearch-dashboards/transformVis/package.json && \
-        sed -i "s/2\.11\.0/2\.11\.1/g" opensearch-dashboards/transformVis/opensearch_dashboards.json && \
-        sed -i "s/2\.11\.0/2\.11\.1/g" opensearch-dashboards/transformVis/package.json && \
-        zip transformVis.zip opensearch-dashboards/transformVis/opensearch_dashboards.json opensearch-dashboards/transformVis/package.json && \
+        # unzip transformVis.zip opensearch-dashboards/transformVis/opensearch_dashboards.json opensearch-dashboards/transformVis/package.json && \
+        # sed -i "s/2\.12\.0/2\.12\.0/g" opensearch-dashboards/transformVis/opensearch_dashboards.json && \
+        # sed -i "s/2\.12\.0/2\.12\.0/g" opensearch-dashboards/transformVis/package.json && \
+        # zip transformVis.zip opensearch-dashboards/transformVis/opensearch_dashboards.json opensearch-dashboards/transformVis/package.json && \
         cd /usr/share/opensearch-dashboards/plugins && \
         /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin install file:///tmp/transformVis.zip --allow-root && \
         rm -rf /tmp/transformVis /tmp/opensearch-dashboards && \
@@ -59,37 +59,6 @@ ADD dashboards/opensearch_dashboards.yml /usr/share/opensearch-dashboards/config
 ADD dashboards/scripts/docker_entrypoint.sh /usr/local/bin/
 ADD scripts/malcolm_utils.py /usr/local/bin/
 
-# Yeah, I know about https://opensearch.org/docs/latest/dashboards/branding ... but I can't figure out a way
-# to specify the entries in the opensearch_dashboards.yml such that they are valid BOTH from the
-# internal opensearch code validating them AND the web browser retrieving them. So we're going scorched earth instead.
-
-COPY --chmod=644 docs/images/favicon/favicon192.png /usr/share/opensearch-dashboards/src/core/server/core_app/assets/favicons/android-chrome-192x192.png
-COPY --chmod=644 docs/images/favicon/favicon512.png /usr/share/opensearch-dashboards/src/core/server/core_app/assets/favicons/android-chrome-512x512.png
-COPY --chmod=644 docs/images/favicon/apple-touch-icon-precomposed.png /usr/share/opensearch-dashboards/src/core/server/core_app/assets/favicons/apple-touch-icon.png
-COPY --chmod=644 docs/images/favicon/favicon16.png /usr/share/opensearch-dashboards/src/core/server/core_app/assets/favicons/favicon-16x16.png
-COPY --chmod=644 docs/images/favicon/favicon32.png /usr/share/opensearch-dashboards/src/core/server/core_app/assets/favicons/favicon-32x32.png
-COPY --chmod=644 docs/images/favicon/favicon.ico /usr/share/opensearch-dashboards/src/core/server/core_app/assets/favicons/favicon.ico
-COPY --chmod=644 docs/images/favicon/favicon144.png /usr/share/opensearch-dashboards/src/core/server/core_app/assets/favicons/mstile-144x144.png
-COPY --chmod=644 docs/images/favicon/favicon150.png /usr/share/opensearch-dashboards/src/core/server/core_app/assets/favicons/mstile-150x150.png
-COPY --chmod=644 docs/images/favicon/favicon310.png /usr/share/opensearch-dashboards/src/core/server/core_app/assets/favicons/mstile-310x310.png
-COPY --chmod=644 docs/images/favicon/favicon70.png /usr/share/opensearch-dashboards/src/core/server/core_app/assets/favicons/mstile-70x70.png
-COPY --chmod=644 docs/images/logo/Malcolm.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch.svg
-COPY --chmod=644 docs/images/icon/malcolm_mark_dashboards.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch_center_mark.svg
-COPY --chmod=644 docs/images/icon/malcolm_mark_dashboards.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch_center_mark_on_dark.svg
-COPY --chmod=644 docs/images/icon/malcolm_mark_dashboards.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch_center_mark_on_light.svg
-COPY --chmod=644 docs/images/logo/Malcolm.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch_dashboards.svg
-COPY --chmod=644 docs/images/logo/malcolm_logo.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch_dashboards_on_dark.svg
-COPY --chmod=644 docs/images/logo/Malcolm.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch_dashboards_on_light.svg
-COPY --chmod=644 docs/images/icon/malcolm_mark_dashboards.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch_mark.svg
-COPY --chmod=644 docs/images/icon/malcolm_mark_dashboards.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch_mark_on_dark.svg
-COPY --chmod=644 docs/images/icon/malcolm_mark_dashboards.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch_mark_on_light.svg
-COPY --chmod=644 docs/images/logo/malcolm_logo.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch_on_dark.svg
-COPY --chmod=644 docs/images/logo/Malcolm.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch_on_light.svg
-COPY --chmod=644 docs/images/icon/malcolm_mark_dashboards.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch_spinner.svg
-COPY --chmod=644 docs/images/icon/malcolm_mark_dashboards.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch_spinner_on_dark.svg
-COPY --chmod=644 docs/images/icon/malcolm_mark_dashboards.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch_spinner_on_light.svg
-
-
 ENTRYPOINT ["/usr/bin/tini", \
             "--", \
             "/usr/local/bin/docker-uid-gid-setup.sh", \

Dockerfiles/netbox.Dockerfile

@@ -32,7 +32,7 @@ ENV SUPERCRONIC_CRONTAB "/etc/crontab"
 
 ENV NETBOX_INITIALIZERS_VERSION "ebf1f76"
 
-ENV YQ_VERSION "4.33.3"
+ENV YQ_VERSION "4.42.1"
 ENV YQ_URL "https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_amd64"
 
 ENV NETBOX_DEVICETYPE_LIBRARY_IMPORT_URL "https://codeload.github.com/netbox-community/Device-Type-Library-Import/tar.gz/develop"
@@ -106,6 +106,13 @@ RUN apt-get -q update && \
       mkdir -p ./repo && \
       curl -sSL "${NETBOX_DEVICETYPE_LIBRARY_URL}" | tar xzf - -C ./repo --strip-components 1 && \
       rm -rf ./repo/device-types/WatchGuard && \
+    "${NETBOX_PATH}/venv/bin/python" -m pip install --break-system-packages --no-compile --no-cache-dir --upgrade \
+      cryptography \
+      GitPython \
+      Jinja2 \
+      "Django>=4.2.10,<5" \
+      paramiko \
+      pillow && \
     mkdir -p "${NETBOX_PATH}/netbox/${BASE_PATH}" && \
       mv "${NETBOX_PATH}/netbox/static" "${NETBOX_PATH}/netbox/${BASE_PATH}/static" && \
       jq '. += { "settings": { "http": { "discard_unsafe_fields": false } } }' /etc/unit/nginx-unit.json | jq 'del(.listeners."[::]:8080")' | jq 'del(.listeners."[::]:8081")' | jq ".routes.main[0].match.uri = \"/${BASE_PATH}/static/*\"" > /etc/unit/nginx-unit-new.json && \

Dockerfiles/nginx.Dockerfile

@@ -232,7 +232,9 @@ RUN set -x ; \
   rm -rf /usr/src/* /var/tmp/* /var/cache/apk/* /nginx.tar.gz /nginx-auth-ldap.tar.gz /ngx_http_substitutions_filter_module-master.tar.gz; \
   touch /etc/nginx/nginx_ldap.conf /etc/nginx/nginx_blank.conf && \
   find /usr/share/nginx/html/ -type d -exec chmod 755 "{}" \; && \
-  find /usr/share/nginx/html/ -type f -exec chmod 644 "{}" \;
+  find /usr/share/nginx/html/ -type f -exec chmod 644 "{}" \; && \
+  cd /usr/share/nginx/html/assets/img && \
+  ln -s ./Malcolm_background.png ./bg-masthead.png
 
 COPY --from=docbuild /site/_site /usr/share/nginx/html/readme
 
@@ -242,9 +244,13 @@ ADD nginx/scripts /usr/local/bin/
 ADD nginx/*.conf /etc/nginx/
 ADD nginx/templates /etc/nginx/templates/
 ADD nginx/supervisord.conf /etc/
+COPY --chmod=644 docs/images/favicon/*.png /usr/share/nginx/html/assets/img/
+COPY --chmod=644 docs/images/icon/*.png /usr/share/nginx/html/assets/img/
+COPY --chmod=644 docs/images/icon/*.svg /usr/share/nginx/html/assets/img/
 COPY --chmod=644 docs/images/icon/favicon.ico /usr/share/nginx/html/assets/favicon.ico
 COPY --chmod=644 docs/images/icon/favicon.ico /usr/share/nginx/html/favicon.ico
-COPY --chmod=644 docs/images/logo/Malcolm_background.png /usr/share/nginx/html/assets/img/bg-masthead.png
+COPY --chmod=644 docs/images/logo/*.png /usr/share/nginx/html/assets/img/
+COPY --chmod=644 docs/images/logo/*.svg /usr/share/nginx/html/assets/img/
 
 VOLUME ["/etc/nginx/certs", "/etc/nginx/dhparam"]
 

Dockerfiles/opensearch.Dockerfile

@@ -1,4 +1,4 @@
-FROM opensearchproject/opensearch:2.11.1
+FROM opensearchproject/opensearch:2.12.0
 
 # Copyright (c) 2024 Battelle Energy Alliance, LLC.  All rights reserved.
 LABEL maintainer="malcolm@inl.gov"

Dockerfiles/suricata.Dockerfile

@@ -36,7 +36,7 @@ ENV SUPERCRONIC "supercronic-linux-amd64"
 ENV SUPERCRONIC_SHA1SUM "cd48d45c4b10f3f0bfdd3a57d054cd05ac96812b"
 ENV SUPERCRONIC_CRONTAB "/etc/crontab"
 
-ENV YQ_VERSION "4.33.3"
+ENV YQ_VERSION "4.42.1"
 ENV YQ_URL "https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_amd64"
 
 ENV SURICATA_CONFIG_DIR /etc/suricata

Dockerfiles/zeek.Dockerfile

@@ -160,8 +160,8 @@ ADD shared/bin/zeekdeploy.sh ${ZEEK_DIR}/bin/
 
 # sanity checks to make sure the plugins installed and copied over correctly
 # these ENVs should match the number of third party scripts/plugins installed by zeek_install_plugins.sh
-ENV ZEEK_THIRD_PARTY_PLUGINS_COUNT 22
-ENV ZEEK_THIRD_PARTY_PLUGINS_GREP  "(Zeek::Spicy|ANALYZER_SPICY_DHCP|ANALYZER_SPICY_DNS|ANALYZER_SPICY_HTTP|ANALYZER_SPICY_OSPF|ANALYZER_SPICY_OPENVPN_UDP\b|ANALYZER_SPICY_IPSEC_UDP\b|ANALYZER_SPICY_TFTP|ANALYZER_SPICY_WIREGUARD|ANALYZER_SYNCHROPHASOR_TCP|ANALYZER_GENISYS_TCP|ANALYZER_S7COMM_TCP|Corelight::CommunityID|Corelight::PE_XOR|ICSNPP::BACnet|ICSNPP::BSAP|ICSNPP::ENIP|ICSNPP::ETHERCAT|ICSNPP::OPCUA_Binary|Salesforce::GQUIC|Zeek::PROFINET|Zeek::TDS)"
+ENV ZEEK_THIRD_PARTY_PLUGINS_COUNT 23
+ENV ZEEK_THIRD_PARTY_PLUGINS_GREP  "(Zeek::Spicy|ANALYZER_SPICY_DHCP|ANALYZER_SPICY_DNS|ANALYZER_SPICY_HTTP|ANALYZER_SPICY_OSPF|ANALYZER_SPICY_OPENVPN_UDP\b|ANALYZER_SPICY_IPSEC_UDP\b|ANALYZER_SPICY_TFTP|ANALYZER_SPICY_WIREGUARD|ANALYZER_SYNCHROPHASOR_TCP|ANALYZER_GENISYS_TCP|ANALYZER_SPICY_PROFINET_IO_CM|ANALYZER_S7COMM_TCP|Corelight::CommunityID|Corelight::PE_XOR|ICSNPP::BACnet|ICSNPP::BSAP|ICSNPP::ENIP|ICSNPP::ETHERCAT|ICSNPP::OPCUA_Binary|Salesforce::GQUIC|Zeek::PROFINET|Zeek::TDS)"
 ENV ZEEK_THIRD_PARTY_SCRIPTS_COUNT 25
 ENV ZEEK_THIRD_PARTY_SCRIPTS_GREP  "(bro-is-darknet/main|bro-simple-scan/scan|bzar/main|callstranger-detector/callstranger|cve-2020-0601/cve-2020-0601|cve-2020-13777/cve-2020-13777|CVE-2020-16898/CVE-2020-16898|CVE-2021-38647/omigod|CVE-2021-31166/detect|CVE-2021-41773/CVE_2021_41773|CVE-2021-42292/main|cve-2021-44228/CVE_2021_44228|cve-2022-22954/main|cve-2022-26809/main|CVE-2022-3602/__load__|hassh/hassh|http-more-files-names/main|ja3/ja3|pingback/detect|ripple20/ripple20|SIGRed/CVE-2020-1350|zeek-EternalSafety/main|zeek-httpattacks/main|zeek-sniffpass/__load__|zerologon/main)\.(zeek|bro)"
 
@@ -199,6 +199,7 @@ ARG ZEEK_INTEL_ITEM_EXPIRATION=-1min
 ARG ZEEK_INTEL_REFRESH_CRON_EXPRESSION=
 ARG ZEEK_INTEL_REFRESH_THREADS=2
 ARG ZEEK_INTEL_FEED_SINCE=
+ARG ZEEK_INTEL_FEED_SSL_CERTIFICATE_VERIFICATION=false
 ARG ZEEK_EXTRACTOR_MODE=none
 ARG ZEEK_EXTRACTOR_PATH=/zeek/extract_files
 ARG ZEEK_INTEL_PATH=/opt/zeek/share/zeek/site/intel
@@ -222,6 +223,7 @@ ENV ZEEK_INTEL_ITEM_EXPIRATION $ZEEK_INTEL_ITEM_EXPIRATION
 ENV ZEEK_INTEL_REFRESH_CRON_EXPRESSION $ZEEK_INTEL_REFRESH_CRON_EXPRESSION
 ENV ZEEK_INTEL_REFRESH_THREADS $ZEEK_INTEL_REFRESH_THREADS
 ENV ZEEK_INTEL_FEED_SINCE $ZEEK_INTEL_FEED_SINCE
+eNV ZEEK_INTEL_FEED_SSL_CERTIFICATE_VERIFICATION $ZEEK_INTEL_FEED_SSL_CERTIFICATE_VERIFICATION
 ENV ZEEK_EXTRACTOR_MODE $ZEEK_EXTRACTOR_MODE
 ENV ZEEK_EXTRACTOR_PATH $ZEEK_EXTRACTOR_PATH
 ENV ZEEK_INTEL_PATH $ZEEK_INTEL_PATH

_config.yml

@@ -84,7 +84,7 @@ exclude:
   - pcap-capture
   - pcap-monitor
   - scripts
-  - sensor-iso
+  - hedgehog-iso
   - shared
   - suricata
   - suricata-logs

api/requirements.txt

@@ -5,5 +5,5 @@ opensearch-py==2.4.2
 requests==2.31.0
 regex==2022.3.2
 dateparser==1.1.1
-elasticsearch==8.12.0
+elasticsearch==8.12.1
 elasticsearch-dsl==8.12.0

arkime/arkime_regression_test_harness/docker-compose.yml

@@ -7,7 +7,7 @@ services:
     environment:
       logger.level : 'INFO'
       bootstrap.memory_lock : 'true'
-      OPENSEARCH_JAVA_OPTS : '-Xms4g -Xmx4g -Xss256k -Djava.security.egd=file:/dev/./urandom'
+      OPENSEARCH_JAVA_OPTS : '-Xmx4g -Xms4g -Xss256k -Djava.security.egd=file:/dev/./urandom'
       discovery.type : 'single-node'
       cluster.routing.allocation.disk.threshold_enabled : 'false'
       cluster.routing.allocation.node_initial_primaries_recoveries : 8