for idaholab#456, adjustments to how Zeek intel files are generated i…

…n Malcolm containers

* remove ZEEK_CRON variable
* moved ZEEK_INTEL_REFRESH_CRON_EXPRESSION variable to zeek-offline.env for zeek-offline container only
* added ZEEK_INTEL_REFRESH_ON_ENTRYPOINT=false and ZEEK_INTEL_REFRESH_ON_DEPLOY=false
  - ZEEK_INTEL_REFRESH_ON_ENTRYPOINT is set to true for the zeek-offline container only
  - ZEEK_INTEL_REFRESH_ON_DEPLOY is only set to true on Hedgehog Linux installations (where we're not using Docker), in control_vars.conf
* zeek_intel_setup.sh now only sets up the crontab file for the container if ZEEK_INTEL_REFRESH_CRON_EXPRESSION is set
* zeek_deploy.sh checks ZEEK_INTEL_REFRESH_ON_DEPLOY before running zeek_intel_setup.sh
* zeek/scripts/docker_entrypoint.sh checks ZEEK_INTEL_REFRESH_ON_ENTRYPOINT before running zeek_intel_setup.sh
* supercronic runs in both containers, but only actually gets set up for Zeek intel pull when ZEEK_INTEL_REFRESH_CRON_EXPRESSION is set
  - this is because there may be other things we want cron to do here in the future, for now with an empty crontab the zeek-live one will just sleep

Dockerfiles/zeek.Dockerfile

@@ -190,13 +190,14 @@ RUN mkdir -p /tmp/logs && \
 ARG AUTO_TAG=true
 #Whether or not to start up the pcap_processor script to monitor pcaps
 ARG ZEEK_PCAP_PROCESSOR=true
-#Whether or not to start up supercronic for updating intel definitions
-ARG ZEEK_CRON=true
 #Whether or not to run "zeek -r XXXXX.pcap local" on each pcap file
 ARG ZEEK_AUTO_ANALYZE_PCAP_FILES=false
 ARG ZEEK_AUTO_ANALYZE_PCAP_THREADS=1
-ARG ZEEK_INTEL_ITEM_EXPIRATION=-1min
+#Whether or not to refresh intel at various points during processing
+ARG ZEEK_INTEL_REFRESH_ON_ENTRYPOINT=false
+ARG ZEEK_INTEL_REFRESH_ON_DEPLOY=false
 ARG ZEEK_INTEL_REFRESH_CRON_EXPRESSION=
+ARG ZEEK_INTEL_ITEM_EXPIRATION=-1min
 ARG ZEEK_INTEL_REFRESH_THREADS=2
 ARG ZEEK_INTEL_FEED_SINCE=
 ARG ZEEK_INTEL_FEED_SSL_CERTIFICATE_VERIFICATION=false
@@ -216,11 +217,12 @@ ARG PCAP_NODE_NAME=malcolm
 
 ENV AUTO_TAG $AUTO_TAG
 ENV ZEEK_PCAP_PROCESSOR $ZEEK_PCAP_PROCESSOR
-ENV ZEEK_CRON $ZEEK_CRON
+ENV ZEEK_INTEL_REFRESH_ON_ENTRYPOINT $ZEEK_INTEL_REFRESH_ON_ENTRYPOINT
+ENV ZEEK_INTEL_REFRESH_ON_DEPLOY $ZEEK_INTEL_REFRESH_ON_DEPLOY
+ENV ZEEK_INTEL_REFRESH_CRON_EXPRESSION $ZEEK_INTEL_REFRESH_CRON_EXPRESSION
 ENV ZEEK_AUTO_ANALYZE_PCAP_FILES $ZEEK_AUTO_ANALYZE_PCAP_FILES
 ENV ZEEK_AUTO_ANALYZE_PCAP_THREADS $ZEEK_AUTO_ANALYZE_PCAP_THREADS
 ENV ZEEK_INTEL_ITEM_EXPIRATION $ZEEK_INTEL_ITEM_EXPIRATION
-ENV ZEEK_INTEL_REFRESH_CRON_EXPRESSION $ZEEK_INTEL_REFRESH_CRON_EXPRESSION
 ENV ZEEK_INTEL_REFRESH_THREADS $ZEEK_INTEL_REFRESH_THREADS
 ENV ZEEK_INTEL_FEED_SINCE $ZEEK_INTEL_FEED_SINCE
 eNV ZEEK_INTEL_FEED_SSL_CERTIFICATE_VERIFICATION $ZEEK_INTEL_FEED_SSL_CERTIFICATE_VERIFICATION

config/zeek-live.env.example

@@ -5,6 +5,5 @@ ZEEK_LIVE_CAPTURE=false
 ZEEK_DISABLE_STATS=true
 
 ZEEK_PCAP_PROCESSOR=false
-ZEEK_CRON=true
 ZEEK_LOG_PATH=/zeek/live
 EXTRACT_FILES_PATH=/zeek/extract_files

config/zeek-offline.env.example

@@ -10,4 +10,10 @@ ZEEK_AUTO_ANALYZE_PCAP_THREADS=1
 ZEEK_ROTATED_PCAP=true
 
 ZEEK_PCAP_PROCESSOR=true
-ZEEK_CRON=false
+
+# Specifies whether or not to refresh Zeek Intelligence Framework files in
+#   the container entrypoint
+ZEEK_INTEL_REFRESH_ON_ENTRYPOINT=true
+# Specifies a cron expression indicating the refresh interval for generating the
+#   Zeek Intelligence Framework files (or blank to disable automatic refresh)
+ZEEK_INTEL_REFRESH_CRON_EXPRESSION=

config/zeek.env.example

@@ -13,9 +13,6 @@ ZEEK_INTEL_ITEM_EXPIRATION=-1min
 ZEEK_INTEL_FEED_SINCE=
 # Whether or not to require SSL certificate verification when querying a TAXII or MISP feed
 ZEEK_INTEL_FEED_SSL_CERTIFICATE_VERIFICATION=false
-# Specifies a cron expression indicating the refresh interval for generating the
-#   Zeek Intelligence Framework files ('' disables automatic refresh)
-ZEEK_INTEL_REFRESH_CRON_EXPRESSION=
 # Number of threads to use for querying feeds for generating Zeek Intelligence Framework files
 ZEEK_INTEL_REFRESH_THREADS=2
 # Determines the file extraction behavior for file transfers detected by Zeek

hedgehog-iso/interface/sensor_ctl/control_vars.conf

@@ -71,6 +71,7 @@ export ZEEK_AF_PACKET_BUFFER_SIZE=67108864
 export ZEEK_LOCAL_NETS=
 export ZEEK_JSON=
 export ZEEK_RULESET=local
+export ZEEK_INTEL_REFRESH_ON_DEPLOY=true
 export ZEEK_INTEL_ITEM_EXPIRATION=-1min
 export ZEEK_INTEL_FEED_SINCE=
 export ZEEK_EXTRACTOR_MODE=none

shared/bin/zeek_intel_setup.sh

@@ -12,6 +12,7 @@ shopt -s nocasematch
 ENCODING="utf-8"
 
 SCRIPT_FILESPEC="$(realpath -e "${BASH_SOURCE[0]}")"
+SCRIPT_FILESPEC_ESCAPED="$(printf '%s\n' "${SCRIPT_FILESPEC}" | sed -e 's/[\/&]/\\&/g')"
 ZEEK_DIR=${ZEEK_DIR:-"/opt/zeek"}
 ZEEK_INTEL_ITEM_EXPIRATION=${ZEEK_INTEL_ITEM_EXPIRATION:-"-1min"}
 ZEEK_INTEL_FEED_SINCE=${ZEEK_INTEL_FEED_SINCE:-""}
@@ -131,10 +132,10 @@ fi # singleton lock check
 # $ZEEK_INTEL_REFRESH_CRON_EXPRESSION (e.g., 15 1 * * *) to execute this script
 set +u
 if [[ -n "${SUPERCRONIC_CRONTAB}" ]] && [[ -f "${SUPERCRONIC_CRONTAB}" ]]; then
+    touch "${SUPERCRONIC_CRONTAB}"
+    sed -i -e "/${SCRIPT_FILESPEC_ESCAPED}/d" "${SUPERCRONIC_CRONTAB}"
     if [[ -n "${ZEEK_INTEL_REFRESH_CRON_EXPRESSION}" ]]; then
-        echo "${ZEEK_INTEL_REFRESH_CRON_EXPRESSION} ${SCRIPT_FILESPEC} true" > "${SUPERCRONIC_CRONTAB}"
-    else
-        > "${SUPERCRONIC_CRONTAB}"
+        echo "${ZEEK_INTEL_REFRESH_CRON_EXPRESSION} ${SCRIPT_FILESPEC} true" >> "${SUPERCRONIC_CRONTAB}"
     fi
     # reload supercronic if it's running
     killall -s USR2 supercronic >/dev/null 2>&1 || true

shared/bin/zeekdeploy.sh

@@ -67,6 +67,7 @@ fi
 [[ -z $WORKER_LB_PROCS ]] && WORKER_LB_PROCS="$ZEEK_LB_PROCS"
 [[ -z $ZEEK_LB_METHOD ]] && ZEEK_LB_METHOD="custom"
 [[ -z $ZEEK_AF_PACKET_BUFFER_SIZE ]] && ZEEK_AF_PACKET_BUFFER_SIZE="$(echo "64*1024*1024" | bc)"
+[[ -z $ZEEK_INTEL_REFRESH_ON_DEPLOY ]] && ZEEK_INTEL_REFRESH_ON_DEPLOY="true"
 
 # if zeek log path is unspecified, write logs to pwd
 [[ -z $ZEEK_LOG_PATH ]] && ZEEK_LOG_PATH=.
@@ -89,16 +90,18 @@ ZEEK_EXTRACTOR_SCRIPT="$ZEEK_INSTALL_PATH"/share/zeek/site/"$EXTRACTOR_ZEEK_SCRI
 [[ -n "$ZEEK_INTEL_PATH" ]] && INTEL_DIR="$ZEEK_INTEL_PATH" || INTEL_DIR=/opt/sensor/sensor_ctl/zeek/intel
 export INTEL_DIR
 mkdir -p "$INTEL_DIR"/STIX "$INTEL_DIR"/MISP
-touch "$INTEL_DIR"/__load__.zeek || true
+touch "$INTEL_DIR"/__load__.zeek 2>/dev/null || true
 # autoconfigure load directives for intel files
-[[ -x "$ZEEK_INSTALL_PATH"/bin/zeek_intel_setup.sh ]] && "$ZEEK_INSTALL_PATH"/bin/zeek_intel_setup.sh /bin/true
+[[ -x "$ZEEK_INSTALL_PATH"/bin/zeek_intel_setup.sh ]] && \
+  [[ "$ZEEK_INTEL_REFRESH_ON_DEPLOY" == "true" ]] && \
+  "$ZEEK_INSTALL_PATH"/bin/zeek_intel_setup.sh /bin/true
 INTEL_UPDATE_TIME_PREV=0
 
 # make sure "custom" directory exists, even if empty
 [[ -n "$ZEEK_CUSTOM_PATH" ]] && CUSTOM_DIR="$ZEEK_CUSTOM_PATH" || CUSTOM_DIR=/opt/sensor/sensor_ctl/zeek/custom
 export CUSTOM_DIR
 mkdir -p "$CUSTOM_DIR"
-touch "$CUSTOM_DIR"/__load__.zeek || true
+touch "$CUSTOM_DIR"/__load__.zeek 2>/dev/null || true
 
 # configure zeek cfg files
 pushd "$ZEEK_INSTALL_PATH"/etc >/dev/null 2>&1

zeek/scripts/docker_entrypoint.sh

@@ -6,8 +6,8 @@ ZEEK_DIR=${ZEEK_DIR:-"/opt/zeek"}
 setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' "${ZEEK_DIR}"/bin/zeek || true
 setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' "${ZEEK_DIR}"/bin/capstats || true
 
-if [[ "${ZEEK_LIVE_CAPTURE:-false}" != "true" ]] && [[ -x "${ZEEK_DIR}"/bin/zeek_intel_setup.sh ]]; then
-    sleep 15 # give the "live" instance, if there is one, a chance to go first
+if [[ "${ZEEK_INTEL_REFRESH_ON_ENTRYPOINT:-false}" == "true" ]] && \
+   [[ -x "${ZEEK_DIR}"/bin/zeek_intel_setup.sh ]]; then
     if [[ "$(id -u)" == "0" ]] && [[ -n "$PUSER" ]]; then
         su -s /bin/bash -p ${PUSER} << EOF
             "${ZEEK_DIR}"/bin/zeek_intel_setup.sh /bin/true

zeek/supervisord.conf

@@ -43,8 +43,8 @@ user=%(ENV_PUSER)s
 
 [program:cron]
 command=/usr/local/bin/supercronic -json "%(ENV_SUPERCRONIC_CRONTAB)s"
-autostart=%(ENV_ZEEK_CRON)s
-autorestart=%(ENV_ZEEK_CRON)s
+autostart=true
+autorestart=true
 stopasgroup=true
 killasgroup=true
 stdout_logfile=/dev/fd/1