diff --git a/Dockerfiles/dashboards-helper.Dockerfile b/Dockerfiles/dashboards-helper.Dockerfile index b7e17c98b..9426311f1 100644 --- a/Dockerfiles/dashboards-helper.Dockerfile +++ b/Dockerfiles/dashboards-helper.Dockerfile @@ -84,7 +84,15 @@ RUN apk update --no-cache && \ find /opt/ecs-templates-os -name "*.json" -exec sed -i 's/\("type"[[:space:]]*:[[:space:]]*\)"flattened"/\1"nested"/' "{}" \; && \ find /opt/ecs-templates-os -name "*.json" -exec sed -i 's/\("type"[[:space:]]*:[[:space:]]*\)"number"/\1"long"/' "{}" \; && \ rm -rf /opt/ecs && \ - chown -R ${PUSER}:${PGROUP} /opt/dashboards /opt/templates /opt/ecs-templates /opt/ecs-templates-os /opt/maps /data/init /opt/anomaly_detectors && \ + chown -R ${PUSER}:${PGROUP} /data/init \ + /opt/alerting \ + /opt/anomaly_detectors \ + /opt/dashboards \ + /opt/ecs-templates \ + /opt/ecs-templates-os \ + /opt/maps \ + /opt/notifications \ + /opt/templates && \ chmod 755 /data/*.sh /data/*.py /data/init && \ chmod 400 /opt/maps/* && \ (echo -e "*/2 * * * * /data/create-arkime-sessions-index.sh\n0 10 * * * /data/index-refresh.py --template malcolm_template --unassigned\n30 */2 * * * /data/index-refresh.py --index 'malcolm_beats_*' --template malcolm_beats_template --unassigned\n*/20 * * * * /data/opensearch_index_size_prune.py" > ${SUPERCRONIC_CRONTAB}) diff --git a/dashboards/scripts/create-arkime-sessions-index.sh b/dashboards/scripts/create-arkime-sessions-index.sh index 0af4afb7e..02c3dea46 100755 --- a/dashboards/scripts/create-arkime-sessions-index.sh +++ b/dashboards/scripts/create-arkime-sessions-index.sh @@ -37,6 +37,15 @@ function DoReplacersInFile() { fi } +function DoReplacersForDir() { + REPLDIR="$1" + if [[ -n "$REPLDIR" ]] && [[ -d "$REPLDIR" ]]; then + while IFS= read -r fname; do + DoReplacersInFile "$fname" + done < <( find "$REPLDIR"/ -type f 2>/dev/null ) + fi +} + # is the argument to automatically create this index enabled? if [[ "$CREATE_OS_ARKIME_SESSION_INDEX" = "true" ]] ; then @@ -109,9 +118,7 @@ if [[ "$CREATE_OS_ARKIME_SESSION_INDEX" = "true" ]] ; then TEMPLATES_IMPORT_DIR="$(mktemp -d -t templates-XXXXXX)" rsync -a "$MALCOLM_TEMPLATES_DIR"/ "$TEMPLATES_IMPORT_DIR"/ - while IFS= read -r fname; do - DoReplacersInFile "$fname" - done < <( find "$TEMPLATES_IMPORT_DIR"/ -type f 2>/dev/null ) + DoReplacersForDir "$TEMPLATES_IMPORT_DIR" MALCOLM_TEMPLATE_FILE_ORIG_TMP="$(echo "$MALCOLM_TEMPLATE_FILE_ORIG" | sed "s@$MALCOLM_TEMPLATES_DIR@$TEMPLATES_IMPORT_DIR@")" # calculate combined SHA sum of all templates to save as _meta.hash to determine if @@ -217,7 +224,8 @@ if [[ "$CREATE_OS_ARKIME_SESSION_INDEX" = "true" ]] ; then # install default dashboards DASHBOARDS_IMPORT_DIR="$(mktemp -d -t dashboards-XXXXXX)" - cp /opt/dashboards/*.json "${DASHBOARDS_IMPORT_DIR}"/ + rsync -a /opt/dashboards/ "$DASHBOARDS_IMPORT_DIR"/ + DoReplacersForDir "$DASHBOARDS_IMPORT_DIR"/ for i in "${DASHBOARDS_IMPORT_DIR}"/*.json; do if [[ "$DATASTORE_TYPE" == "elasticsearch" ]]; then # strip out Arkime and NetBox links from dashboards' navigation pane when doing Kibana import (idaholab/Malcolm#286) @@ -225,7 +233,6 @@ if [[ "$CREATE_OS_ARKIME_SESSION_INDEX" = "true" ]] ; then # take care of a few other substitutions sed -i 's/opensearchDashboardsAddFilter/kibanaAddFilter/g' "$i" fi - DoReplacersInFile "$i" curl "${CURL_CONFIG_PARAMS[@]}" -L --silent --output /dev/null --show-error -XPOST "$DASHB_URL/api/$DASHBOARDS_URI_PATH/dashboards/import?force=true" -H "$XSRF_HEADER:true" -H 'Content-type:application/json' -d "@$i" done rm -rf "${DASHBOARDS_IMPORT_DIR}" @@ -235,9 +242,9 @@ if [[ "$CREATE_OS_ARKIME_SESSION_INDEX" = "true" ]] ; then # opensearch-project/OpenSearch-Dashboards#831). As such, we're going to # manually add load our dashboards in /opt/dashboards/beats as well. BEATS_DASHBOARDS_IMPORT_DIR="$(mktemp -d -t beats-XXXXXX)" - cp /opt/dashboards/beats/*.json "${BEATS_DASHBOARDS_IMPORT_DIR}"/ + rsync -a /opt/dashboards/beats/ "$BEATS_DASHBOARDS_IMPORT_DIR"/ + DoReplacersForDir "$BEATS_DASHBOARDS_IMPORT_DIR" for i in "${BEATS_DASHBOARDS_IMPORT_DIR}"/*.json; do - DoReplacersInFile "$i" curl "${CURL_CONFIG_PARAMS[@]}" -L --silent --output /dev/null --show-error -XPOST "$DASHB_URL/api/$DASHBOARDS_URI_PATH/dashboards/import?force=true" -H "$XSRF_HEADER:true" -H 'Content-type:application/json' -d "@$i" done rm -rf "${BEATS_DASHBOARDS_IMPORT_DIR}" @@ -275,9 +282,9 @@ if [[ "$CREATE_OS_ARKIME_SESSION_INDEX" = "true" ]] ; then # Create anomaly detectors here ANOMALY_IMPORT_DIR="$(mktemp -d -t anomaly-XXXXXX)" - cp /opt/anomaly_detectors/*.json "${ANOMALY_IMPORT_DIR}"/ + rsync -a /opt/anomaly_detectors/ "$ANOMALY_IMPORT_DIR"/ + DoReplacersForDir "$ANOMALY_IMPORT_DIR" for i in "${ANOMALY_IMPORT_DIR}"/*.json; do - DoReplacersInFile "$i" curl "${CURL_CONFIG_PARAMS[@]}" -L --silent --output /dev/null --show-error -XPOST "$OPENSEARCH_URL_TO_USE/_plugins/_anomaly_detection/detectors" -H "$XSRF_HEADER:true" -H 'Content-type:application/json' -d "@$i" done rm -rf "${ANOMALY_IMPORT_DIR}" @@ -316,9 +323,9 @@ if [[ "$CREATE_OS_ARKIME_SESSION_INDEX" = "true" ]] ; then # monitors ALERTING_IMPORT_DIR="$(mktemp -d -t alerting-XXXXXX)" - cp /opt/alerting/monitors/*.json "${ALERTING_IMPORT_DIR}"/ + rsync -a /opt/alerting/monitors/ "$ALERTING_IMPORT_DIR"/ + DoReplacersForDir "$ALERTING_IMPORT_DIR" for i in "${ALERTING_IMPORT_DIR}"/*.json; do - DoReplacersInFile "$i" curl "${CURL_CONFIG_PARAMS[@]}" -L --silent --output /dev/null --show-error -XPOST "$OPENSEARCH_URL_TO_USE/_plugins/_alerting/monitors" -H "$XSRF_HEADER:true" -H 'Content-type:application/json' -d "@$i" done rm -rf "${ALERTING_IMPORT_DIR}"