diff --git a/arkime/scripts/docker_entrypoint.sh b/arkime/scripts/docker_entrypoint.sh index ddea2fc2f..8f94bfa94 100755 --- a/arkime/scripts/docker_entrypoint.sh +++ b/arkime/scripts/docker_entrypoint.sh @@ -13,6 +13,7 @@ ARKIME_RULES_DIR=${ARKIME_RULES_DIR:-"/opt/arkime/rules"} ARKIME_CONFIG_FILE="${ARKIME_DIR}"/etc/config.ini ARKIME_PASSWORD_SECRET=${ARKIME_PASSWORD_SECRET:-"Malcolm"} ARKIME_FREESPACEG=${ARKIME_FREESPACEG:-"10%"} +ARKIME_ROTATE_INDEX=${ARKIME_ROTATE_INDEX:-"daily"} CAPTURE_INTERFACE=${PCAP_IFACE:-} LIVE_CAPTURE=${ARKIME_LIVE_CAPTURE:-false} VIEWER_PORT=${ARKIME_VIEWER_PORT:-8005} @@ -58,6 +59,7 @@ if [[ ! -f "${ARKIME_CONFIG_FILE}" ]] && [[ -r "${ARKIME_DIR}"/etc/config.orig.i sed -i "s|^\(elasticsearch=\).*|\1"${OPENSEARCH_URL_FINAL}"|" "${ARKIME_CONFIG_FILE}" sed -i "s/^\(passwordSecret=\).*/\1"${ARKIME_PASSWORD_SECRET}"/" "${ARKIME_CONFIG_FILE}" sed -i "s/^\(freeSpaceG=\).*/\1"${ARKIME_FREESPACEG}"/" "${ARKIME_CONFIG_FILE}" + sed -i "s/^\(rotateIndex=\).*/\1"${ARKIME_ROTATE_INDEX}"/" "${ARKIME_CONFIG_FILE}" sed -i "s/^\(viewPort=\).*/\1"${VIEWER_PORT}"/" "${ARKIME_CONFIG_FILE}" sed -i "s/^\(pcapDir=\).*/\1\/data\/pcap\/arkime-live/" "${ARKIME_CONFIG_FILE}" # note: when setting the node name, the viewer_service.sh script needs to match diff --git a/config/arkime.env.example b/config/arkime.env.example index 3df6ecfc1..c3afc8271 100644 --- a/config/arkime.env.example +++ b/config/arkime.env.example @@ -1,7 +1,10 @@ ARKIME_VIEWER_PORT=8005 -# Whether or not Arkime is allowed to delete uploaded/captured PCAP (see -# https://arkime.com/faq#pcap-deletion) +# Whether or not Arkime is allowed to delete uploaded/captured PCAP +# https://arkime.com/faq#pcap-deletion MANAGE_PCAP_FILES=false ARKIME_FREESPACEG=10% +# How often to create a new index in OpenSearch/Elasticsearch +# https://arkime.com/settings#rotateIndex +ARKIME_ROTATE_INDEX=daily OPENSEARCH_MAX_SHARDS_PER_NODE=2500 \ No newline at end of file diff --git a/config/opensearch.env.example b/config/opensearch.env.example index fbdf87a17..e9322a433 100644 --- a/config/opensearch.env.example +++ b/config/opensearch.env.example @@ -44,8 +44,10 @@ OPENSEARCH_JAVA_OPTS=-server -Xms10g -Xmx10g -Xss256k -XX:-HeapDumpOnOutOfMemory # OpenSearch index patterns and timestamp fields MALCOLM_NETWORK_INDEX_PATTERN=arkime_sessions3-* MALCOLM_NETWORK_INDEX_TIME_FIELD=firstPacket +MALCOLM_NETWORK_INDEX_SUFFIX=-%{+YYMMdd} MALCOLM_OTHER_INDEX_PATTERN=malcolm_beats_* MALCOLM_OTHER_INDEX_TIME_FIELD=@timestamp +MALCOLM_OTHER_INDEX_SUFFIX=-%{+YYMMdd} ARKIME_NETWORK_INDEX_PATTERN=arkime_sessions3-* ARKIME_NETWORK_INDEX_TIME_FIELD=firstPacket diff --git a/logstash/pipelines/beats/98_finalize.conf b/logstash/pipelines/beats/98_finalize.conf index 4cdafa7a6..fe0d56542 100644 --- a/logstash/pipelines/beats/98_finalize.conf +++ b/logstash/pipelines/beats/98_finalize.conf @@ -107,19 +107,35 @@ filter { } } - # this field is used to determine opensearch index prefix - if ([event][module]) { - mutate { id => "mutate_add_malcolm_beats_index_prefix_event_module" - add_field => { "[@metadata][malcolm_opensearch_index_prefix]" => "malcolm_beats_%{[event][module]}" } } - } else if ([agent][type]) { - mutate { id => "mutate_add_malcolm_beats_index_prefix_agent_type" - add_field => { "[@metadata][malcolm_opensearch_index_prefix]" => "malcolm_beats_%{[agent][type]}" } } - } else if ([input][type]) { - mutate { id => "mutate_add_malcolm_beats_index_prefix_input_type" - add_field => { "[@metadata][malcolm_opensearch_index_prefix]" => "malcolm_beats_%{[input][type]}" } } - } else { - mutate { id => "mutate_add_malcolm_beats_index_prefix_beats" - add_field => { "[@metadata][malcolm_opensearch_index_prefix]" => "malcolm_beats_" } } + # these fields are used to determine opensearch index format + if (![@metadata][malcolm_opensearch_index_prefix]) { + ruby { + id => "rube_add_beats_logs_index_prefix" + init => "$indexPrefix = ENV['MALCOLM_OTHER_INDEX_PATTERN'].sub(/[\*_-]*$/, '')" + code => " + ipref = ($indexPrefix.nil? || $indexPrefix.empty?) ? 'malcolm_beats' : $indexPrefix + icat = event.get('[event][module]') + if icat.nil? then + icat = event.get('[agent][type]') + if icat.nil? then + icat = event.get('[input][type]') + if icat.nil? then + icat = 'malcolm_beats_' + end + end + end + event.set('[@metadata][malcolm_opensearch_index_prefix]', ipref + '_' + icat) + " + } + } + if (![@metadata][malcolm_opensearch_index_suffix]) { + ruby { + id => "rube_add_beats_logs_index_suffix" + init => "$indexSuffix = ENV['MALCOLM_OTHER_INDEX_PATTERN']" + code => " + event.set('[@metadata][malcolm_opensearch_index_suffix]', ($indexSuffix.nil? || $indexSuffix.empty?) ? '-%{+YYMMdd}' : $indexSuffix) + " + } } diff --git a/logstash/pipelines/enrichment/98_finalize.conf b/logstash/pipelines/enrichment/98_finalize.conf index 51ec8f212..8bfaff8a5 100644 --- a/logstash/pipelines/enrichment/98_finalize.conf +++ b/logstash/pipelines/enrichment/98_finalize.conf @@ -4,7 +4,7 @@ filter { - # this field is used to determine opensearch index prefix + # these fields are used to determine opensearch index format if (![@metadata][malcolm_opensearch_index_prefix]) { ruby { id => "rube_add_network_logs_index_prefix" @@ -14,6 +14,15 @@ filter { " } } + if (![@metadata][malcolm_opensearch_index_suffix]) { + ruby { + id => "rube_add_network_logs_index_suffix" + init => "$indexSuffix = ENV['MALCOLM_NETWORK_INDEX_SUFFIX']" + code => " + event.set('[@metadata][malcolm_opensearch_index_suffix]', ($indexSuffix.nil? || $indexSuffix.empty?) ? '-%{+YYMMdd}' : $indexSuffix) + " + } + } # event.hash is used for document ID, it really needed to be set by now if (![event][hash]) { diff --git a/logstash/pipelines/external/99_opensearch_output.conf b/logstash/pipelines/external/99_opensearch_output.conf index 2c750669e..a0e8d251d 100644 --- a/logstash/pipelines/external/99_opensearch_output.conf +++ b/logstash/pipelines/external/99_opensearch_output.conf @@ -6,7 +6,7 @@ output { user => "_MALCOLM_LOGSTASH_OPENSEARCH_SECONDARY_USER_" password => "_MALCOLM_LOGSTASH_OPENSEARCH_SECONDARY_PASSWORD_" manage_template => false - index => "%{[@metadata][malcolm_opensearch_index_prefix]}-%{+YYMMdd}" + index => "%{[@metadata][malcolm_opensearch_index_prefix]}%{[@metadata][malcolm_opensearch_index_suffix}" document_id => "%{+YYMMdd}-%{[event][hash]}" } } diff --git a/logstash/pipelines/output/99_opensearch_output.conf b/logstash/pipelines/output/99_opensearch_output.conf index 34125fefd..7de474d14 100644 --- a/logstash/pipelines/output/99_opensearch_output.conf +++ b/logstash/pipelines/output/99_opensearch_output.conf @@ -6,7 +6,7 @@ output { user => "_MALCOLM_LOGSTASH_OPENSEARCH_USER_" password => "_MALCOLM_LOGSTASH_OPENSEARCH_PASSWORD_" manage_template => false - index => "%{[@metadata][malcolm_opensearch_index_prefix]}-%{+YYMMdd}" + index => "%{[@metadata][malcolm_opensearch_index_prefix]}%{[@metadata][malcolm_opensearch_index_suffix}" document_id => "%{+YYMMdd}-%{[event][hash]}" } } diff --git a/sensor-iso/interface/sensor_ctl/control_vars.conf b/sensor-iso/interface/sensor_ctl/control_vars.conf index d821c64d8..7d72f4d18 100644 --- a/sensor-iso/interface/sensor_ctl/control_vars.conf +++ b/sensor-iso/interface/sensor_ctl/control_vars.conf @@ -23,6 +23,7 @@ export ARKIME_VIEWER_KEY=viewer.key # Password hash secret for Arkime viewer cluster (see https://arkime.com/settings) export ARKIME_PASSWORD_SECRET=Malcolm export ARKIME_FREESPACEG=7% +export ARKIME_ROTATE_INDEX=daily export DOCUMENTATION_PORT=8420 export MISCBEAT_PORT=9516 diff --git a/sensor-iso/interface/sensor_ctl/supervisor.init/arkime_config_populate.sh b/sensor-iso/interface/sensor_ctl/supervisor.init/arkime_config_populate.sh index fd7c32ce2..0a627c95f 100644 --- a/sensor-iso/interface/sensor_ctl/supervisor.init/arkime_config_populate.sh +++ b/sensor-iso/interface/sensor_ctl/supervisor.init/arkime_config_populate.sh @@ -78,6 +78,11 @@ if [[ -n $SUPERVISOR_PATH ]] && [[ -r "$SUPERVISOR_PATH"/arkime/config.ini ]]; t sed -r -i "s/(simpleGzipLevel)\s*=\s*.*/\1=$COMPRESSION_LEVEL/" "$ARKIME_CONFIG_FILE" fi + # how often OpenSearch/Elasticsearch should create a new index + if [[ -n $ARKIME_ROTATE_INDEX ]]; then + sed -r -i "s/(rotateIndex)\s*=\s*.*/\1=$ARKIME_ROTATE_INDEX/" "$ARKIME_CONFIG_FILE" + fi + # identify node in session metadata for PCAP reachback PRIMARY_IP=$(ip route get 255.255.255.255 | grep -Po '(?<=src )(\d{1,3}.){4}' | sed "s/ //g") export ARKIME_NODE_NAME="$(hostname --long)"