diff --git a/config/opensearch.env.example b/config/opensearch.env.example index fd5aca877..95408c7f4 100644 --- a/config/opensearch.env.example +++ b/config/opensearch.env.example @@ -47,13 +47,13 @@ MALCOLM_NETWORK_INDEX_PATTERN=arkime_sessions3-* # Default time field to use for network traffic logs in Logstash and Dashboards MALCOLM_NETWORK_INDEX_TIME_FIELD=firstPacket # Suffix used to create index to which network traffic logs are written (supports Ruby strftime strings in %{}) -MALCOLM_NETWORK_INDEX_SUFFIX=-%{%y%m%d} +MALCOLM_NETWORK_INDEX_SUFFIX=%{%y%m%d} # Index pattern for other logs written via Logstash (e.g., nginx, beats, fluent-bit, etc.) MALCOLM_OTHER_INDEX_PATTERN=malcolm_beats_* # Default time field to use for other logs in Logstash and Dashboards MALCOLM_OTHER_INDEX_TIME_FIELD=@timestamp # Suffix used to create index to which other logs are written (supports Ruby strftime strings in %{}) -MALCOLM_OTHER_INDEX_SUFFIX=-%{%y%m%d} +MALCOLM_OTHER_INDEX_SUFFIX=%{%y%m%d} # Index pattern used specifically by Arkime (will probably match MALCOLM_NETWORK_INDEX_PATTERN, should probably be arkime_sessions3-*) ARKIME_NETWORK_INDEX_PATTERN=arkime_sessions3-* # Default time field used by for sessions in Arkime viewer diff --git a/logstash/pipelines/beats/98_finalize.conf b/logstash/pipelines/beats/98_finalize.conf index 2dc7c5539..b3b3dff30 100644 --- a/logstash/pipelines/beats/98_finalize.conf +++ b/logstash/pipelines/beats/98_finalize.conf @@ -115,9 +115,9 @@ filter { script_params => { "target" => "[@metadata][malcolm_opensearch_index]" "prefix_env" => "MALCOLM_OTHER_INDEX_PATTERN" - "prefix_default" => "malcolm_beats" + "prefix_default" => "malcolm_beats_*" "suffix_env" => "MALCOLM_OTHER_INDEX_SUFFIX" - "suffix_default" => "-%{%y%m%d}" + "suffix_default" => "%{%y%m%d}" "midfix_fields" => [ "[event][module]", "[agent][type]", "[input][type]" ] } } diff --git a/logstash/pipelines/enrichment/98_finalize.conf b/logstash/pipelines/enrichment/98_finalize.conf index 5e5176cb2..0d979c734 100644 --- a/logstash/pipelines/enrichment/98_finalize.conf +++ b/logstash/pipelines/enrichment/98_finalize.conf @@ -12,9 +12,9 @@ filter { script_params => { "target" => "[@metadata][malcolm_opensearch_index]" "prefix_env" => "MALCOLM_NETWORK_INDEX_PATTERN" - "prefix_default" => "arkime_sessions3" + "prefix_default" => "arkime_sessions3-*" "suffix_env" => "MALCOLM_NETWORK_INDEX_SUFFIX" - "suffix_default" => "-%{%y%m%d}" + "suffix_default" => "%{%y%m%d}" } } } diff --git a/logstash/ruby/format_index_string.rb b/logstash/ruby/format_index_string.rb index 0882e03ca..258f0a42a 100644 --- a/logstash/ruby/format_index_string.rb +++ b/logstash/ruby/format_index_string.rb @@ -49,6 +49,14 @@ def filter(event) tstamp = Time.now.utc end + prefix_resolved = @prefix.delete_suffix('*') + if prefix_resolved[-1].count("^a-z0-9").zero? then + suffix_separator = '' + else + suffix_separator = prefix_resolved[-1] + prefix_resolved = prefix_resolved[0..-2] + end + suffix_resolved = @suffix if parts = @suffix.scan(/(%{([^}]+)})/) then if parts.kind_of?(Array) then @@ -69,7 +77,7 @@ def filter(event) end end - event.set("#{@target}", @prefix.sub(/[\*_-]*$/, '') + String(midfix_first) + suffix_resolved) + event.set("#{@target}", prefix_resolved + String(midfix_first) + suffix_separator + suffix_resolved) [event] end