customizing index settings work in progress, idaholab#313 and idahola…

…b#283
mmguero-dev · Jan 11, 2024 · ea7cc3d · ea7cc3d
1 parent 91569c3
commit ea7cc3d
Show file tree

Hide file tree

Showing 4 changed files with 15 additions and 7 deletions.
diff --git a/config/opensearch.env.example b/config/opensearch.env.example
@@ -47,13 +47,13 @@ MALCOLM_NETWORK_INDEX_PATTERN=arkime_sessions3-*
 # Default time field to use for network traffic logs in Logstash and Dashboards
 MALCOLM_NETWORK_INDEX_TIME_FIELD=firstPacket
 # Suffix used to create index to which network traffic logs are written (supports Ruby strftime strings in %{})
-MALCOLM_NETWORK_INDEX_SUFFIX=-%{%y%m%d}
+MALCOLM_NETWORK_INDEX_SUFFIX=%{%y%m%d}
 # Index pattern for other logs written via Logstash (e.g., nginx, beats, fluent-bit, etc.)
 MALCOLM_OTHER_INDEX_PATTERN=malcolm_beats_*
 # Default time field to use for other logs in Logstash and Dashboards
 MALCOLM_OTHER_INDEX_TIME_FIELD=@timestamp
 # Suffix used to create index to which other logs are written (supports Ruby strftime strings in %{})
-MALCOLM_OTHER_INDEX_SUFFIX=-%{%y%m%d}
+MALCOLM_OTHER_INDEX_SUFFIX=%{%y%m%d}
 # Index pattern used specifically by Arkime (will probably match MALCOLM_NETWORK_INDEX_PATTERN, should probably be arkime_sessions3-*)
 ARKIME_NETWORK_INDEX_PATTERN=arkime_sessions3-*
 # Default time field used by for sessions in Arkime viewer

diff --git a/logstash/pipelines/beats/98_finalize.conf b/logstash/pipelines/beats/98_finalize.conf
@@ -115,9 +115,9 @@ filter {
         script_params => {
           "target" => "[@metadata][malcolm_opensearch_index]"
           "prefix_env" => "MALCOLM_OTHER_INDEX_PATTERN"
-          "prefix_default" => "malcolm_beats"
+          "prefix_default" => "malcolm_beats_*"
           "suffix_env" => "MALCOLM_OTHER_INDEX_SUFFIX"
-          "suffix_default" => "-%{%y%m%d}"
+          "suffix_default" => "%{%y%m%d}"
           "midfix_fields" => [ "[event][module]", "[agent][type]", "[input][type]" ]
         }
       }

diff --git a/logstash/pipelines/enrichment/98_finalize.conf b/logstash/pipelines/enrichment/98_finalize.conf
@@ -12,9 +12,9 @@ filter {
         script_params => {
           "target" => "[@metadata][malcolm_opensearch_index]"
           "prefix_env" => "MALCOLM_NETWORK_INDEX_PATTERN"
-          "prefix_default" => "arkime_sessions3"
+          "prefix_default" => "arkime_sessions3-*"
           "suffix_env" => "MALCOLM_NETWORK_INDEX_SUFFIX"
-          "suffix_default" => "-%{%y%m%d}"
+          "suffix_default" => "%{%y%m%d}"
         }
       }
     }

diff --git a/logstash/ruby/format_index_string.rb b/logstash/ruby/format_index_string.rb
@@ -49,6 +49,14 @@ def filter(event)
     tstamp = Time.now.utc
   end
 
+  prefix_resolved = @prefix.delete_suffix('*')
+  if prefix_resolved[-1].count("^a-z0-9").zero? then
+    suffix_separator = ''
+  else
+    suffix_separator = prefix_resolved[-1]
+    prefix_resolved = prefix_resolved[0..-2]
+  end
+
   suffix_resolved = @suffix
   if parts = @suffix.scan(/(%{([^}]+)})/) then
       if parts.kind_of?(Array) then
@@ -69,7 +77,7 @@ def filter(event)
     end
   end
 
-  event.set("#{@target}", @prefix.sub(/[\*_-]*$/, '') + String(midfix_first) + suffix_resolved)
+  event.set("#{@target}", prefix_resolved + String(midfix_first) + suffix_separator + suffix_resolved)
 
   [event]
 end