Skip to content

Commit

Permalink
[Filebeat] Replace copy_from with templated value (#26631) (#26964)
Browse files Browse the repository at this point in the history
* Replace copy_from with templated value

To ensure compatibility with Elasticsearch versions <7.13 this removes usage of `copy_from` in `set` processors.

Relates #26629

* panw: replace copy_from usage with script

Replaces the usage of a set processor with copy_from (ES 7.13+)
with a painless script that performs the same operation and it's
backwards compatible.

* cyberarkpas: Replace usage of copy_from with script

This updates the ID-mapping script to set fields instead of constructing
and op-list that is latter processed with foreach/set.

* Update CHANGELOG.next.asciidoc

Co-authored-by: Adrian Serrano <adrisr83@gmail.com>

Co-authored-by: Adrian Serrano <adrisr83@gmail.com>
(cherry picked from commit a7b0110)

Co-authored-by: Andrew Kroh <andrew.kroh@elastic.co>
  • Loading branch information
mergify[bot] and andrewkroh authored Jul 20, 2021
1 parent 957648d commit 3ec07ce
Show file tree
Hide file tree
Showing 12 changed files with 74 additions and 34 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Improve inode reuse handling by removing state for removed files more eagerly from the internal state table in the logs inputs. {pull}25756[25756]
- Fix default config template values for paths on oracle module: {pull}26276[26276]
- Change type of max_bytes in all configs to be cfgtype.ByteSize {pull}26699[26699]
- Fix Elasticsearch compatibility for modules that use `copy_from` in `set` processors. {issue}26629[26629]

*Filebeat*

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@ processors:
path: elasticsearch.audit
- set:
field: http.request.id
copy_from: elasticsearch.audit.request.id
value: '{{{elasticsearch.audit.request.id}}}'
ignore_empty_value: true
- dot_expander:
field: cluster.name
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ processors:
value: ""
- set:
field: http.request.id
copy_from: nginx.ingress_controller.http.request.id
value: '{{{nginx.ingress_controller.http.request.id}}}'
ignore_empty_value: true
ignore_failure: true
- script:
Expand Down
59 changes: 39 additions & 20 deletions x-pack/filebeat/module/cyberarkpas/audit/ingest/pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -914,40 +914,59 @@ processors:
value: "success"
- set: event.reason
from: cyberarkpas.audit.reason
on_failure:
- append:
field: error.message
value: 'Failed to enrich based on ID #{{{ event.code }}}: {{{_ingest.on_failure_message}}}'
source: >
def clone(def val) {
return val instanceof List? new ArrayList(val) : val;
def clone(def ref) {
if (ref == null) return ref;
if (ref instanceof Map) {
ref = ref.entrySet().stream().collect(
Collectors.toMap(
e -> e.getKey(),
e -> clone(e.getValue())
)
);
} else if (ref instanceof List) {
ref = ref.stream().map(e -> clone(e)).collect(
Collectors.toList()
);
}
return ref;
}
def read_field(def map, String name) {
if (map == null || !(map instanceof Map)) return null;
int pos = name.indexOf(".");
return pos == -1? map[name]
: read_field(map[name.substring(0, pos)], name.substring(pos+1));
}
boolean set_field(Map map, String name, def value) {
int pos = name.indexOf(".");
if (pos == -1) {
map[name] = clone(value);
return true;
}
String key = name.substring(0, pos),
path = name.substring(pos+1);
if (!map.containsKey(key)) {
map[key] = new HashMap();
}
map = map[key];
return map instanceof Map? set_field(map, path, value)
: false;
}
String msgID = ctx.event?.code;
def actions = params.get(msgID);
if (actions == null) return;
List values = new ArrayList();
for (def item : actions) {
def val = item.value;
if (val == null && (val = read_field(ctx, item.from)) == null || val == "") continue;
values.add([
"to": item.set,
"value": clone(val)
]);
if (!set_field(ctx, item.set, val)) {
throw new Exception("Failed to set field " + item.set);
}
}
if (!values.isEmpty()) ctx._tmp["values"] = values;
- foreach:
field: _tmp.values
ignore_missing: true
processor:
set:
field: '{{{_ingest._value.to}}}'
copy_from: '_ingest._value.value'
ignore_empty_value: true
override: true
#
# Force event.outcome: unknown in case it gets a value other than one of the allowed.
Expand Down Expand Up @@ -994,7 +1013,7 @@ processors:
on_failure:
- set:
field: source.domain
copy_from: source.address
value: '{{{source.address}}}'
- convert:
field: destination.address
target_field: destination.ip
Expand All @@ -1003,7 +1022,7 @@ processors:
on_failure:
- set:
field: destination.domain
copy_from: destination.address
value: '{{{destination.address}}}'
#
# Populate related.ip
Expand Down
28 changes: 24 additions & 4 deletions x-pack/filebeat/module/panw/panos/ingest/pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -462,10 +462,30 @@ processors:
ANY: '.*'
if: 'ctx?.file?.name != null && ctx?.file?.name != ""'

- set:
field: user
copy_from: source.user
if: "ctx?.source?.user != null"
- script:
lang: painless
description: Copy source.user to user
source: >
def clone(def ref) {
if (ref == null) return ref;
if (ref instanceof Map) {
ref = ref.entrySet().stream().collect(
Collectors.toMap(
e -> e.getKey(),
e -> clone(e.getValue())
)
);
} else if (ref instanceof List) {
ref = ref.stream().map(e -> clone(e)).collect(
Collectors.toList()
);
}
return ref;
}
def u = ctx?.source?.user;
if (u != null) {
ctx["user"] = clone(u);
}
- append:
field: related.user
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ processors:
remove_if_successful: true
- set:
field: threatintel.indicator.url.full
copy_from: threatintel.indicator.url.original
value: '{{{threatintel.indicator.url.original}}}'
ignore_empty_value: true
- rename:
field: threatintel.abuseurl.host
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ processors:
if: ctx?.threatintel?.indicator?.type == 'url'
- set:
field: threatintel.indicator.url.full
copy_from: threatintel.indicator.url.original
value: '{{{threatintel.indicator.url.original}}}'
ignore_empty_value: true
- rename:
field: _tmp.threatvalue
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -265,7 +265,7 @@ processors:

- set:
field: threatintel.indicator.url.full
copy_from: threatintel.indicator.url.original
value: '{{{threatintel.indicator.url.original}}}'
ignore_empty_value: true

- rename:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ processors:

- set:
field: threatintel.indicator.url.full
copy_from: threatintel.indicator.url.original
value: '{{{threatintel.indicator.url.original}}}'
ignore_empty_value: true
if: "ctx?.threatintel?.indicator?.type == 'url' && ctx?.threatintel?.misp?.attribute?.type != 'uri'"

Expand Down
2 changes: 1 addition & 1 deletion x-pack/filebeat/module/threatintel/otx/ingest/pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ processors:
if: ctx?.threatintel?.indicator?.type == 'url'
- set:
field: threatintel.indicator.url.full
copy_from: threatintel.indicator.url.original
value: '{{{threatintel.indicator.url.original}}}'
ignore_empty_value: true
if: "ctx?.threatintel?.otx?.type == 'URL'"

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ processors:

- set:
field: json.ip_range
copy_from: json.entity.name
value: '{{{json.entity.name}}}'
if: 'ctx.json.entity?.type == "IpAddress" && ctx.json.entity.name != null && ctx.json.entity.name.contains("/")'
- set:
field: json.ip_range
Expand All @@ -134,7 +134,7 @@ processors:
if: 'ctx.threatintel?.indicator?.type == "ipv6-addr" && ctx.json.entity.name != null && !ctx.json.entity.name.contains("/")'
- set:
field: json.ip_range
copy_from: json.entity.name
value: '{{{json.entity.name}}}'
if: 'ctx.json.entity?.type == "IpAddress" && ctx.json.entity.name != null && ctx.json.entity.name.contains("/")'

- rename:
Expand Down
2 changes: 1 addition & 1 deletion x-pack/filebeat/module/zoom/webhook/ingest/meeting.yml
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ processors:
if: ctx?.url?.original != null
- set:
field: url.full
copy_from: url.original
value: '{{{url.original}}}'
ignore_failure: true
if: ctx?.url?.original != null
#
Expand Down

0 comments on commit 3ec07ce

Please sign in to comment.