From 21f0a8d9781af85322f33a2e462baa831d3b6049 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Wed, 20 Sep 2023 13:13:18 -0500 Subject: [PATCH 1/5] migrate top_level:false handling to helper func --- scripts/generators/ecs_helpers.py | 11 +++++++++++ scripts/generators/es_template.py | 15 ++------------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/scripts/generators/ecs_helpers.py b/scripts/generators/ecs_helpers.py index a39a5bc463..84c6759e3e 100644 --- a/scripts/generators/ecs_helpers.py +++ b/scripts/generators/ecs_helpers.py @@ -254,6 +254,17 @@ def is_intermediate(field: FieldEntry) -> bool: return ('intermediate' in field['field_details'] and field['field_details']['intermediate']) +def remove_top_level_reusable_false(ecs_nested: Dict[str, FieldNestedEntry]) -> Dict[str, FieldNestedEntry]: + """Returns same structure as ecs_nested, but skips all field sets with reusable.top_level: False""" + components: Dict[str, FieldNestedEntry] = {} + for (fieldset_name, fieldset) in ecs_nested.items(): + if fieldset.get('reusable', None): + if not fieldset['reusable']['top_level']: + continue + components[fieldset_name] = fieldset + return components + + # Warning helper diff --git a/scripts/generators/es_template.py b/scripts/generators/es_template.py index 4c78b9ff6f..fa9fdda9c0 100644 --- a/scripts/generators/es_template.py +++ b/scripts/generators/es_template.py @@ -65,7 +65,7 @@ def all_component_templates( component_dir: str = join(out_dir, 'elasticsearch/composable/component') ecs_helpers.make_dirs(component_dir) - for (fieldset_name, fieldset) in candidate_components(ecs_nested).items(): + for (fieldset_name, fieldset) in ecs_helpers.remove_top_level_reusable_false(ecs_nested).items(): field_mappings = {} for (flat_name, field) in fieldset['fields'].items(): name_parts = flat_name.split('.') @@ -104,22 +104,11 @@ def component_name_convention( ) -> List[str]: version: str = ecs_version.replace('+', '-') names: List[str] = [] - for (fieldset_name, fieldset) in candidate_components(ecs_nested).items(): + for (fieldset_name, fieldset) in ecs_helpers.remove_top_level_reusable_false(ecs_nested).items(): names.append("ecs_{}_{}".format(version, fieldset_name.lower())) return names -def candidate_components(ecs_nested: Dict[str, FieldNestedEntry]) -> Dict[str, FieldNestedEntry]: - """Returns same structure as ecs_nested, but skips all field sets with reusable.top_level: False""" - components: Dict[str, FieldNestedEntry] = {} - for (fieldset_name, fieldset) in ecs_nested.items(): - if fieldset.get('reusable', None): - if not fieldset['reusable']['top_level']: - continue - components[fieldset_name] = fieldset - return components - - # Legacy template From 9debbfa7558f178442649c28e4e33498578155fa Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Wed, 20 Sep 2023 13:28:36 -0500 Subject: [PATCH 2/5] handle top_level:false in beats generator --- scripts/generators/beats.py | 1 + scripts/tests/test_ecs_helpers.py | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/scripts/generators/beats.py b/scripts/generators/beats.py index b6422a86fd..fc9d46f972 100644 --- a/scripts/generators/beats.py +++ b/scripts/generators/beats.py @@ -36,6 +36,7 @@ def generate( out_dir: str ) -> None: # base first + ecs_nested = ecs_helpers.remove_top_level_reusable_false(ecs_nested) beats_fields: List[OrderedDict] = fieldset_field_array(ecs_nested['base']['fields'], ecs_nested['base']['prefix']) allowed_fieldset_keys: List[str] = ['name', 'title', 'group', 'description', 'footnote', 'type'] diff --git a/scripts/tests/test_ecs_helpers.py b/scripts/tests/test_ecs_helpers.py index d2413cd64b..4949a22382 100644 --- a/scripts/tests/test_ecs_helpers.py +++ b/scripts/tests/test_ecs_helpers.py @@ -157,6 +157,17 @@ def test_glob_yaml_files(self): ecs_helpers.glob_yaml_files('schemas/*.yml'))) >= min_schema_count) self.assertEqual(len(ecs_helpers.glob_yaml_files(ecs_helpers.glob_yaml_files('schemas/*.yaml'))), 0) + # Remove top_level:false field sets helper + + def test_remove_top_level_false_field_sets(self): + nested_schema_original = { + 'as': {'group': 2, 'name': 'as', 'reusable': {'top_level': False}}, + 'agent': {'group': 2, 'name': 'agent'}, + } + nested_schema_expected = { + 'agent': {'group': 2, 'name': 'agent'} + } + self.assertEqual(ecs_helpers.remove_top_level_reusable_false(nested_schema_original), nested_schema_expected) if __name__ == '__main__': unittest.main() From 9b6cb3cced220bd270ca725843e92c91a972d999 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Wed, 20 Sep 2023 13:29:08 -0500 Subject: [PATCH 3/5] regenerate Beats fields minus top_level:false field sets --- experimental/generated/beats/fields.ecs.yml | 1452 ++----------------- generated/beats/fields.ecs.yml | 1452 ++----------------- 2 files changed, 270 insertions(+), 2634 deletions(-) diff --git a/experimental/generated/beats/fields.ecs.yml b/experimental/generated/beats/fields.ecs.yml index 0eb0ccec7f..41ce694b58 100644 --- a/experimental/generated/beats/fields.ecs.yml +++ b/experimental/generated/beats/fields.ecs.yml @@ -122,32 +122,6 @@ ignore_above: 1024 description: Version of the agent. example: 6.0.0-rc2 - - name: as - title: Autonomous System - group: 2 - description: An autonomous system (AS) is a collection of connected Internet Protocol - (IP) routing prefixes under the control of one or more network operators on - behalf of a single administrative entity or domain that presents a common, clearly - defined routing policy to the internet. - type: group - default_field: true - fields: - - name: number - level: extended - type: long - description: Unique number allocated to the autonomous system. The autonomous - system number (ASN) uniquely identifies each network on the Internet. - example: 15169 - - name: organization.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: match_only_text - default_field: false - description: Organization name. - example: Google LLC - name: cgroup title: Common cgroup metrics group: 2 @@ -777,91 +751,6 @@ Examples: app engine, app service, cloud run, fargate, lambda.' example: lambda default_field: false - - name: code_signature - title: Code Signature - group: 2 - description: These fields contain information about binary code signatures. - type: group - default_field: true - fields: - - name: digest_algorithm - level: extended - type: keyword - ignore_above: 1024 - description: 'The hashing algorithm used to sign the process. - - This value can distinguish signatures when a file is signed multiple times - by the same signer but with a different digest algorithm.' - example: sha256 - default_field: false - - name: exists - level: core - type: boolean - description: Boolean to capture if a signature is present. - example: 'true' - default_field: false - - name: signing_id - level: extended - type: keyword - ignore_above: 1024 - description: 'The identifier used to sign the process. - - This is used to identify the application manufactured by a software vendor. - The field is relevant to Apple *OS only.' - example: com.apple.xpc.proxy - default_field: false - - name: status - level: extended - type: keyword - ignore_above: 1024 - description: 'Additional information about the certificate status. - - This is useful for logging cryptographic errors with the certificate validity - or trust status. Leave unpopulated if the validity or trust of the certificate - was unchecked.' - example: ERROR_UNTRUSTED_ROOT - default_field: false - - name: subject_name - level: core - type: keyword - ignore_above: 1024 - description: Subject name of the code signer - example: Microsoft Corporation - default_field: false - - name: team_id - level: extended - type: keyword - ignore_above: 1024 - description: 'The team identifier used to sign the process. - - This is used to identify the team or vendor of a software product. The field - is relevant to Apple *OS only.' - example: EQHXZ8M8AV - default_field: false - - name: timestamp - level: extended - type: date - description: Date and time when the code signature was generated and signed. - example: '2021-01-01T12:10:30Z' - default_field: false - - name: trusted - level: extended - type: boolean - description: 'Stores the trust status of the certificate chain. - - Validating the trust of the certificate chain may be complicated, and this - field should only be populated by tools that actively check the status.' - example: 'true' - default_field: false - - name: valid - level: extended - type: boolean - description: 'Boolean to capture if the digital signature is verified against - the binary content. - - Leave unpopulated if a certificate was unchecked.' - example: 'true' - default_field: false - name: container title: Container group: 2 @@ -1847,425 +1736,172 @@ ECS versions -- this field lets integrations adjust to the schema version of the events.' example: 1.0.0 - - name: elf - title: ELF Header + - name: email + title: Email group: 2 - description: These fields contain Linux Executable Linkable Format (ELF) metadata. + description: 'Event details relating to an email transaction. + + This field set focuses on the email message header, body, and attachments. Network + protocols that send and receive email messages such as SMTP are outside the + scope of the `email.*` fields.' type: group default_field: true fields: - - name: architecture + - name: attachments level: extended - type: keyword - ignore_above: 1024 - description: Machine architecture of the ELF file. - example: x86-64 + type: nested + description: A list of objects describing the attachment files sent along with + an email message. default_field: false - - name: byte_order + - name: attachments.file.extension level: extended type: keyword ignore_above: 1024 - description: Byte sequence of ELF file. - example: Little Endian + description: Attachment file extension, excluding the leading dot. + example: txt default_field: false - - name: cpu_type + - name: attachments.file.hash.md5 level: extended type: keyword ignore_above: 1024 - description: CPU type of the ELF file. - example: Intel - default_field: false - - name: creation_date - level: extended - type: date - description: Extracted when possible from the file's metadata. Indicates when - it was built or compiled. It can also be faked by malware creators. - default_field: false - - name: exports - level: extended - type: flattened - description: List of exported element names and types. + description: MD5 hash. default_field: false - - name: go_import_hash + - name: attachments.file.hash.sha1 level: extended type: keyword ignore_above: 1024 - description: 'A hash of the Go language imports in an ELF file excluding standard - library imports. An import hash can be used to fingerprint binaries even after - recompilation or other code-level transformations have occurred, which would - change more traditional hash values. - - The algorithm used to calculate the Go symbol hash and a reference implementation - are available [here](https://github.com/elastic/toutoumomoma).' - example: 10bddcb4cee42080f76c88d9ff964491 + description: SHA1 hash. default_field: false - - name: go_imports + - name: attachments.file.hash.sha256 level: extended - type: flattened - description: List of imported Go language element names and types. + type: keyword + ignore_above: 1024 + description: SHA256 hash. default_field: false - - name: go_imports_names_entropy + - name: attachments.file.hash.sha384 level: extended - type: long - format: number - description: Shannon entropy calculation from the list of Go imports. + type: keyword + ignore_above: 1024 + description: SHA384 hash. default_field: false - - name: go_imports_names_var_entropy + - name: attachments.file.hash.sha512 level: extended - type: long - format: number - description: Variance for Shannon entropy calculation from the list of Go imports. + type: keyword + ignore_above: 1024 + description: SHA512 hash. default_field: false - - name: go_stripped + - name: attachments.file.hash.ssdeep level: extended - type: boolean - description: Set to true if the file is a Go executable that has had its symbols - stripped or obfuscated and false if an unobfuscated Go executable. + type: keyword + ignore_above: 1024 + description: SSDEEP hash. default_field: false - - name: header.abi_version + - name: attachments.file.hash.tlsh level: extended type: keyword ignore_above: 1024 - description: Version of the ELF Application Binary Interface (ABI). + description: TLSH hash. default_field: false - - name: header.class + - name: attachments.file.mime_type level: extended type: keyword ignore_above: 1024 - description: Header class of the ELF file. + description: 'The MIME media type of the attachment. + + This value will typically be extracted from the `Content-Type` MIME header + field.' + example: text/plain default_field: false - - name: header.data + - name: attachments.file.name level: extended type: keyword ignore_above: 1024 - description: Data table of the ELF header. + description: Name of the attachment file including the file extension. + example: attachment.txt default_field: false - - name: header.entrypoint + - name: attachments.file.size level: extended type: long - format: string - description: Header entrypoint of the ELF file. + description: Attachment file size in bytes. + example: 64329 default_field: false - - name: header.object_version + - name: bcc.address level: extended type: keyword ignore_above: 1024 - description: '"0x1" for original ELF files.' + description: The email address of BCC recipient + example: bcc.user1@example.com default_field: false - - name: header.os_abi + - name: cc.address level: extended type: keyword ignore_above: 1024 - description: Application Binary Interface (ABI) of the Linux OS. + description: The email address of CC recipient + example: cc.user1@example.com default_field: false - - name: header.type + - name: content_type level: extended type: keyword ignore_above: 1024 - description: Header type of the ELF file. + description: 'Information about how the message is to be displayed. + + Typically a MIME type.' + example: text/plain + default_field: false + - name: delivery_timestamp + level: extended + type: date + description: The date and time when the email message was received by the service + or client. + example: '2020-11-10T22:12:34.8196921Z' default_field: false - - name: header.version + - name: direction level: extended type: keyword ignore_above: 1024 - description: Version of the ELF header. + description: The direction of the message based on the sending and receiving + domains. + example: inbound default_field: false - - name: import_hash + - name: from.address level: extended type: keyword ignore_above: 1024 - description: 'A hash of the imports in an ELF file. An import hash can be used - to fingerprint binaries even after recompilation or other code-level transformations - have occurred, which would change more traditional hash values. - - This is an ELF implementation of the Windows PE imphash.' - example: d41d8cd98f00b204e9800998ecf8427e + description: The email address of the sender, typically from the RFC 5322 `From:` + header field. + example: sender@example.com default_field: false - - name: imports + - name: local_id level: extended - type: flattened - description: List of imported element names and types. + type: keyword + ignore_above: 1024 + description: 'Unique identifier given to the email by the source that created + the event. + + Identifier is not persistent across hops.' + example: c26dbea0-80d5-463b-b93c-4e8b708219ce default_field: false - - name: imports_names_entropy + - name: message_id level: extended - type: long - format: number - description: Shannon entropy calculation from the list of imported element names - and types. + type: wildcard + description: Identifier from the RFC 5322 `Message-ID:` email header that refers + to a particular email message. + example: 81ce15$8r2j59@mail01.example.com default_field: false - - name: imports_names_var_entropy + - name: origination_timestamp level: extended - type: long - format: number - description: Variance for Shannon entropy calculation from the list of imported - element names and types. + type: date + description: The date and time the email message was composed. Many email clients + will fill in this value automatically when the message is sent by a user. + example: '2020-11-10T22:12:34.8196921Z' default_field: false - - name: sections + - name: reply_to.address level: extended - type: nested - description: 'An array containing an object for each section of the ELF file. - - The keys that should be present in these objects are defined by sub-fields - underneath `elf.sections.*`.' - default_field: false - - name: sections.chi2 - level: extended - type: long - format: number - description: Chi-square probability distribution of the section. - default_field: false - - name: sections.entropy - level: extended - type: long - format: number - description: Shannon entropy calculation from the section. - default_field: false - - name: sections.flags - level: extended - type: keyword - ignore_above: 1024 - description: ELF Section List flags. - default_field: false - - name: sections.name - level: extended - type: keyword - ignore_above: 1024 - description: ELF Section List name. - default_field: false - - name: sections.physical_offset - level: extended - type: keyword - ignore_above: 1024 - description: ELF Section List offset. - default_field: false - - name: sections.physical_size - level: extended - type: long - format: bytes - description: ELF Section List physical size. - default_field: false - - name: sections.type - level: extended - type: keyword - ignore_above: 1024 - description: ELF Section List type. - default_field: false - - name: sections.var_entropy - level: extended - type: long - format: number - description: Variance for Shannon entropy calculation from the section. - default_field: false - - name: sections.virtual_address - level: extended - type: long - format: string - description: ELF Section List virtual address. - default_field: false - - name: sections.virtual_size - level: extended - type: long - format: string - description: ELF Section List virtual size. - default_field: false - - name: segments - level: extended - type: nested - description: 'An array containing an object for each segment of the ELF file. - - The keys that should be present in these objects are defined by sub-fields - underneath `elf.segments.*`.' - default_field: false - - name: segments.sections - level: extended - type: keyword - ignore_above: 1024 - description: ELF object segment sections. - default_field: false - - name: segments.type - level: extended - type: keyword - ignore_above: 1024 - description: ELF object segment type. - default_field: false - - name: shared_libraries - level: extended - type: keyword - ignore_above: 1024 - description: List of shared libraries used by this ELF object. - default_field: false - - name: telfhash - level: extended - type: keyword - ignore_above: 1024 - description: telfhash symbol hash for ELF file. - default_field: false - - name: email - title: Email - group: 2 - description: 'Event details relating to an email transaction. - - This field set focuses on the email message header, body, and attachments. Network - protocols that send and receive email messages such as SMTP are outside the - scope of the `email.*` fields.' - type: group - default_field: true - fields: - - name: attachments - level: extended - type: nested - description: A list of objects describing the attachment files sent along with - an email message. - default_field: false - - name: attachments.file.extension - level: extended - type: keyword - ignore_above: 1024 - description: Attachment file extension, excluding the leading dot. - example: txt - default_field: false - - name: attachments.file.hash.md5 - level: extended - type: keyword - ignore_above: 1024 - description: MD5 hash. - default_field: false - - name: attachments.file.hash.sha1 - level: extended - type: keyword - ignore_above: 1024 - description: SHA1 hash. - default_field: false - - name: attachments.file.hash.sha256 - level: extended - type: keyword - ignore_above: 1024 - description: SHA256 hash. - default_field: false - - name: attachments.file.hash.sha384 - level: extended - type: keyword - ignore_above: 1024 - description: SHA384 hash. - default_field: false - - name: attachments.file.hash.sha512 - level: extended - type: keyword - ignore_above: 1024 - description: SHA512 hash. - default_field: false - - name: attachments.file.hash.ssdeep - level: extended - type: keyword - ignore_above: 1024 - description: SSDEEP hash. - default_field: false - - name: attachments.file.hash.tlsh - level: extended - type: keyword - ignore_above: 1024 - description: TLSH hash. - default_field: false - - name: attachments.file.mime_type - level: extended - type: keyword - ignore_above: 1024 - description: 'The MIME media type of the attachment. - - This value will typically be extracted from the `Content-Type` MIME header - field.' - example: text/plain - default_field: false - - name: attachments.file.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the attachment file including the file extension. - example: attachment.txt - default_field: false - - name: attachments.file.size - level: extended - type: long - description: Attachment file size in bytes. - example: 64329 - default_field: false - - name: bcc.address - level: extended - type: keyword - ignore_above: 1024 - description: The email address of BCC recipient - example: bcc.user1@example.com - default_field: false - - name: cc.address - level: extended - type: keyword - ignore_above: 1024 - description: The email address of CC recipient - example: cc.user1@example.com - default_field: false - - name: content_type - level: extended - type: keyword - ignore_above: 1024 - description: 'Information about how the message is to be displayed. - - Typically a MIME type.' - example: text/plain - default_field: false - - name: delivery_timestamp - level: extended - type: date - description: The date and time when the email message was received by the service - or client. - example: '2020-11-10T22:12:34.8196921Z' - default_field: false - - name: direction - level: extended - type: keyword - ignore_above: 1024 - description: The direction of the message based on the sending and receiving - domains. - example: inbound - default_field: false - - name: from.address - level: extended - type: keyword - ignore_above: 1024 - description: The email address of the sender, typically from the RFC 5322 `From:` - header field. - example: sender@example.com - default_field: false - - name: local_id - level: extended - type: keyword - ignore_above: 1024 - description: 'Unique identifier given to the email by the source that created - the event. - - Identifier is not persistent across hops.' - example: c26dbea0-80d5-463b-b93c-4e8b708219ce - default_field: false - - name: message_id - level: extended - type: wildcard - description: Identifier from the RFC 5322 `Message-ID:` email header that refers - to a particular email message. - example: 81ce15$8r2j59@mail01.example.com - default_field: false - - name: origination_timestamp - level: extended - type: date - description: The date and time the email message was composed. Many email clients - will fill in this value automatically when the message is sent by a user. - example: '2020-11-10T22:12:34.8196921Z' - default_field: false - - name: reply_to.address - level: extended - type: keyword - ignore_above: 1024 - description: The address that replies should be delivered to based on the value - in the RFC 5322 `Reply-To:` header. - example: reply.here@example.com + type: keyword + ignore_above: 1024 + description: The address that replies should be delivered to based on the value + in the RFC 5322 `Reply-To:` header. + example: reply.here@example.com default_field: false - name: sender.address level: extended @@ -3722,104 +3358,16 @@ description: Version of x509 format. example: 3 default_field: false - - name: geo - title: Geo + - name: group + title: Group group: 2 - description: 'Geo fields can carry data about a specific location related to an - event. - - This geolocation information can be derived from techniques such as Geo IP, - or be user-supplied.' + description: The group fields are meant to represent groups that are relevant + to the event. type: group default_field: true fields: - - name: city_name - level: core - type: keyword - ignore_above: 1024 - description: City name. - example: Montreal - - name: continent_code - level: core - type: keyword - ignore_above: 1024 - description: Two-letter code representing continent's name. - example: NA - default_field: false - - name: continent_name - level: core - type: keyword - ignore_above: 1024 - description: Name of the continent. - example: North America - - name: country_iso_code - level: core - type: keyword - ignore_above: 1024 - description: Country ISO code. - example: CA - - name: country_name - level: core - type: keyword - ignore_above: 1024 - description: Country name. - example: Canada - - name: location - level: core - type: geo_point - description: Longitude and latitude. - example: '{ "lon": -73.614830, "lat": 45.505918 }' - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: 'User-defined description of a location, at the level of granularity - they care about. - - Could be the name of their data centers, the floor number, if this describes - a local physical entity, city names. - - Not typically used in automated geolocation.' - example: boston-dc - - name: postal_code - level: core - type: keyword - ignore_above: 1024 - description: 'Postal code associated with the location. - - Values appropriate for this field may also be known as a postcode or ZIP code - and will vary widely from country to country.' - example: 94040 - default_field: false - - name: region_iso_code - level: core - type: keyword - ignore_above: 1024 - description: Region ISO code. - example: CA-QC - - name: region_name - level: core - type: keyword - ignore_above: 1024 - description: Region name. - example: Quebec - - name: timezone - level: core - type: keyword - ignore_above: 1024 - description: The time zone of the location, such as IANA time zone name. - example: America/Argentina/Buenos_Aires - default_field: false - - name: group - title: Group - group: 2 - description: The group fields are meant to represent groups that are relevant - to the event. - type: group - default_field: true - fields: - - name: domain - level: extended + - name: domain + level: extended type: keyword ignore_above: 1024 description: 'Name of the directory the group is a member of. @@ -3835,60 +3383,6 @@ type: keyword ignore_above: 1024 description: Name of the group. - - name: hash - title: Hash - group: 2 - description: 'The hash fields represent different bitwise hash algorithms and - their values. - - Field names for common hashes (e.g. MD5, SHA1) are predefined. Add fields for - other hashes by lowercasing the hash algorithm name and using underscore separators - as appropriate (snake case, e.g. sha3_512). - - Note that this fieldset is used for common hashes that may be computed over - a range of generic bytes. Entity-specific hashes such as ja3 or imphash are - placed in the fieldsets to which they relate (tls and pe, respectively).' - type: group - default_field: true - fields: - - name: md5 - level: extended - type: keyword - ignore_above: 1024 - description: MD5 hash. - - name: sha1 - level: extended - type: keyword - ignore_above: 1024 - description: SHA1 hash. - - name: sha256 - level: extended - type: keyword - ignore_above: 1024 - description: SHA256 hash. - - name: sha384 - level: extended - type: keyword - ignore_above: 1024 - description: SHA384 hash. - default_field: false - - name: sha512 - level: extended - type: keyword - ignore_above: 1024 - description: SHA512 hash. - - name: ssdeep - level: extended - type: keyword - ignore_above: 1024 - description: SSDEEP hash. - default_field: false - - name: tlsh - level: extended - type: keyword - ignore_above: 1024 - description: TLSH hash. - default_field: false - name: host title: Host group: 2 @@ -4329,40 +3823,6 @@ ignore_above: 1024 description: HTTP version. example: 1.1 - - name: interface - title: Interface - group: 2 - description: The interface fields are used to record ingress and egress interface - information when reported by an observer (e.g. firewall, router, load balancer) - in the context of the observer handling a network connection. In the case of - a single observer interface (e.g. network sensor on a span port) only the observer.ingress - information should be populated. - type: group - default_field: true - fields: - - name: alias - level: extended - type: keyword - ignore_above: 1024 - description: Interface alias as reported by the system, typically used in firewall - implementations for e.g. inside, outside, or dmz logical interface naming. - example: outside - default_field: false - - name: id - level: extended - type: keyword - ignore_above: 1024 - description: Interface ID as reported by an observer (typically SNMP interface - ID). - example: 10 - default_field: false - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Interface name as reported by the system. - example: eth0 - default_field: false - name: log title: Log group: 2 @@ -4527,128 +3987,6 @@ for RFC 5424 messages. example: 1 default_field: false - - name: macho - title: Mach-O Header - group: 2 - description: These fields contain Mac OS Mach Object file format (Mach-O) metadata. - type: group - default_field: true - fields: - - name: go_import_hash - level: extended - type: keyword - ignore_above: 1024 - description: 'A hash of the Go language imports in a Mach-O file excluding standard - library imports. An import hash can be used to fingerprint binaries even after - recompilation or other code-level transformations have occurred, which would - change more traditional hash values. - - The algorithm used to calculate the Go symbol hash and a reference implementation - are available [here](https://github.com/elastic/toutoumomoma).' - example: 10bddcb4cee42080f76c88d9ff964491 - default_field: false - - name: go_imports - level: extended - type: flattened - description: List of imported Go language element names and types. - default_field: false - - name: go_imports_names_entropy - level: extended - type: long - format: number - description: Shannon entropy calculation from the list of Go imports. - default_field: false - - name: go_imports_names_var_entropy - level: extended - type: long - format: number - description: Variance for Shannon entropy calculation from the list of Go imports. - default_field: false - - name: go_stripped - level: extended - type: boolean - description: Set to true if the file is a Go executable that has had its symbols - stripped or obfuscated and false if an unobfuscated Go executable. - default_field: false - - name: import_hash - level: extended - type: keyword - ignore_above: 1024 - description: 'A hash of the imports in a Mach-O file. An import hash can be - used to fingerprint binaries even after recompilation or other code-level - transformations have occurred, which would change more traditional hash values. - - This is a synonym for symhash.' - example: d41d8cd98f00b204e9800998ecf8427e - default_field: false - - name: imports - level: extended - type: flattened - description: List of imported element names and types. - default_field: false - - name: imports_names_entropy - level: extended - type: long - format: number - description: Shannon entropy calculation from the list of imported element names - and types. - default_field: false - - name: imports_names_var_entropy - level: extended - type: long - format: number - description: Variance for Shannon entropy calculation from the list of imported - element names and types. - default_field: false - - name: sections - level: extended - type: nested - description: 'An array containing an object for each section of the Mach-O file. - - The keys that should be present in these objects are defined by sub-fields - underneath `macho.sections.*`.' - default_field: false - - name: sections.entropy - level: extended - type: long - format: number - description: Shannon entropy calculation from the section. - default_field: false - - name: sections.name - level: extended - type: keyword - ignore_above: 1024 - description: Mach-O Section List name. - default_field: false - - name: sections.physical_size - level: extended - type: long - format: bytes - description: Mach-O Section List physical size. - default_field: false - - name: sections.var_entropy - level: extended - type: long - format: number - description: Variance for Shannon entropy calculation from the section. - default_field: false - - name: sections.virtual_size - level: extended - type: long - format: string - description: Mach-O Section List virtual size. This is always the same as `physical_size`. - default_field: false - - name: symhash - level: extended - type: keyword - ignore_above: 1024 - description: 'A hash of the imports in a Mach-O file. An import hash can be - used to fingerprint binaries even after recompilation or other code-level - transformations have occurred, which would change more traditional hash values. - - This is a Mach-O implementation of the Windows PE imphash' - example: d3ccf195b62a9279c3c19af1080497ec - default_field: false - name: network title: Network group: 2 @@ -5262,69 +4600,6 @@ type: match_only_text default_field: false description: Organization name. - - name: os - title: Operating System - group: 2 - description: The OS fields contain information about the operating system. - type: group - default_field: true - fields: - - name: family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: full - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: match_only_text - default_field: false - description: Operating system name, including the version or code name. - example: Mac OS Mojave - - name: kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: match_only_text - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: type - level: extended - type: keyword - ignore_above: 1024 - description: 'Use the `os.type` field to categorize the operating system into - one of the broad commercial families. - - If the OS you''re dealing with is not listed as an expected value, the field - should not be populated. Please let us know by opening an issue with ECS, - to propose its addition.' - example: macos - default_field: false - - name: version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - name: package title: Package group: 2 @@ -5386,219 +4661,44 @@ level: extended type: keyword ignore_above: 1024 - description: Package name - example: go - - name: path - level: extended - type: keyword - ignore_above: 1024 - description: Path where the package is installed. - example: /usr/local/Cellar/go/1.12.9/ - - name: reference - level: extended - type: keyword - ignore_above: 1024 - description: Home page or reference URL of the software in this package, if - available. - example: https://golang.org - default_field: false - - name: size - level: extended - type: long - format: string - description: Package size in bytes. - example: 62231 - - name: type - level: extended - type: keyword - ignore_above: 1024 - description: 'Type of package. - - This should contain the package file type, rather than the package manager - name. Examples: rpm, dpkg, brew, npm, gem, nupkg, jar.' - example: rpm - default_field: false - - name: version - level: extended - type: keyword - ignore_above: 1024 - description: Package version - example: 1.12.9 - - name: pe - title: PE Header - group: 2 - description: These fields contain Windows Portable Executable (PE) metadata. - type: group - default_field: true - fields: - - name: architecture - level: extended - type: keyword - ignore_above: 1024 - description: CPU architecture target for the file. - example: x64 - default_field: false - - name: company - level: extended - type: keyword - ignore_above: 1024 - description: Internal company name of the file, provided at compile-time. - example: Microsoft Corporation - default_field: false - - name: description - level: extended - type: keyword - ignore_above: 1024 - description: Internal description of the file, provided at compile-time. - example: Paint - default_field: false - - name: file_version - level: extended - type: keyword - ignore_above: 1024 - description: Internal version of the file, provided at compile-time. - example: 6.3.9600.17415 - default_field: false - - name: go_import_hash - level: extended - type: keyword - ignore_above: 1024 - description: 'A hash of the Go language imports in a PE file excluding standard - library imports. An import hash can be used to fingerprint binaries even after - recompilation or other code-level transformations have occurred, which would - change more traditional hash values. - - The algorithm used to calculate the Go symbol hash and a reference implementation - are available [here](https://github.com/elastic/toutoumomoma).' - example: 10bddcb4cee42080f76c88d9ff964491 - default_field: false - - name: go_imports - level: extended - type: flattened - description: List of imported Go language element names and types. - default_field: false - - name: go_imports_names_entropy - level: extended - type: long - format: number - description: Shannon entropy calculation from the list of Go imports. - default_field: false - - name: go_imports_names_var_entropy - level: extended - type: long - format: number - description: Variance for Shannon entropy calculation from the list of Go imports. - default_field: false - - name: go_stripped - level: extended - type: boolean - description: Set to true if the file is a Go executable that has had its symbols - stripped or obfuscated and false if an unobfuscated Go executable. - default_field: false - - name: imphash - level: extended - type: keyword - ignore_above: 1024 - description: 'A hash of the imports in a PE file. An imphash -- or import hash - -- can be used to fingerprint binaries even after recompilation or other code-level - transformations have occurred, which would change more traditional hash values. - - Learn more at https://www.fireeye.com/blog/threat-research/2014/01/tracking-malware-import-hashing.html.' - example: 0c6803c4e922103c4dca5963aad36ddf - default_field: false - - name: import_hash - level: extended - type: keyword - ignore_above: 1024 - description: 'A hash of the imports in a PE file. An import hash can be used - to fingerprint binaries even after recompilation or other code-level transformations - have occurred, which would change more traditional hash values. - - This is a synonym for imphash.' - example: d41d8cd98f00b204e9800998ecf8427e - default_field: false - - name: imports - level: extended - type: flattened - description: List of imported element names and types. - default_field: false - - name: imports_names_entropy - level: extended - type: long - format: number - description: Shannon entropy calculation from the list of imported element names - and types. - default_field: false - - name: imports_names_var_entropy - level: extended - type: long - format: number - description: Variance for Shannon entropy calculation from the list of imported - element names and types. - default_field: false - - name: original_file_name - level: extended - type: keyword - ignore_above: 1024 - description: Internal name of the file, provided at compile-time. - example: MSPAINT.EXE - default_field: false - - name: pehash - level: extended - type: keyword - ignore_above: 1024 - description: 'A hash of the PE header and data from one or more PE sections. - An pehash can be used to cluster files by transforming structural information - about a file into a hash value. - - Learn more at https://www.usenix.org/legacy/events/leet09/tech/full_papers/wicherski/wicherski_html/index.html.' - example: 73ff189b63cd6be375a7ff25179a38d347651975 - default_field: false - - name: product - level: extended - type: keyword - ignore_above: 1024 - description: Internal product name of the file, provided at compile-time. - example: "Microsoft\xAE Windows\xAE Operating System" - default_field: false - - name: sections - level: extended - type: nested - description: 'An array containing an object for each section of the PE file. - - The keys that should be present in these objects are defined by sub-fields - underneath `pe.sections.*`.' - default_field: false - - name: sections.entropy - level: extended - type: long - format: number - description: Shannon entropy calculation from the section. - default_field: false - - name: sections.name - level: extended - type: keyword - ignore_above: 1024 - description: PE Section List name. - default_field: false - - name: sections.physical_size + description: Package name + example: go + - name: path level: extended - type: long - format: bytes - description: PE Section List physical size. - default_field: false - - name: sections.var_entropy + type: keyword + ignore_above: 1024 + description: Path where the package is installed. + example: /usr/local/Cellar/go/1.12.9/ + - name: reference level: extended - type: long - format: number - description: Variance for Shannon entropy calculation from the section. + type: keyword + ignore_above: 1024 + description: Home page or reference URL of the software in this package, if + available. + example: https://golang.org default_field: false - - name: sections.virtual_size + - name: size level: extended type: long format: string - description: PE Section List virtual size. This is always the same as `physical_size`. + description: Package size in bytes. + example: 62231 + - name: type + level: extended + type: keyword + ignore_above: 1024 + description: 'Type of package. + + This should contain the package file type, rather than the package manager + name. Examples: rpm, dpkg, brew, npm, gem, nupkg, jar.' + example: rpm default_field: false + - name: version + level: extended + type: keyword + ignore_above: 1024 + description: Package version + example: 1.12.9 - name: process title: Process group: 2 @@ -8777,62 +7877,6 @@ ignore_above: 1024 description: All the user names or other user identifiers seen on the event. default_field: false - - name: risk - title: Risk information - group: 2 - description: Fields for describing risk score and risk level of entities such - as hosts and users. These fields are not allowed to be nested under `event.*`. - Please continue to use `event.risk_score` and `event.risk_score_norm` for event - risk. - type: group - default_field: true - fields: - - name: calculated_level - level: extended - type: keyword - ignore_above: 1024 - description: A risk classification level calculated by an internal system as - part of entity analytics and entity risk scoring. - example: High - default_field: false - - name: calculated_score - level: extended - type: float - description: A risk classification score calculated by an internal system as - part of entity analytics and entity risk scoring. - example: 880.73 - default_field: false - - name: calculated_score_norm - level: extended - type: float - description: A risk classification score calculated by an internal system as - part of entity analytics and entity risk scoring, and normalized to a range - of 0 to 100. - example: 88.73 - default_field: false - - name: static_level - level: extended - type: keyword - ignore_above: 1024 - description: A risk classification level obtained from outside the system, such - as from some external Threat Intelligence Platform. - example: High - default_field: false - - name: static_score - level: extended - type: float - description: A risk classification score obtained from outside the system, such - as from some external Threat Intelligence Platform. - example: 830.0 - default_field: false - - name: static_score_norm - level: extended - type: float - description: A risk classification score obtained from outside the system, such - as from some external Threat Intelligence Platform, and normalized to a range - of 0 to 100. - example: 83.0 - default_field: false - name: rule title: Rule group: 2 @@ -14401,42 +13445,6 @@ ignore_above: 1024 description: Version of the user agent. example: 12.0 - - name: vlan - title: VLAN - group: 2 - description: 'The VLAN fields are used to identify 802.1q tag(s) of a packet, - as well as ingress and egress VLAN associations of an observer in relation to - a specific packet or connection. - - Network.vlan fields are used to record a single VLAN tag, or the outer tag in - the case of q-in-q encapsulations, for a packet or connection as observed, typically - provided by a network sensor (e.g. Zeek, Wireshark) passively reporting on traffic. - - Network.inner VLAN fields are used to report inner q-in-q 802.1q tags (multiple - 802.1q encapsulations) as observed, typically provided by a network sensor (e.g. - Zeek, Wireshark) passively reporting on traffic. Network.inner VLAN fields should - only be used in addition to network.vlan fields to indicate q-in-q tagging. - - Observer.ingress and observer.egress VLAN values are used to record observer - specific information when observer events contain discrete ingress and egress - VLAN information, typically provided by firewalls, routers, or load balancers.' - type: group - default_field: true - fields: - - name: id - level: extended - type: keyword - ignore_above: 1024 - description: VLAN ID as reported by the observer. - example: 10 - default_field: false - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Optional VLAN name as reported by the observer. - example: outside - default_field: false - name: vulnerability title: Vulnerability group: 2 @@ -14566,193 +13574,3 @@ prioritization regarding remediation. For example (https://nvd.nist.gov/vuln-metrics/cvss) example: Critical default_field: false - - name: x509 - title: x509 Certificate - group: 2 - description: 'This implements the common core fields for x509 certificates. This - information is likely logged with TLS sessions, digital signatures found in - executable binaries, S/MIME information in email bodies, or analysis of files - on disk. - - When the certificate relates to a file, use the fields at `file.x509`. When - hashes of the DER-encoded certificate are available, the `hash` data set should - be populated as well (e.g. `file.hash.sha256`). - - Events that contain certificate information about network connections, should - use the x509 fields under the relevant TLS fields: `tls.server.x509` and/or - `tls.client.x509`.' - type: group - default_field: true - fields: - - name: alternative_names - level: extended - type: keyword - ignore_above: 1024 - description: List of subject alternative names (SAN). Name types vary by certificate - authority and certificate type but commonly contain IP addresses, DNS names - (and wildcards), and email addresses. - example: '*.elastic.co' - default_field: false - - name: issuer.common_name - level: extended - type: keyword - ignore_above: 1024 - description: List of common name (CN) of issuing certificate authority. - example: Example SHA2 High Assurance Server CA - default_field: false - - name: issuer.country - level: extended - type: keyword - ignore_above: 1024 - description: List of country \(C) codes - example: US - default_field: false - - name: issuer.distinguished_name - level: extended - type: keyword - ignore_above: 1024 - description: Distinguished name (DN) of issuing certificate authority. - example: C=US, O=Example Inc, OU=www.example.com, CN=Example SHA2 High Assurance - Server CA - default_field: false - - name: issuer.locality - level: extended - type: keyword - ignore_above: 1024 - description: List of locality names (L) - example: Mountain View - default_field: false - - name: issuer.organization - level: extended - type: keyword - ignore_above: 1024 - description: List of organizations (O) of issuing certificate authority. - example: Example Inc - default_field: false - - name: issuer.organizational_unit - level: extended - type: keyword - ignore_above: 1024 - description: List of organizational units (OU) of issuing certificate authority. - example: www.example.com - default_field: false - - name: issuer.state_or_province - level: extended - type: keyword - ignore_above: 1024 - description: List of state or province names (ST, S, or P) - example: California - default_field: false - - name: not_after - level: extended - type: date - description: Time at which the certificate is no longer considered valid. - example: '2020-07-16T03:15:39Z' - default_field: false - - name: not_before - level: extended - type: date - description: Time at which the certificate is first considered valid. - example: '2019-08-16T01:40:25Z' - default_field: false - - name: public_key_algorithm - level: extended - type: keyword - ignore_above: 1024 - description: Algorithm used to generate the public key. - example: RSA - default_field: false - - name: public_key_curve - level: extended - type: keyword - ignore_above: 1024 - description: The curve used by the elliptic curve public key algorithm. This - is algorithm specific. - example: nistp521 - default_field: false - - name: public_key_exponent - level: extended - type: long - description: Exponent used to derive the public key. This is algorithm specific. - example: 65537 - index: false - doc_values: false - default_field: false - - name: public_key_size - level: extended - type: long - description: The size of the public key space in bits. - example: 2048 - default_field: false - - name: serial_number - level: extended - type: keyword - ignore_above: 1024 - description: Unique serial number issued by the certificate authority. For consistency, - if this value is alphanumeric, it should be formatted without colons and uppercase - characters. - example: 55FBB9C7DEBF09809D12CCAA - default_field: false - - name: signature_algorithm - level: extended - type: keyword - ignore_above: 1024 - description: Identifier for certificate signature algorithm. We recommend using - names found in Go Lang Crypto library. See https://github.com/golang/go/blob/go1.14/src/crypto/x509/x509.go#L337-L353. - example: SHA256-RSA - default_field: false - - name: subject.common_name - level: extended - type: keyword - ignore_above: 1024 - description: List of common names (CN) of subject. - example: shared.global.example.net - default_field: false - - name: subject.country - level: extended - type: keyword - ignore_above: 1024 - description: List of country \(C) code - example: US - default_field: false - - name: subject.distinguished_name - level: extended - type: keyword - ignore_above: 1024 - description: Distinguished name (DN) of the certificate subject entity. - example: C=US, ST=California, L=San Francisco, O=Example, Inc., CN=shared.global.example.net - default_field: false - - name: subject.locality - level: extended - type: keyword - ignore_above: 1024 - description: List of locality names (L) - example: San Francisco - default_field: false - - name: subject.organization - level: extended - type: keyword - ignore_above: 1024 - description: List of organizations (O) of subject. - example: Example, Inc. - default_field: false - - name: subject.organizational_unit - level: extended - type: keyword - ignore_above: 1024 - description: List of organizational units (OU) of subject. - default_field: false - - name: subject.state_or_province - level: extended - type: keyword - ignore_above: 1024 - description: List of state or province names (ST, S, or P) - example: California - default_field: false - - name: version_number - level: extended - type: keyword - ignore_above: 1024 - description: Version of x509 format. - example: 3 - default_field: false diff --git a/generated/beats/fields.ecs.yml b/generated/beats/fields.ecs.yml index c13db67d1a..3e0b8c5d52 100644 --- a/generated/beats/fields.ecs.yml +++ b/generated/beats/fields.ecs.yml @@ -122,32 +122,6 @@ ignore_above: 1024 description: Version of the agent. example: 6.0.0-rc2 - - name: as - title: Autonomous System - group: 2 - description: An autonomous system (AS) is a collection of connected Internet Protocol - (IP) routing prefixes under the control of one or more network operators on - behalf of a single administrative entity or domain that presents a common, clearly - defined routing policy to the internet. - type: group - default_field: true - fields: - - name: number - level: extended - type: long - description: Unique number allocated to the autonomous system. The autonomous - system number (ASN) uniquely identifies each network on the Internet. - example: 15169 - - name: organization.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: match_only_text - default_field: false - description: Organization name. - example: Google LLC - name: client title: Client group: 2 @@ -727,91 +701,6 @@ Examples: app engine, app service, cloud run, fargate, lambda.' example: lambda default_field: false - - name: code_signature - title: Code Signature - group: 2 - description: These fields contain information about binary code signatures. - type: group - default_field: true - fields: - - name: digest_algorithm - level: extended - type: keyword - ignore_above: 1024 - description: 'The hashing algorithm used to sign the process. - - This value can distinguish signatures when a file is signed multiple times - by the same signer but with a different digest algorithm.' - example: sha256 - default_field: false - - name: exists - level: core - type: boolean - description: Boolean to capture if a signature is present. - example: 'true' - default_field: false - - name: signing_id - level: extended - type: keyword - ignore_above: 1024 - description: 'The identifier used to sign the process. - - This is used to identify the application manufactured by a software vendor. - The field is relevant to Apple *OS only.' - example: com.apple.xpc.proxy - default_field: false - - name: status - level: extended - type: keyword - ignore_above: 1024 - description: 'Additional information about the certificate status. - - This is useful for logging cryptographic errors with the certificate validity - or trust status. Leave unpopulated if the validity or trust of the certificate - was unchecked.' - example: ERROR_UNTRUSTED_ROOT - default_field: false - - name: subject_name - level: core - type: keyword - ignore_above: 1024 - description: Subject name of the code signer - example: Microsoft Corporation - default_field: false - - name: team_id - level: extended - type: keyword - ignore_above: 1024 - description: 'The team identifier used to sign the process. - - This is used to identify the team or vendor of a software product. The field - is relevant to Apple *OS only.' - example: EQHXZ8M8AV - default_field: false - - name: timestamp - level: extended - type: date - description: Date and time when the code signature was generated and signed. - example: '2021-01-01T12:10:30Z' - default_field: false - - name: trusted - level: extended - type: boolean - description: 'Stores the trust status of the certificate chain. - - Validating the trust of the certificate chain may be complicated, and this - field should only be populated by tools that actively check the status.' - example: 'true' - default_field: false - - name: valid - level: extended - type: boolean - description: 'Boolean to capture if the digital signature is verified against - the binary content. - - Leave unpopulated if a certificate was unchecked.' - example: 'true' - default_field: false - name: container title: Container group: 2 @@ -1797,425 +1686,172 @@ ECS versions -- this field lets integrations adjust to the schema version of the events.' example: 1.0.0 - - name: elf - title: ELF Header + - name: email + title: Email group: 2 - description: These fields contain Linux Executable Linkable Format (ELF) metadata. + description: 'Event details relating to an email transaction. + + This field set focuses on the email message header, body, and attachments. Network + protocols that send and receive email messages such as SMTP are outside the + scope of the `email.*` fields.' type: group default_field: true fields: - - name: architecture + - name: attachments level: extended - type: keyword - ignore_above: 1024 - description: Machine architecture of the ELF file. - example: x86-64 + type: nested + description: A list of objects describing the attachment files sent along with + an email message. default_field: false - - name: byte_order + - name: attachments.file.extension level: extended type: keyword ignore_above: 1024 - description: Byte sequence of ELF file. - example: Little Endian + description: Attachment file extension, excluding the leading dot. + example: txt default_field: false - - name: cpu_type + - name: attachments.file.hash.md5 level: extended type: keyword ignore_above: 1024 - description: CPU type of the ELF file. - example: Intel - default_field: false - - name: creation_date - level: extended - type: date - description: Extracted when possible from the file's metadata. Indicates when - it was built or compiled. It can also be faked by malware creators. - default_field: false - - name: exports - level: extended - type: flattened - description: List of exported element names and types. + description: MD5 hash. default_field: false - - name: go_import_hash + - name: attachments.file.hash.sha1 level: extended type: keyword ignore_above: 1024 - description: 'A hash of the Go language imports in an ELF file excluding standard - library imports. An import hash can be used to fingerprint binaries even after - recompilation or other code-level transformations have occurred, which would - change more traditional hash values. - - The algorithm used to calculate the Go symbol hash and a reference implementation - are available [here](https://github.com/elastic/toutoumomoma).' - example: 10bddcb4cee42080f76c88d9ff964491 + description: SHA1 hash. default_field: false - - name: go_imports + - name: attachments.file.hash.sha256 level: extended - type: flattened - description: List of imported Go language element names and types. + type: keyword + ignore_above: 1024 + description: SHA256 hash. default_field: false - - name: go_imports_names_entropy + - name: attachments.file.hash.sha384 level: extended - type: long - format: number - description: Shannon entropy calculation from the list of Go imports. + type: keyword + ignore_above: 1024 + description: SHA384 hash. default_field: false - - name: go_imports_names_var_entropy + - name: attachments.file.hash.sha512 level: extended - type: long - format: number - description: Variance for Shannon entropy calculation from the list of Go imports. + type: keyword + ignore_above: 1024 + description: SHA512 hash. default_field: false - - name: go_stripped + - name: attachments.file.hash.ssdeep level: extended - type: boolean - description: Set to true if the file is a Go executable that has had its symbols - stripped or obfuscated and false if an unobfuscated Go executable. + type: keyword + ignore_above: 1024 + description: SSDEEP hash. default_field: false - - name: header.abi_version + - name: attachments.file.hash.tlsh level: extended type: keyword ignore_above: 1024 - description: Version of the ELF Application Binary Interface (ABI). + description: TLSH hash. default_field: false - - name: header.class + - name: attachments.file.mime_type level: extended type: keyword ignore_above: 1024 - description: Header class of the ELF file. + description: 'The MIME media type of the attachment. + + This value will typically be extracted from the `Content-Type` MIME header + field.' + example: text/plain default_field: false - - name: header.data + - name: attachments.file.name level: extended type: keyword ignore_above: 1024 - description: Data table of the ELF header. + description: Name of the attachment file including the file extension. + example: attachment.txt default_field: false - - name: header.entrypoint + - name: attachments.file.size level: extended type: long - format: string - description: Header entrypoint of the ELF file. + description: Attachment file size in bytes. + example: 64329 default_field: false - - name: header.object_version + - name: bcc.address level: extended type: keyword ignore_above: 1024 - description: '"0x1" for original ELF files.' + description: The email address of BCC recipient + example: bcc.user1@example.com default_field: false - - name: header.os_abi + - name: cc.address level: extended type: keyword ignore_above: 1024 - description: Application Binary Interface (ABI) of the Linux OS. + description: The email address of CC recipient + example: cc.user1@example.com default_field: false - - name: header.type + - name: content_type level: extended type: keyword ignore_above: 1024 - description: Header type of the ELF file. + description: 'Information about how the message is to be displayed. + + Typically a MIME type.' + example: text/plain + default_field: false + - name: delivery_timestamp + level: extended + type: date + description: The date and time when the email message was received by the service + or client. + example: '2020-11-10T22:12:34.8196921Z' default_field: false - - name: header.version + - name: direction level: extended type: keyword ignore_above: 1024 - description: Version of the ELF header. + description: The direction of the message based on the sending and receiving + domains. + example: inbound default_field: false - - name: import_hash + - name: from.address level: extended type: keyword ignore_above: 1024 - description: 'A hash of the imports in an ELF file. An import hash can be used - to fingerprint binaries even after recompilation or other code-level transformations - have occurred, which would change more traditional hash values. - - This is an ELF implementation of the Windows PE imphash.' - example: d41d8cd98f00b204e9800998ecf8427e + description: The email address of the sender, typically from the RFC 5322 `From:` + header field. + example: sender@example.com default_field: false - - name: imports + - name: local_id level: extended - type: flattened - description: List of imported element names and types. + type: keyword + ignore_above: 1024 + description: 'Unique identifier given to the email by the source that created + the event. + + Identifier is not persistent across hops.' + example: c26dbea0-80d5-463b-b93c-4e8b708219ce default_field: false - - name: imports_names_entropy + - name: message_id level: extended - type: long - format: number - description: Shannon entropy calculation from the list of imported element names - and types. + type: wildcard + description: Identifier from the RFC 5322 `Message-ID:` email header that refers + to a particular email message. + example: 81ce15$8r2j59@mail01.example.com default_field: false - - name: imports_names_var_entropy + - name: origination_timestamp level: extended - type: long - format: number - description: Variance for Shannon entropy calculation from the list of imported - element names and types. + type: date + description: The date and time the email message was composed. Many email clients + will fill in this value automatically when the message is sent by a user. + example: '2020-11-10T22:12:34.8196921Z' default_field: false - - name: sections + - name: reply_to.address level: extended - type: nested - description: 'An array containing an object for each section of the ELF file. - - The keys that should be present in these objects are defined by sub-fields - underneath `elf.sections.*`.' - default_field: false - - name: sections.chi2 - level: extended - type: long - format: number - description: Chi-square probability distribution of the section. - default_field: false - - name: sections.entropy - level: extended - type: long - format: number - description: Shannon entropy calculation from the section. - default_field: false - - name: sections.flags - level: extended - type: keyword - ignore_above: 1024 - description: ELF Section List flags. - default_field: false - - name: sections.name - level: extended - type: keyword - ignore_above: 1024 - description: ELF Section List name. - default_field: false - - name: sections.physical_offset - level: extended - type: keyword - ignore_above: 1024 - description: ELF Section List offset. - default_field: false - - name: sections.physical_size - level: extended - type: long - format: bytes - description: ELF Section List physical size. - default_field: false - - name: sections.type - level: extended - type: keyword - ignore_above: 1024 - description: ELF Section List type. - default_field: false - - name: sections.var_entropy - level: extended - type: long - format: number - description: Variance for Shannon entropy calculation from the section. - default_field: false - - name: sections.virtual_address - level: extended - type: long - format: string - description: ELF Section List virtual address. - default_field: false - - name: sections.virtual_size - level: extended - type: long - format: string - description: ELF Section List virtual size. - default_field: false - - name: segments - level: extended - type: nested - description: 'An array containing an object for each segment of the ELF file. - - The keys that should be present in these objects are defined by sub-fields - underneath `elf.segments.*`.' - default_field: false - - name: segments.sections - level: extended - type: keyword - ignore_above: 1024 - description: ELF object segment sections. - default_field: false - - name: segments.type - level: extended - type: keyword - ignore_above: 1024 - description: ELF object segment type. - default_field: false - - name: shared_libraries - level: extended - type: keyword - ignore_above: 1024 - description: List of shared libraries used by this ELF object. - default_field: false - - name: telfhash - level: extended - type: keyword - ignore_above: 1024 - description: telfhash symbol hash for ELF file. - default_field: false - - name: email - title: Email - group: 2 - description: 'Event details relating to an email transaction. - - This field set focuses on the email message header, body, and attachments. Network - protocols that send and receive email messages such as SMTP are outside the - scope of the `email.*` fields.' - type: group - default_field: true - fields: - - name: attachments - level: extended - type: nested - description: A list of objects describing the attachment files sent along with - an email message. - default_field: false - - name: attachments.file.extension - level: extended - type: keyword - ignore_above: 1024 - description: Attachment file extension, excluding the leading dot. - example: txt - default_field: false - - name: attachments.file.hash.md5 - level: extended - type: keyword - ignore_above: 1024 - description: MD5 hash. - default_field: false - - name: attachments.file.hash.sha1 - level: extended - type: keyword - ignore_above: 1024 - description: SHA1 hash. - default_field: false - - name: attachments.file.hash.sha256 - level: extended - type: keyword - ignore_above: 1024 - description: SHA256 hash. - default_field: false - - name: attachments.file.hash.sha384 - level: extended - type: keyword - ignore_above: 1024 - description: SHA384 hash. - default_field: false - - name: attachments.file.hash.sha512 - level: extended - type: keyword - ignore_above: 1024 - description: SHA512 hash. - default_field: false - - name: attachments.file.hash.ssdeep - level: extended - type: keyword - ignore_above: 1024 - description: SSDEEP hash. - default_field: false - - name: attachments.file.hash.tlsh - level: extended - type: keyword - ignore_above: 1024 - description: TLSH hash. - default_field: false - - name: attachments.file.mime_type - level: extended - type: keyword - ignore_above: 1024 - description: 'The MIME media type of the attachment. - - This value will typically be extracted from the `Content-Type` MIME header - field.' - example: text/plain - default_field: false - - name: attachments.file.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the attachment file including the file extension. - example: attachment.txt - default_field: false - - name: attachments.file.size - level: extended - type: long - description: Attachment file size in bytes. - example: 64329 - default_field: false - - name: bcc.address - level: extended - type: keyword - ignore_above: 1024 - description: The email address of BCC recipient - example: bcc.user1@example.com - default_field: false - - name: cc.address - level: extended - type: keyword - ignore_above: 1024 - description: The email address of CC recipient - example: cc.user1@example.com - default_field: false - - name: content_type - level: extended - type: keyword - ignore_above: 1024 - description: 'Information about how the message is to be displayed. - - Typically a MIME type.' - example: text/plain - default_field: false - - name: delivery_timestamp - level: extended - type: date - description: The date and time when the email message was received by the service - or client. - example: '2020-11-10T22:12:34.8196921Z' - default_field: false - - name: direction - level: extended - type: keyword - ignore_above: 1024 - description: The direction of the message based on the sending and receiving - domains. - example: inbound - default_field: false - - name: from.address - level: extended - type: keyword - ignore_above: 1024 - description: The email address of the sender, typically from the RFC 5322 `From:` - header field. - example: sender@example.com - default_field: false - - name: local_id - level: extended - type: keyword - ignore_above: 1024 - description: 'Unique identifier given to the email by the source that created - the event. - - Identifier is not persistent across hops.' - example: c26dbea0-80d5-463b-b93c-4e8b708219ce - default_field: false - - name: message_id - level: extended - type: wildcard - description: Identifier from the RFC 5322 `Message-ID:` email header that refers - to a particular email message. - example: 81ce15$8r2j59@mail01.example.com - default_field: false - - name: origination_timestamp - level: extended - type: date - description: The date and time the email message was composed. Many email clients - will fill in this value automatically when the message is sent by a user. - example: '2020-11-10T22:12:34.8196921Z' - default_field: false - - name: reply_to.address - level: extended - type: keyword - ignore_above: 1024 - description: The address that replies should be delivered to based on the value - in the RFC 5322 `Reply-To:` header. - example: reply.here@example.com + type: keyword + ignore_above: 1024 + description: The address that replies should be delivered to based on the value + in the RFC 5322 `Reply-To:` header. + example: reply.here@example.com default_field: false - name: sender.address level: extended @@ -3672,104 +3308,16 @@ description: Version of x509 format. example: 3 default_field: false - - name: geo - title: Geo + - name: group + title: Group group: 2 - description: 'Geo fields can carry data about a specific location related to an - event. - - This geolocation information can be derived from techniques such as Geo IP, - or be user-supplied.' + description: The group fields are meant to represent groups that are relevant + to the event. type: group default_field: true fields: - - name: city_name - level: core - type: keyword - ignore_above: 1024 - description: City name. - example: Montreal - - name: continent_code - level: core - type: keyword - ignore_above: 1024 - description: Two-letter code representing continent's name. - example: NA - default_field: false - - name: continent_name - level: core - type: keyword - ignore_above: 1024 - description: Name of the continent. - example: North America - - name: country_iso_code - level: core - type: keyword - ignore_above: 1024 - description: Country ISO code. - example: CA - - name: country_name - level: core - type: keyword - ignore_above: 1024 - description: Country name. - example: Canada - - name: location - level: core - type: geo_point - description: Longitude and latitude. - example: '{ "lon": -73.614830, "lat": 45.505918 }' - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: 'User-defined description of a location, at the level of granularity - they care about. - - Could be the name of their data centers, the floor number, if this describes - a local physical entity, city names. - - Not typically used in automated geolocation.' - example: boston-dc - - name: postal_code - level: core - type: keyword - ignore_above: 1024 - description: 'Postal code associated with the location. - - Values appropriate for this field may also be known as a postcode or ZIP code - and will vary widely from country to country.' - example: 94040 - default_field: false - - name: region_iso_code - level: core - type: keyword - ignore_above: 1024 - description: Region ISO code. - example: CA-QC - - name: region_name - level: core - type: keyword - ignore_above: 1024 - description: Region name. - example: Quebec - - name: timezone - level: core - type: keyword - ignore_above: 1024 - description: The time zone of the location, such as IANA time zone name. - example: America/Argentina/Buenos_Aires - default_field: false - - name: group - title: Group - group: 2 - description: The group fields are meant to represent groups that are relevant - to the event. - type: group - default_field: true - fields: - - name: domain - level: extended + - name: domain + level: extended type: keyword ignore_above: 1024 description: 'Name of the directory the group is a member of. @@ -3785,60 +3333,6 @@ type: keyword ignore_above: 1024 description: Name of the group. - - name: hash - title: Hash - group: 2 - description: 'The hash fields represent different bitwise hash algorithms and - their values. - - Field names for common hashes (e.g. MD5, SHA1) are predefined. Add fields for - other hashes by lowercasing the hash algorithm name and using underscore separators - as appropriate (snake case, e.g. sha3_512). - - Note that this fieldset is used for common hashes that may be computed over - a range of generic bytes. Entity-specific hashes such as ja3 or imphash are - placed in the fieldsets to which they relate (tls and pe, respectively).' - type: group - default_field: true - fields: - - name: md5 - level: extended - type: keyword - ignore_above: 1024 - description: MD5 hash. - - name: sha1 - level: extended - type: keyword - ignore_above: 1024 - description: SHA1 hash. - - name: sha256 - level: extended - type: keyword - ignore_above: 1024 - description: SHA256 hash. - - name: sha384 - level: extended - type: keyword - ignore_above: 1024 - description: SHA384 hash. - default_field: false - - name: sha512 - level: extended - type: keyword - ignore_above: 1024 - description: SHA512 hash. - - name: ssdeep - level: extended - type: keyword - ignore_above: 1024 - description: SSDEEP hash. - default_field: false - - name: tlsh - level: extended - type: keyword - ignore_above: 1024 - description: TLSH hash. - default_field: false - name: host title: Host group: 2 @@ -4279,40 +3773,6 @@ ignore_above: 1024 description: HTTP version. example: 1.1 - - name: interface - title: Interface - group: 2 - description: The interface fields are used to record ingress and egress interface - information when reported by an observer (e.g. firewall, router, load balancer) - in the context of the observer handling a network connection. In the case of - a single observer interface (e.g. network sensor on a span port) only the observer.ingress - information should be populated. - type: group - default_field: true - fields: - - name: alias - level: extended - type: keyword - ignore_above: 1024 - description: Interface alias as reported by the system, typically used in firewall - implementations for e.g. inside, outside, or dmz logical interface naming. - example: outside - default_field: false - - name: id - level: extended - type: keyword - ignore_above: 1024 - description: Interface ID as reported by an observer (typically SNMP interface - ID). - example: 10 - default_field: false - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Interface name as reported by the system. - example: eth0 - default_field: false - name: log title: Log group: 2 @@ -4477,128 +3937,6 @@ for RFC 5424 messages. example: 1 default_field: false - - name: macho - title: Mach-O Header - group: 2 - description: These fields contain Mac OS Mach Object file format (Mach-O) metadata. - type: group - default_field: true - fields: - - name: go_import_hash - level: extended - type: keyword - ignore_above: 1024 - description: 'A hash of the Go language imports in a Mach-O file excluding standard - library imports. An import hash can be used to fingerprint binaries even after - recompilation or other code-level transformations have occurred, which would - change more traditional hash values. - - The algorithm used to calculate the Go symbol hash and a reference implementation - are available [here](https://github.com/elastic/toutoumomoma).' - example: 10bddcb4cee42080f76c88d9ff964491 - default_field: false - - name: go_imports - level: extended - type: flattened - description: List of imported Go language element names and types. - default_field: false - - name: go_imports_names_entropy - level: extended - type: long - format: number - description: Shannon entropy calculation from the list of Go imports. - default_field: false - - name: go_imports_names_var_entropy - level: extended - type: long - format: number - description: Variance for Shannon entropy calculation from the list of Go imports. - default_field: false - - name: go_stripped - level: extended - type: boolean - description: Set to true if the file is a Go executable that has had its symbols - stripped or obfuscated and false if an unobfuscated Go executable. - default_field: false - - name: import_hash - level: extended - type: keyword - ignore_above: 1024 - description: 'A hash of the imports in a Mach-O file. An import hash can be - used to fingerprint binaries even after recompilation or other code-level - transformations have occurred, which would change more traditional hash values. - - This is a synonym for symhash.' - example: d41d8cd98f00b204e9800998ecf8427e - default_field: false - - name: imports - level: extended - type: flattened - description: List of imported element names and types. - default_field: false - - name: imports_names_entropy - level: extended - type: long - format: number - description: Shannon entropy calculation from the list of imported element names - and types. - default_field: false - - name: imports_names_var_entropy - level: extended - type: long - format: number - description: Variance for Shannon entropy calculation from the list of imported - element names and types. - default_field: false - - name: sections - level: extended - type: nested - description: 'An array containing an object for each section of the Mach-O file. - - The keys that should be present in these objects are defined by sub-fields - underneath `macho.sections.*`.' - default_field: false - - name: sections.entropy - level: extended - type: long - format: number - description: Shannon entropy calculation from the section. - default_field: false - - name: sections.name - level: extended - type: keyword - ignore_above: 1024 - description: Mach-O Section List name. - default_field: false - - name: sections.physical_size - level: extended - type: long - format: bytes - description: Mach-O Section List physical size. - default_field: false - - name: sections.var_entropy - level: extended - type: long - format: number - description: Variance for Shannon entropy calculation from the section. - default_field: false - - name: sections.virtual_size - level: extended - type: long - format: string - description: Mach-O Section List virtual size. This is always the same as `physical_size`. - default_field: false - - name: symhash - level: extended - type: keyword - ignore_above: 1024 - description: 'A hash of the imports in a Mach-O file. An import hash can be - used to fingerprint binaries even after recompilation or other code-level - transformations have occurred, which would change more traditional hash values. - - This is a Mach-O implementation of the Windows PE imphash' - example: d3ccf195b62a9279c3c19af1080497ec - default_field: false - name: network title: Network group: 2 @@ -5212,69 +4550,6 @@ type: match_only_text default_field: false description: Organization name. - - name: os - title: Operating System - group: 2 - description: The OS fields contain information about the operating system. - type: group - default_field: true - fields: - - name: family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: full - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: match_only_text - default_field: false - description: Operating system name, including the version or code name. - example: Mac OS Mojave - - name: kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: match_only_text - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: type - level: extended - type: keyword - ignore_above: 1024 - description: 'Use the `os.type` field to categorize the operating system into - one of the broad commercial families. - - If the OS you''re dealing with is not listed as an expected value, the field - should not be populated. Please let us know by opening an issue with ECS, - to propose its addition.' - example: macos - default_field: false - - name: version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - name: package title: Package group: 2 @@ -5336,219 +4611,44 @@ level: extended type: keyword ignore_above: 1024 - description: Package name - example: go - - name: path - level: extended - type: keyword - ignore_above: 1024 - description: Path where the package is installed. - example: /usr/local/Cellar/go/1.12.9/ - - name: reference - level: extended - type: keyword - ignore_above: 1024 - description: Home page or reference URL of the software in this package, if - available. - example: https://golang.org - default_field: false - - name: size - level: extended - type: long - format: string - description: Package size in bytes. - example: 62231 - - name: type - level: extended - type: keyword - ignore_above: 1024 - description: 'Type of package. - - This should contain the package file type, rather than the package manager - name. Examples: rpm, dpkg, brew, npm, gem, nupkg, jar.' - example: rpm - default_field: false - - name: version - level: extended - type: keyword - ignore_above: 1024 - description: Package version - example: 1.12.9 - - name: pe - title: PE Header - group: 2 - description: These fields contain Windows Portable Executable (PE) metadata. - type: group - default_field: true - fields: - - name: architecture - level: extended - type: keyword - ignore_above: 1024 - description: CPU architecture target for the file. - example: x64 - default_field: false - - name: company - level: extended - type: keyword - ignore_above: 1024 - description: Internal company name of the file, provided at compile-time. - example: Microsoft Corporation - default_field: false - - name: description - level: extended - type: keyword - ignore_above: 1024 - description: Internal description of the file, provided at compile-time. - example: Paint - default_field: false - - name: file_version - level: extended - type: keyword - ignore_above: 1024 - description: Internal version of the file, provided at compile-time. - example: 6.3.9600.17415 - default_field: false - - name: go_import_hash - level: extended - type: keyword - ignore_above: 1024 - description: 'A hash of the Go language imports in a PE file excluding standard - library imports. An import hash can be used to fingerprint binaries even after - recompilation or other code-level transformations have occurred, which would - change more traditional hash values. - - The algorithm used to calculate the Go symbol hash and a reference implementation - are available [here](https://github.com/elastic/toutoumomoma).' - example: 10bddcb4cee42080f76c88d9ff964491 - default_field: false - - name: go_imports - level: extended - type: flattened - description: List of imported Go language element names and types. - default_field: false - - name: go_imports_names_entropy - level: extended - type: long - format: number - description: Shannon entropy calculation from the list of Go imports. - default_field: false - - name: go_imports_names_var_entropy - level: extended - type: long - format: number - description: Variance for Shannon entropy calculation from the list of Go imports. - default_field: false - - name: go_stripped - level: extended - type: boolean - description: Set to true if the file is a Go executable that has had its symbols - stripped or obfuscated and false if an unobfuscated Go executable. - default_field: false - - name: imphash - level: extended - type: keyword - ignore_above: 1024 - description: 'A hash of the imports in a PE file. An imphash -- or import hash - -- can be used to fingerprint binaries even after recompilation or other code-level - transformations have occurred, which would change more traditional hash values. - - Learn more at https://www.fireeye.com/blog/threat-research/2014/01/tracking-malware-import-hashing.html.' - example: 0c6803c4e922103c4dca5963aad36ddf - default_field: false - - name: import_hash - level: extended - type: keyword - ignore_above: 1024 - description: 'A hash of the imports in a PE file. An import hash can be used - to fingerprint binaries even after recompilation or other code-level transformations - have occurred, which would change more traditional hash values. - - This is a synonym for imphash.' - example: d41d8cd98f00b204e9800998ecf8427e - default_field: false - - name: imports - level: extended - type: flattened - description: List of imported element names and types. - default_field: false - - name: imports_names_entropy - level: extended - type: long - format: number - description: Shannon entropy calculation from the list of imported element names - and types. - default_field: false - - name: imports_names_var_entropy - level: extended - type: long - format: number - description: Variance for Shannon entropy calculation from the list of imported - element names and types. - default_field: false - - name: original_file_name - level: extended - type: keyword - ignore_above: 1024 - description: Internal name of the file, provided at compile-time. - example: MSPAINT.EXE - default_field: false - - name: pehash - level: extended - type: keyword - ignore_above: 1024 - description: 'A hash of the PE header and data from one or more PE sections. - An pehash can be used to cluster files by transforming structural information - about a file into a hash value. - - Learn more at https://www.usenix.org/legacy/events/leet09/tech/full_papers/wicherski/wicherski_html/index.html.' - example: 73ff189b63cd6be375a7ff25179a38d347651975 - default_field: false - - name: product - level: extended - type: keyword - ignore_above: 1024 - description: Internal product name of the file, provided at compile-time. - example: "Microsoft\xAE Windows\xAE Operating System" - default_field: false - - name: sections - level: extended - type: nested - description: 'An array containing an object for each section of the PE file. - - The keys that should be present in these objects are defined by sub-fields - underneath `pe.sections.*`.' - default_field: false - - name: sections.entropy - level: extended - type: long - format: number - description: Shannon entropy calculation from the section. - default_field: false - - name: sections.name - level: extended - type: keyword - ignore_above: 1024 - description: PE Section List name. - default_field: false - - name: sections.physical_size + description: Package name + example: go + - name: path level: extended - type: long - format: bytes - description: PE Section List physical size. - default_field: false - - name: sections.var_entropy + type: keyword + ignore_above: 1024 + description: Path where the package is installed. + example: /usr/local/Cellar/go/1.12.9/ + - name: reference level: extended - type: long - format: number - description: Variance for Shannon entropy calculation from the section. + type: keyword + ignore_above: 1024 + description: Home page or reference URL of the software in this package, if + available. + example: https://golang.org default_field: false - - name: sections.virtual_size + - name: size level: extended type: long format: string - description: PE Section List virtual size. This is always the same as `physical_size`. + description: Package size in bytes. + example: 62231 + - name: type + level: extended + type: keyword + ignore_above: 1024 + description: 'Type of package. + + This should contain the package file type, rather than the package manager + name. Examples: rpm, dpkg, brew, npm, gem, nupkg, jar.' + example: rpm default_field: false + - name: version + level: extended + type: keyword + ignore_above: 1024 + description: Package version + example: 1.12.9 - name: process title: Process group: 2 @@ -8727,62 +7827,6 @@ ignore_above: 1024 description: All the user names or other user identifiers seen on the event. default_field: false - - name: risk - title: Risk information - group: 2 - description: Fields for describing risk score and risk level of entities such - as hosts and users. These fields are not allowed to be nested under `event.*`. - Please continue to use `event.risk_score` and `event.risk_score_norm` for event - risk. - type: group - default_field: true - fields: - - name: calculated_level - level: extended - type: keyword - ignore_above: 1024 - description: A risk classification level calculated by an internal system as - part of entity analytics and entity risk scoring. - example: High - default_field: false - - name: calculated_score - level: extended - type: float - description: A risk classification score calculated by an internal system as - part of entity analytics and entity risk scoring. - example: 880.73 - default_field: false - - name: calculated_score_norm - level: extended - type: float - description: A risk classification score calculated by an internal system as - part of entity analytics and entity risk scoring, and normalized to a range - of 0 to 100. - example: 88.73 - default_field: false - - name: static_level - level: extended - type: keyword - ignore_above: 1024 - description: A risk classification level obtained from outside the system, such - as from some external Threat Intelligence Platform. - example: High - default_field: false - - name: static_score - level: extended - type: float - description: A risk classification score obtained from outside the system, such - as from some external Threat Intelligence Platform. - example: 830.0 - default_field: false - - name: static_score_norm - level: extended - type: float - description: A risk classification score obtained from outside the system, such - as from some external Threat Intelligence Platform, and normalized to a range - of 0 to 100. - example: 83.0 - default_field: false - name: rule title: Rule group: 2 @@ -14351,42 +13395,6 @@ ignore_above: 1024 description: Version of the user agent. example: 12.0 - - name: vlan - title: VLAN - group: 2 - description: 'The VLAN fields are used to identify 802.1q tag(s) of a packet, - as well as ingress and egress VLAN associations of an observer in relation to - a specific packet or connection. - - Network.vlan fields are used to record a single VLAN tag, or the outer tag in - the case of q-in-q encapsulations, for a packet or connection as observed, typically - provided by a network sensor (e.g. Zeek, Wireshark) passively reporting on traffic. - - Network.inner VLAN fields are used to report inner q-in-q 802.1q tags (multiple - 802.1q encapsulations) as observed, typically provided by a network sensor (e.g. - Zeek, Wireshark) passively reporting on traffic. Network.inner VLAN fields should - only be used in addition to network.vlan fields to indicate q-in-q tagging. - - Observer.ingress and observer.egress VLAN values are used to record observer - specific information when observer events contain discrete ingress and egress - VLAN information, typically provided by firewalls, routers, or load balancers.' - type: group - default_field: true - fields: - - name: id - level: extended - type: keyword - ignore_above: 1024 - description: VLAN ID as reported by the observer. - example: 10 - default_field: false - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Optional VLAN name as reported by the observer. - example: outside - default_field: false - name: vulnerability title: Vulnerability group: 2 @@ -14516,193 +13524,3 @@ prioritization regarding remediation. For example (https://nvd.nist.gov/vuln-metrics/cvss) example: Critical default_field: false - - name: x509 - title: x509 Certificate - group: 2 - description: 'This implements the common core fields for x509 certificates. This - information is likely logged with TLS sessions, digital signatures found in - executable binaries, S/MIME information in email bodies, or analysis of files - on disk. - - When the certificate relates to a file, use the fields at `file.x509`. When - hashes of the DER-encoded certificate are available, the `hash` data set should - be populated as well (e.g. `file.hash.sha256`). - - Events that contain certificate information about network connections, should - use the x509 fields under the relevant TLS fields: `tls.server.x509` and/or - `tls.client.x509`.' - type: group - default_field: true - fields: - - name: alternative_names - level: extended - type: keyword - ignore_above: 1024 - description: List of subject alternative names (SAN). Name types vary by certificate - authority and certificate type but commonly contain IP addresses, DNS names - (and wildcards), and email addresses. - example: '*.elastic.co' - default_field: false - - name: issuer.common_name - level: extended - type: keyword - ignore_above: 1024 - description: List of common name (CN) of issuing certificate authority. - example: Example SHA2 High Assurance Server CA - default_field: false - - name: issuer.country - level: extended - type: keyword - ignore_above: 1024 - description: List of country \(C) codes - example: US - default_field: false - - name: issuer.distinguished_name - level: extended - type: keyword - ignore_above: 1024 - description: Distinguished name (DN) of issuing certificate authority. - example: C=US, O=Example Inc, OU=www.example.com, CN=Example SHA2 High Assurance - Server CA - default_field: false - - name: issuer.locality - level: extended - type: keyword - ignore_above: 1024 - description: List of locality names (L) - example: Mountain View - default_field: false - - name: issuer.organization - level: extended - type: keyword - ignore_above: 1024 - description: List of organizations (O) of issuing certificate authority. - example: Example Inc - default_field: false - - name: issuer.organizational_unit - level: extended - type: keyword - ignore_above: 1024 - description: List of organizational units (OU) of issuing certificate authority. - example: www.example.com - default_field: false - - name: issuer.state_or_province - level: extended - type: keyword - ignore_above: 1024 - description: List of state or province names (ST, S, or P) - example: California - default_field: false - - name: not_after - level: extended - type: date - description: Time at which the certificate is no longer considered valid. - example: '2020-07-16T03:15:39Z' - default_field: false - - name: not_before - level: extended - type: date - description: Time at which the certificate is first considered valid. - example: '2019-08-16T01:40:25Z' - default_field: false - - name: public_key_algorithm - level: extended - type: keyword - ignore_above: 1024 - description: Algorithm used to generate the public key. - example: RSA - default_field: false - - name: public_key_curve - level: extended - type: keyword - ignore_above: 1024 - description: The curve used by the elliptic curve public key algorithm. This - is algorithm specific. - example: nistp521 - default_field: false - - name: public_key_exponent - level: extended - type: long - description: Exponent used to derive the public key. This is algorithm specific. - example: 65537 - index: false - doc_values: false - default_field: false - - name: public_key_size - level: extended - type: long - description: The size of the public key space in bits. - example: 2048 - default_field: false - - name: serial_number - level: extended - type: keyword - ignore_above: 1024 - description: Unique serial number issued by the certificate authority. For consistency, - if this value is alphanumeric, it should be formatted without colons and uppercase - characters. - example: 55FBB9C7DEBF09809D12CCAA - default_field: false - - name: signature_algorithm - level: extended - type: keyword - ignore_above: 1024 - description: Identifier for certificate signature algorithm. We recommend using - names found in Go Lang Crypto library. See https://github.com/golang/go/blob/go1.14/src/crypto/x509/x509.go#L337-L353. - example: SHA256-RSA - default_field: false - - name: subject.common_name - level: extended - type: keyword - ignore_above: 1024 - description: List of common names (CN) of subject. - example: shared.global.example.net - default_field: false - - name: subject.country - level: extended - type: keyword - ignore_above: 1024 - description: List of country \(C) code - example: US - default_field: false - - name: subject.distinguished_name - level: extended - type: keyword - ignore_above: 1024 - description: Distinguished name (DN) of the certificate subject entity. - example: C=US, ST=California, L=San Francisco, O=Example, Inc., CN=shared.global.example.net - default_field: false - - name: subject.locality - level: extended - type: keyword - ignore_above: 1024 - description: List of locality names (L) - example: San Francisco - default_field: false - - name: subject.organization - level: extended - type: keyword - ignore_above: 1024 - description: List of organizations (O) of subject. - example: Example, Inc. - default_field: false - - name: subject.organizational_unit - level: extended - type: keyword - ignore_above: 1024 - description: List of organizational units (OU) of subject. - default_field: false - - name: subject.state_or_province - level: extended - type: keyword - ignore_above: 1024 - description: List of state or province names (ST, S, or P) - example: California - default_field: false - - name: version_number - level: extended - type: keyword - ignore_above: 1024 - description: Version of x509 format. - example: 3 - default_field: false From f6b5f64e63acf5141181c098c171abf2310291a0 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Wed, 20 Sep 2023 13:47:49 -0500 Subject: [PATCH 4/5] linting --- scripts/tests/test_ecs_helpers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/tests/test_ecs_helpers.py b/scripts/tests/test_ecs_helpers.py index 4949a22382..a1b2af4fc0 100644 --- a/scripts/tests/test_ecs_helpers.py +++ b/scripts/tests/test_ecs_helpers.py @@ -169,5 +169,6 @@ def test_remove_top_level_false_field_sets(self): } self.assertEqual(ecs_helpers.remove_top_level_reusable_false(nested_schema_original), nested_schema_expected) + if __name__ == '__main__': unittest.main() From 198579a50ecbff76583799a989104b0ce5a5d5ba Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Wed, 20 Sep 2023 14:56:30 -0500 Subject: [PATCH 5/5] changelog --- CHANGELOG.next.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.next.md b/CHANGELOG.next.md index a1517826ff..2fec6bc55b 100644 --- a/CHANGELOG.next.md +++ b/CHANGELOG.next.md @@ -26,6 +26,8 @@ Thanks, you're awesome :-) --> #### Bugfixes +* Respect reusable.top_level in Beats generator #2278 + #### Added #### Improvements