From 725e3092c3d235798e2ca23064e53e6ac5a44f7d Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Wed, 3 Jun 2020 20:32:14 -0400 Subject: [PATCH] Add Okta module documentation, config cleanup, _id field (#18953) This add documentation for the Okta module. It contains descriptions of the configuration options and general information about the module. I fixed an issue with the module where it was not setting the _id field for Elasticsearch events. I also did some cleanup to the pipeline.js (indentation, semi-colons, strict equality checks). The module's manifest was updated to not duplicate httpjson's default values. The module was accepting configuration as JSON strings for some parameters (http_headers, http_request_body, pagination, rate_limit, ssl) which is inconsistent with how other parts of Beats are configured so I removed this. Now these options expect regular YAML objects for values. None of these options are required to use the module so the impact to users should be minimal. --- CHANGELOG.next.asciidoc | 2 + filebeat/docs/modules/okta.asciidoc | 96 ++++++++++++++++++- x-pack/filebeat/filebeat.reference.yml | 15 +-- x-pack/filebeat/module/okta/_meta/config.yml | 15 +-- .../filebeat/module/okta/_meta/docs.asciidoc | 96 ++++++++++++++++++- .../module/okta/system/config/input.yml | 41 ++++++-- .../module/okta/system/config/pipeline.js | 55 ++++++----- .../filebeat/module/okta/system/manifest.yml | 41 +++----- x-pack/filebeat/modules.d/okta.yml.disabled | 15 +-- 9 files changed, 279 insertions(+), 97 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index e8191fa04f73..49c0755caaa9 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -42,6 +42,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d * iptables {pull}18756[18756] * Checkpoint {pull}18754[18754] - Preserve case of http.request.method. ECS prior to 1.6 specified normalizing to lowercase, which lost information. Affects filesets: apache/access, elasticsearch/audit, iis/access, iis/error, nginx/access, nginx/ingress_controller, aws/elb, suricata/eve, zeek/http. {issue}18154[18154] {pull}18359[18359] +- Okta module now requires objects instead of JSON strings for the `http_headers`, `http_request_body`, `pagination`, `rate_limit`, and `ssl` variables. {pull}18953[18953] *Heartbeat* @@ -164,6 +165,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix `googlecloud.audit` pipeline to only take in fields that are explicitly defined by the dataset. {issue}18465[18465] {pull}18472[18472] - Fix `o365.audit` failing to ingest events when ip address is surrounded by square brackets. {issue}18587[18587] {pull}18591[18591] - Fix Kubernetes Watcher goroutine leaks when input config is invalid and `input.reload` is enabled. {issue}18629[18629] {pull}18630[18630] +- Okta module now sets the Elasticsearch `_id` field to the Okta UUID value contained in each system log to minimize the possibility of duplicating events. {pull}18953[18953] *Heartbeat* diff --git a/filebeat/docs/modules/okta.asciidoc b/filebeat/docs/modules/okta.asciidoc index 8c81f6c9c5f5..038f6d088dd0 100644 --- a/filebeat/docs/modules/okta.asciidoc +++ b/filebeat/docs/modules/okta.asciidoc @@ -12,15 +12,105 @@ This file is generated! See scripts/docs_collector.py beta[] -This is a filebeat module for retrieving system logs from Okta (www.okta.com) via API. +The Okta module collects events from the +https://developer.okta.com/docs/reference/[Okta API]. Specifically this supports +reading from the https://developer.okta.com/docs/reference/api/system-log/[Okta +System Log API]. -:has-dashboards!: +:fileset_ex: system + +include::../include/config-option-intro.asciidoc[] + +[float] +==== `system` fileset settings + +The Okta System Log records system events related to your organization in order +to provide an audit trail that can be used to understand platform activity and +to diagnose problems. This module is implemented using the +<> input and is configured to paginate through +the logs while honoring any +https://developer.okta.com/docs/reference/rate-limits/[rate-limiting] headers +sent by Okta. + +NOTE: This module does not persist the timestamp of the last read event in +order to facilitate resuming on restart. This feature will be coming in a future +version. When you restart the module will read events from the beginning of the +log. To minimize duplicates documents the module uses the event's Okta UUID +value as the Elasticsearch `_id`. + +This is an example configuration for the module. + +[source,yaml] +---- +- module okta + system: + var.url: https://yourOktaDomain/api/v1/logs + var.api_key: '00QCjAl4MlV-WPXM...0HmjFx-vbGua' +---- + +[float] +===== Configuration options + +*`var.url`*:: + +Specifies the URL to the Okta System Log API. Required. ++ +[source,yaml] +---- + var.url: https://mycompany.okta.com/api/v1/logs +---- + +*`var.api_key`*:: + +Specifies the Okta API token to use in requests to the API. Required. +The token is used in an HTTP `Authorization` header with the `SSWS` scheme. +See https://developer.okta.com/docs/guides/create-an-api-token/create-the-token/[ +Create an API token] for information on how to obtain a token. ++ +[source,yaml] +---- + var.api_key: '00QCjAl4MlV-WPXM...0HmjFx-vbGua' +---- -This module comes with a sample dashboard. For example: +*`var.http_client_timeout`*:: + +Duration of the time limit on HTTP requests made by the module. Defaults to +`60s`. + +*`var.interval`*:: + +Duration between requests to the API. Defaults to `60s`. + +*`var.keep_original_message`*:: + +Boolean flag indicating if the original JSON event string should be included in +the `event.original` field. Defaults to `true`. + +*`var.ssl`*:: + +Configuration options for SSL parameters like the certificate authority to use +for HTTPS-based connections. If the `ssl` section is missing, the host CAs are +used for HTTPS connections to Okta. See <> for more +information. ++ +[source,yaml] +---- + var.ssl: + supported_protocols: [TLSv1.2] +---- + +[float] +=== Example dashboard + +This module comes with a sample dashboard: [role="screenshot"] image::./images/filebeat-okta-dashboard.png[] +:has-dashboards!: + +:fileset_ex!: + :modulename!: diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 2f59e5ad6066..ebe415b688c1 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -860,17 +860,10 @@ filebeat.modules: - module: okta system: enabled: true - # API key to access Okta - #var.api_key - - # URL of the Okta REST API - #var.url - - # Disable SSL verification - #var.ssl: |- - # { - # "verification_mode": "none" - # } + # You must configure the URL with your Okta domain and provide an + # API token to access the logs API. + #var.url: https://yourOktaDomain/api/v1/logs + #var.api_key: 'yourApiTokenHere' #------------------------------- Osquery Module ------------------------------- - module: osquery diff --git a/x-pack/filebeat/module/okta/_meta/config.yml b/x-pack/filebeat/module/okta/_meta/config.yml index 31853e0130dd..bb2da13eca4f 100644 --- a/x-pack/filebeat/module/okta/_meta/config.yml +++ b/x-pack/filebeat/module/okta/_meta/config.yml @@ -1,14 +1,7 @@ - module: okta system: enabled: true - # API key to access Okta - #var.api_key - - # URL of the Okta REST API - #var.url - - # Disable SSL verification - #var.ssl: |- - # { - # "verification_mode": "none" - # } + # You must configure the URL with your Okta domain and provide an + # API token to access the logs API. + #var.url: https://yourOktaDomain/api/v1/logs + #var.api_key: 'yourApiTokenHere' diff --git a/x-pack/filebeat/module/okta/_meta/docs.asciidoc b/x-pack/filebeat/module/okta/_meta/docs.asciidoc index 9c6b91d66462..1ea5cc6a66d9 100644 --- a/x-pack/filebeat/module/okta/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/okta/_meta/docs.asciidoc @@ -7,13 +7,103 @@ beta[] -This is a filebeat module for retrieving system logs from Okta (www.okta.com) via API. +The Okta module collects events from the +https://developer.okta.com/docs/reference/[Okta API]. Specifically this supports +reading from the https://developer.okta.com/docs/reference/api/system-log/[Okta +System Log API]. -:has-dashboards!: +:fileset_ex: system + +include::../include/config-option-intro.asciidoc[] + +[float] +==== `system` fileset settings + +The Okta System Log records system events related to your organization in order +to provide an audit trail that can be used to understand platform activity and +to diagnose problems. This module is implemented using the +<> input and is configured to paginate through +the logs while honoring any +https://developer.okta.com/docs/reference/rate-limits/[rate-limiting] headers +sent by Okta. + +NOTE: This module does not persist the timestamp of the last read event in +order to facilitate resuming on restart. This feature will be coming in a future +version. When you restart the module will read events from the beginning of the +log. To minimize duplicates documents the module uses the event's Okta UUID +value as the Elasticsearch `_id`. + +This is an example configuration for the module. + +[source,yaml] +---- +- module okta + system: + var.url: https://yourOktaDomain/api/v1/logs + var.api_key: '00QCjAl4MlV-WPXM...0HmjFx-vbGua' +---- + +[float] +===== Configuration options + +*`var.url`*:: + +Specifies the URL to the Okta System Log API. Required. ++ +[source,yaml] +---- + var.url: https://mycompany.okta.com/api/v1/logs +---- + +*`var.api_key`*:: + +Specifies the Okta API token to use in requests to the API. Required. +The token is used in an HTTP `Authorization` header with the `SSWS` scheme. +See https://developer.okta.com/docs/guides/create-an-api-token/create-the-token/[ +Create an API token] for information on how to obtain a token. ++ +[source,yaml] +---- + var.api_key: '00QCjAl4MlV-WPXM...0HmjFx-vbGua' +---- -This module comes with a sample dashboard. For example: +*`var.http_client_timeout`*:: + +Duration of the time limit on HTTP requests made by the module. Defaults to +`60s`. + +*`var.interval`*:: + +Duration between requests to the API. Defaults to `60s`. + +*`var.keep_original_message`*:: + +Boolean flag indicating if the original JSON event string should be included in +the `event.original` field. Defaults to `true`. + +*`var.ssl`*:: + +Configuration options for SSL parameters like the certificate authority to use +for HTTPS-based connections. If the `ssl` section is missing, the host CAs are +used for HTTPS connections to Okta. See <> for more +information. ++ +[source,yaml] +---- + var.ssl: + supported_protocols: [TLSv1.2] +---- + +[float] +=== Example dashboard + +This module comes with a sample dashboard: [role="screenshot"] image::./images/filebeat-okta-dashboard.png[] +:has-dashboards!: + +:fileset_ex!: + :modulename!: diff --git a/x-pack/filebeat/module/okta/system/config/input.yml b/x-pack/filebeat/module/okta/system/config/input.yml index cf646175059a..a544ab8dc656 100644 --- a/x-pack/filebeat/module/okta/system/config/input.yml +++ b/x-pack/filebeat/module/okta/system/config/input.yml @@ -1,19 +1,48 @@ {{ if eq .input "httpjson" }} type: httpjson + +{{ if .api_key }} api_key: {{ .api_key }} -authentication_scheme: {{.authentication_scheme}} +{{ end }} + +authentication_scheme: {{ .authentication_scheme }} + +{{ if .http_client_timeout }} http_client_timeout: {{ .http_client_timeout }} +{{ end }} + +{{ if .http_method }} http_method: {{ .http_method }} -http_headers: {{ .http_headers }} +{{ end }} + +{{ if .http_headers }} +http_headers: {{ .http_headers | tojson }} +{{ end }} + +{{ if .http_request_body }} http_request_body: {{ .http_request_body }} -no_http_body: {{ .no_http_body }} +{{ end }} + interval: {{ .interval }} + +{{ if .json_objects_array }} json_objects_array: {{ .json_objects_array }} -pagination: {{ .pagination }} -rate_limit: {{ .rate_limit }} +{{ end }} + +no_http_body: {{ .no_http_body }} + +pagination: {{ .pagination | tojson }} + +rate_limit: {{ .rate_limit | tojson }} + +{{ if .ssl }} +ssl: {{ .ssl | tojson }} +{{ end }} + +{{ if .url }} url: {{ .url }} -ssl: {{ .ssl }} +{{ end }} {{ else if eq .input "file" }} diff --git a/x-pack/filebeat/module/okta/system/config/pipeline.js b/x-pack/filebeat/module/okta/system/config/pipeline.js index 396650259c52..0d381b0944d6 100644 --- a/x-pack/filebeat/module/okta/system/config/pipeline.js +++ b/x-pack/filebeat/module/okta/system/config/pipeline.js @@ -10,6 +10,13 @@ function OktaSystem(keep_original_message) { target: "json", }); + var setId = function(evt) { + var oktaUuid = evt.Get("json.uuid"); + if (oktaUuid) { + evt.Put("@metadata._id", oktaUuid); + } + }; + var parseTimestamp = new processor.Timestamp({ field: "json.published", timezone: "UTC", @@ -101,58 +108,59 @@ function OktaSystem(keep_original_message) { { from: "okta.security_context.domain", to: "client.domain" }, { from: "okta.security_context.domain", to: "source.domain" }, { from: "okta.uuid", to: "event.id" }, - { from: "okta.uuid", to: "_id" }, ], ignore_missing: true, fail_on_error: false, }); var setEventOutcome = function(evt) { - var outcome = evt.Get("okta.outcome.result") - if (outcome != null) { - var o = outcome.toLowerCase(); - if (o == "success" || o == "allow") { + var outcome = evt.Get("okta.outcome.result"); + if (outcome) { + outcome = outcome.toLowerCase(); + if (outcome === "success" || outcome === "allow") { evt.Put("event.outcome", "success"); - } else if (o == "failure" || o == "deny") { + } else if (outcome === "failure" || outcome === "deny") { evt.Put("event.outcome", "failure"); } else { evt.Put("event.outcome", "unknown"); } } - } - - // Update nested fields + }; + + // Update nested fields var renameNestedFields = function(evt) { var arr = evt.Get("okta.target"); - if (arr != null) { + if (arr) { for (var i = 0; i < arr.length; i++) { arr[i].alternate_id = arr[i].alternateId; arr[i].display_name = arr[i].displayName; delete arr[i].alternateId; delete arr[i].displayName; delete arr[i].detailEntry; - } + } } }; // Set user info if actor type is User var setUserInfo = function(evt) { if (evt.Get("okta.actor.type") === "User") { - evt.Put("client.user.full_name", evt.Get("okta.actor.display_name")); - evt.Put("source.user.full_name", evt.Get("okta.actor.display_name")); - evt.Put("related.user", evt.Get("okta.actor.display_name")); - evt.Put("client.user.id", evt.Get("okta.actor.id")); - evt.Put("source.user.id", evt.Get("okta.actor.id")); + evt.Put("client.user.full_name", evt.Get("okta.actor.display_name")); + evt.Put("source.user.full_name", evt.Get("okta.actor.display_name")); + evt.Put("related.user", evt.Get("okta.actor.display_name")); + evt.Put("client.user.id", evt.Get("okta.actor.id")); + evt.Put("source.user.id", evt.Get("okta.actor.id")); } }; // Set related.ip field var setRelatedIP = function(event) { - if (event.Get("source.ip") != null) { - event.AppendTo("related.ip", event.Get("source.ip")); + var ip = event.Get("source.ip"); + if (ip) { + event.AppendTo("related.ip", ip); } - if (event.Get("destination.ip") != null) { - event.AppendTo("related.ip", event.Get("destination.ip")); + ip = event.Get("destination.ip"); + if (ip) { + event.AppendTo("related.ip", ip); } }; @@ -166,15 +174,16 @@ function OktaSystem(keep_original_message) { function dropNull(obj) { Object.keys(obj).forEach(function(key) { (obj[key] && typeof obj[key] === 'object') && dropNull(obj[key]) || - (obj[key] === null) && delete obj[key] + (obj[key] === null) && delete obj[key]; }); return obj; - }; + } dropNull(evt); }; var pipeline = new processor.Chain() .Add(decodeJson) + .Add(setId) .Add(parseTimestamp) .Add(saveOriginalMessage) .Add(dropOriginalMessage) @@ -192,7 +201,7 @@ function OktaSystem(keep_original_message) { return { process: pipeline.Run, }; -}; +} var oktaSystem; diff --git a/x-pack/filebeat/module/okta/system/manifest.yml b/x-pack/filebeat/module/okta/system/manifest.yml index b5dc38bc55ce..1f3722113b2a 100644 --- a/x-pack/filebeat/module/okta/system/manifest.yml +++ b/x-pack/filebeat/module/okta/system/manifest.yml @@ -4,50 +4,33 @@ var: - name: input default: httpjson - name: api_key - default: "" - name: authentication_scheme default: "SSWS" - name: http_client_timeout - default: 60 - name: http_method - default: GET - name: http_headers - default: |- - {} - name: http_request_body - default: |- - {} - - name: no_http_body - default: true - name: interval - default: 60 + default: 60s - name: json_objects_array - default: "" - name: keep_original_message default: true + - name: no_http_body + default: true - name: pagination - default: |- - { - "enabled": true, - "header": { - "field_name": "Link", - "regex_pattern": "<([^>]+)>; *rel=\"next\"(?:,|$)" - }, - } + default: + header: + field_name: Link + regex_pattern: '<([^>]+)>; *rel="next"(?:,|$)' - name: rate_limit - default: |- - { - "limit": "X-Rate-Limit-Limit", - "remaining": "X-Rate-Limit-Remaining", - "reset": "X-Rate-Limit-Reset" - } - - name: url - default: "" + default: + limit: X-Rate-Limit-Limit + remaining: X-Rate-Limit-Remaining + reset: X-Rate-Limit-Reset - name: ssl - default: |- - {} - name: tags default: [forwarded] + - name: url input: config/input.yml ingest_pipeline: ingest/pipeline.yml diff --git a/x-pack/filebeat/modules.d/okta.yml.disabled b/x-pack/filebeat/modules.d/okta.yml.disabled index 19e2a1ad8f2b..66965ac4ba25 100644 --- a/x-pack/filebeat/modules.d/okta.yml.disabled +++ b/x-pack/filebeat/modules.d/okta.yml.disabled @@ -4,14 +4,7 @@ - module: okta system: enabled: true - # API key to access Okta - #var.api_key - - # URL of the Okta REST API - #var.url - - # Disable SSL verification - #var.ssl: |- - # { - # "verification_mode": "none" - # } + # You must configure the URL with your Okta domain and provide an + # API token to access the logs API. + #var.url: https://yourOktaDomain/api/v1/logs + #var.api_key: 'yourApiTokenHere'