From d9b91a811d28b75f4b58acc6aa872e37e1a7d4d5 Mon Sep 17 00:00:00 2001
From: Zach Christensen <23529374+ZachChristensen28@users.noreply.github.com>
Date: Mon, 19 Sep 2022 22:37:20 -0600
Subject: [PATCH 1/3] Fix for #18 (#19)
---
docs/configure/cleanup.md | 35 +++++++++++++++++++
docs/configure/index.md | 1 +
docs/reference/all-configurations.md | 2 ++
mkdocs.yml | 1 +
.../default/collections.conf | 1 +
src/SA-CrowdstrikeDevices/default/macros.conf | 4 +++
.../default/savedsearches.conf | 18 ++++++++--
.../default/transforms.conf | 2 +-
8 files changed, 61 insertions(+), 3 deletions(-)
create mode 100644 docs/configure/cleanup.md
diff --git a/docs/configure/cleanup.md b/docs/configure/cleanup.md
new file mode 100644
index 0000000..32466d3
--- /dev/null
+++ b/docs/configure/cleanup.md
@@ -0,0 +1,35 @@
+# Update Cleanup
+
+The saved search `Crowdstrike Devices Lookup - Cleanup` runs every hour 29 minutes after the hour to remove old/stale device data from the kvstore. By default, it will remove any device that has not reported in longer than 2 days.
+
+???+ note
+ Even though a device may be removed, it will be re-added by the saved search `Crowdstrike Devices Lookup - Gen` if it begins to send data again.
+
+## Update Search Macro
+
+To change the retention period from the default 2 days, there is a search macro that will need to be updated.
+
+1. Navigate to Settings > Advanced Search > Search Macros.
+1. Set the "App" to `SA-CrowdstrikeDeviecs`.
+1. Set the "Owner" to `Any`.
+1. Click on `sa_crowdstrike_retention` to modify the definition.
+1. Set the definition to a valid [time modifier](https://docs.splunk.com/Documentation/Splunk/9.0.1/SearchReference/SearchTimeModifiers#How_to_specify_relative_time_modifiers).
+
+???+ important
+ __Make sure to keep the quotes around the definition.__
+
+ i.e.
+
+ "-7d@d"
+
+## Update Search Schedule
+
+It may also be necessary to update how often the cleanup search runs (default: hourly).
+
+To update the default schedule perform the following steps:
+
+1. Navigate to Settings > Searches, reports, and alerts.
+1. Set the "App" dropdown to `SA-CrowdstrikeDevices`.
+1. Set the "Owner" dropdown to `All`.
+1. Click "Edit" under actions for the search `Crowdstrike Devices Lookup - Cleanup`
+1. Click "Edit Schedule" and update the schedule and necessary.
diff --git a/docs/configure/index.md b/docs/configure/index.md
index 9c8477f..6976896 100644
--- a/docs/configure/index.md
+++ b/docs/configure/index.md
@@ -8,3 +8,4 @@ Each field can be customized to fit your environment. The following fields shoul
- [Update Category](./category)
- [Update Business Unit](./bunit)
- [Update Schedule](./schedule.md)
+- [Update Cleanup](./cleanup.md)
diff --git a/docs/reference/all-configurations.md b/docs/reference/all-configurations.md
index d079d0e..508aee8 100644
--- a/docs/reference/all-configurations.md
+++ b/docs/reference/all-configurations.md
@@ -9,9 +9,11 @@ Below is a table that list all configuration for this add-on.
Name | Type | Web Location | CLI Location\* | Description
---- | ---- | ------------ | ------------- | -----------
Crowdstrike Devices Lookup - Gen | Saved Search | Settings > Searches reports, and alerts | savedsearches.conf | Populates the lookup file `crowdstrike_devices`.
+Crowdstrike Devices Lookup - Cleanup | Saved Search | Settings > Searches reports, and alerts | savedsearches.conf | removes old entries from kvstore lookup: `crowdstrike_devices`.
crowdstrike_devices | lookup | Settings > Lookups > Lookup definitions | transforms.conf | Lookup definition for the KVStore collection `crowdstrike_devices_collection`.
crowdstrike_devices_collection | KVStore collection | n/a\*\* | collections.conf | KVStore configuration.
sa_crowdstrike_index | Search macro | Settings > Advanced Search > Search Macros | macros.conf | Index definition for the crowdstrike index that contains the sourcetype `crowdstrike:device:json`.
+sa_crowdstrike_retention | Search macro | Settings> Advanced Search > Search Macros | macros.conf | The amount of time for the device not being updated before it is removed from the lookup. `default "-2d"`
identity_manager://crowdstrike_devices | Asset lookup configuration | Enterprise Security > Configure > Data Enrichment > Asset and Identity Management > Asset Lookups | inputs.conf | Asset configuration lookup to load Crowdstrike devices into the asset database.
> \*CLI locations are relative to `SA-SandflyDevices/default`. Any update to CLI configuration files should be done in the local directory.
diff --git a/mkdocs.yml b/mkdocs.yml
index 88c313f..d8817ff 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -79,6 +79,7 @@ nav:
- Update Category: configure/category.md
- Update Business Unit: configure/bunit.md
- Update Schedule: configure/schedule.md
+ - Update Cleanup: configure/cleanup.md
- Best Practice:
- Clone Saved Search: configure/best-practice/clone-search.md
- Reference:
diff --git a/src/SA-CrowdstrikeDevices/default/collections.conf b/src/SA-CrowdstrikeDevices/default/collections.conf
index 8548d81..3111f16 100644
--- a/src/SA-CrowdstrikeDevices/default/collections.conf
+++ b/src/SA-CrowdstrikeDevices/default/collections.conf
@@ -16,4 +16,5 @@ field.long = string
field.mac = string
field.nt_host = string
field.priority = string
+field._last_seen = time
replicate = true
diff --git a/src/SA-CrowdstrikeDevices/default/macros.conf b/src/SA-CrowdstrikeDevices/default/macros.conf
index c7e308c..4aca9c7 100644
--- a/src/SA-CrowdstrikeDevices/default/macros.conf
+++ b/src/SA-CrowdstrikeDevices/default/macros.conf
@@ -6,3 +6,7 @@
[sa_crowdstrike_index]
definition = index=crowdstrike
iseval = false
+
+[sa_crowdstrike_retention]
+definition = "-2d"
+iseval = false
diff --git a/src/SA-CrowdstrikeDevices/default/savedsearches.conf b/src/SA-CrowdstrikeDevices/default/savedsearches.conf
index 707e8ab..8670689 100644
--- a/src/SA-CrowdstrikeDevices/default/savedsearches.conf
+++ b/src/SA-CrowdstrikeDevices/default/savedsearches.conf
@@ -54,7 +54,21 @@ search = `sa_crowdstrike_index` sourcetype="crowdstrike:device:json" \
| eval \
mac=mvjoin(mac, "|"),\
ip=mvjoin(ip, "|"),\
- dns=mvjoin(dns, "|")\
-| table _key,ip,mac,nt_host,dns,bunit,priority,lat,long,city,country,category,is_expected \
+ dns=mvjoin(dns, "|"),\
+ _last_seen=now()\
+| table _key,_last_seen,ip,mac,nt_host,dns,bunit,priority,lat,long,city,country,category,is_expected \
| outputlookup key_field=_key crowdstrike_devices \
| stats count
+
+[Crowdstrike Devices Lookup - Cleanup]
+disabled = false
+cron_schedule = 29 * * * *
+description = removes old entries from kvstore lookup: crowdstrike_devices
+dispatch.earliest_time = -1s
+dispatch.latest_time = now
+enableSched = 1
+schedule_window = auto
+search = | inputlookup crowdstrike_devices \
+| where _last_seen>relative_time(now(), `sa_crowdstrike_retention`) \
+| outputlookup crowdstrike_devices \
+| stats count
diff --git a/src/SA-CrowdstrikeDevices/default/transforms.conf b/src/SA-CrowdstrikeDevices/default/transforms.conf
index 72ba95b..67293e1 100644
--- a/src/SA-CrowdstrikeDevices/default/transforms.conf
+++ b/src/SA-CrowdstrikeDevices/default/transforms.conf
@@ -8,4 +8,4 @@ external_type = kvstore
collection = crowdstrike_devices_collection
max_matches = 1
case_sensitive_match = false
-fields_list = _key,bunit,category,city,country,dns,ip,is_expected,lat,long,mac,nt_host,priority
+fields_list = _key,_last_seen,bunit,category,city,country,dns,ip,is_expected,lat,long,mac,nt_host,priority
From 1df9030799d01d03520d251cb97aba1208d5ba8f Mon Sep 17 00:00:00 2001
From: Zach Christensen <23529374+ZachChristensen28@users.noreply.github.com>
Date: Mon, 19 Sep 2022 22:46:38 -0600
Subject: [PATCH 2/3] version increase
---
README.md | 14 ++++++-------
docs/index.md | 2 +-
docs/releases/index.md | 13 +++++-------
docs/releases/release-history.md | 23 ++++++++++++++++++++++
src/SA-CrowdstrikeDevices/app.manifest | 2 +-
src/SA-CrowdstrikeDevices/default/app.conf | 2 +-
6 files changed, 37 insertions(+), 19 deletions(-)
diff --git a/README.md b/README.md
index 4c0b8fd..816077a 100644
--- a/README.md
+++ b/README.md
@@ -26,23 +26,21 @@ Full documentation can be found at [https://splunk-sa-crowdstrike.ztsplunker.com
Info | Description
------|----------
-SA-CrowdstrikeDevices | 1.0.2 - [Splunkbase](https://splunkbase.splunk.com/app/6573/) \| [GitHub](https://github.com/ZachChristensen28/SA-CrowdstrikeDevices)
+SA-CrowdstrikeDevices | 1.0.3 - [Splunkbase](https://splunkbase.splunk.com/app/6573/) \| [GitHub](https://github.com/ZachChristensen28/SA-CrowdstrikeDevices)
Splunk Enterprise Security Version (Required) | [7.x \| 6.x](https://splunkbase.splunk.com/app/263)
Crowdstrike Devices Add-on (Required) | [3.x](https://splunkbase.splunk.com/app/5570)
Add-on has a web UI | No, this add-on does not contain views.
```text
-Version 1.0.2
+Version 1.0.3
New
-- added `first_seen`, `last_seen`, and `last_updated` to category field (#8).
-- added `site_name` to existing `bunit` field (#13).
+- added cleanup search to remove old/stale devices (#18).
+- added search macro for device retention period (#18).
Updated
-- Changed app logo background to transparent.
-
-Fixed
-- Updated saved search to preserve hosts with multiple IP/MAC addresses (#11).
+- updated collection to include last seen field (#18).
+- updated lookup generating search to include last time seen (#18).
```
## Issues or Feature Requests
diff --git a/docs/index.md b/docs/index.md
index f9726e5..a1e1cc9 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -29,7 +29,7 @@ This documentation assumes the following:
Info | Description
------|----------
-SA-CrowdstrikeDevices | 1.0.2 - [Splunkbase](https://splunkbase.splunk.com/app/6573) \| [GitHub](https://github.com/ZachChristensen28/SA-CrowdstrikeDevices)
+SA-CrowdstrikeDevices | 1.0.3 - [Splunkbase](https://splunkbase.splunk.com/app/6573) \| [GitHub](https://github.com/ZachChristensen28/SA-CrowdstrikeDevices)
Splunk Enterprise Security Version (Required) | [7.x \| 6.x](https://splunkbase.splunk.com/app/263)
Crowdstrike Devices Add-on (Required) | [3.x](https://splunkbase.splunk.com/app/5570)
Add-on has a web UI | No, this add-on does not contain views.
diff --git a/docs/releases/index.md b/docs/releases/index.md
index ee953e7..ed99b15 100644
--- a/docs/releases/index.md
+++ b/docs/releases/index.md
@@ -1,6 +1,6 @@
# Release notes
-## v1.0.2 placeholder
+## v1.0.3 September 20, 2022
### Compatibility
@@ -12,16 +12,13 @@ Crowdstrike Device Add-on Version | [3.x](https://splunkbase.splunk.com/app/5570
### New
-- added `first_seen`, `last_seen`, and `last_updated` to category field ([#8](https://github.com/ZachChristensen28/SA-CrowdstrikeDevices/issues/8)).
-- added `site_name` to existing `bunit` field ([#13](https://github.com/ZachChristensen28/SA-CrowdstrikeDevices/issues/13)).
+- added cleanup search to remove old/stale devices ([#18](https://github.com/ZachChristensen28/SA-CrowdstrikeDevices/issues/18)).
+- added search macro for device retention period ([#18](https://github.com/ZachChristensen28/SA-CrowdstrikeDevices/issues/18)).
### Updated
-- Changed app logo background to transparent.
-
-### Fixed
-
-- Updated saved search to preserve hosts with multiple IP/MAC addresses ([#11](https://github.com/ZachChristensen28/SA-CrowdstrikeDevices/issues/11)).
+- updated collection to include last seen field ([#18](https://github.com/ZachChristensen28/SA-CrowdstrikeDevices/issues/18)).
+- updated lookup generating search to include last time seen ([#18](https://github.com/ZachChristensen28/SA-CrowdstrikeDevices/issues/18)).
## Known issues
diff --git a/docs/releases/release-history.md b/docs/releases/release-history.md
index 5b8d9fd..8d80b5d 100644
--- a/docs/releases/release-history.md
+++ b/docs/releases/release-history.md
@@ -1,5 +1,28 @@
# Release history
+## v1.0.2 September 8,2022
+
+### Compatibility
+
+Product | Version
+--------- | -------
+Splunk platform versions | 9.x, 8.x
+Splunk Enterprise Security version | [7.x, 6.x](https://splunkbase.splunk.com/app/263)
+Crowdstrike Device Add-on Version | [3.x](https://splunkbase.splunk.com/app/5570)
+
+### New
+
+- added `first_seen`, `last_seen`, and `last_updated` to category field ([#8](https://github.com/ZachChristensen28/SA-CrowdstrikeDevices/issues/8)).
+- added `site_name` to existing `bunit` field ([#13](https://github.com/ZachChristensen28/SA-CrowdstrikeDevices/issues/13)).
+
+### Updated
+
+- Changed app logo background to transparent.
+
+### Fixed
+
+- Updated saved search to preserve hosts with multiple IP/MAC addresses ([#11](https://github.com/ZachChristensen28/SA-CrowdstrikeDevices/issues/11)).
+
## v1.0.1 August 25, 2022
- Hotfix for missing `_key` field in saved search.
diff --git a/src/SA-CrowdstrikeDevices/app.manifest b/src/SA-CrowdstrikeDevices/app.manifest
index 4248ed9..fc9cd74 100644
--- a/src/SA-CrowdstrikeDevices/app.manifest
+++ b/src/SA-CrowdstrikeDevices/app.manifest
@@ -5,7 +5,7 @@
"id": {
"group": null,
"name": "SA-CrowdstrikeDevices",
- "version": "1.0.2"
+ "version": "1.0.3"
},
"author": [
{
diff --git a/src/SA-CrowdstrikeDevices/default/app.conf b/src/SA-CrowdstrikeDevices/default/app.conf
index 2dd23a9..98eaf52 100644
--- a/src/SA-CrowdstrikeDevices/default/app.conf
+++ b/src/SA-CrowdstrikeDevices/default/app.conf
@@ -12,7 +12,7 @@ build = 2
[launcher]
author = ZachTheSplunker
description = This supporting add-on allows device information pulled from Crowdstrike to be used with Splunk Enterprise Security's Asset Database.
-version = 1.0.2
+version = 1.0.3
[ui]
is_visible = 0
From 3836d432c22cf40452a9b37703f575451f6fe848 Mon Sep 17 00:00:00 2001
From: Zach Christensen <23529374+ZachChristensen28@users.noreply.github.com>
Date: Mon, 19 Sep 2022 22:52:23 -0600
Subject: [PATCH 3/3] fixing typo
---
docs/releases/index.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docs/releases/index.md b/docs/releases/index.md
index 77f3662..c94a602 100644
--- a/docs/releases/index.md
+++ b/docs/releases/index.md
@@ -1,6 +1,6 @@
# Release notes
-## [v1.0.3 September 20, 2022]((https://github.com/ZachChristensen28/SA-CrowdstrikeDevices/releases/tag/v1.0.3))
+## [v1.0.3 September 20, 2022](https://github.com/ZachChristensen28/SA-CrowdstrikeDevices/releases/tag/v1.0.3)
### Compatibility