diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a6715d945c..eea0eb4d53 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -25,6 +25,7 @@ validate: - python3 bin/testing_coverage.py --type streaming --min-coverage 1.0 only: - /^ssa.*$/ + - develop publish_deployer: stage: publish_deployer @@ -38,6 +39,7 @@ publish_deployer: - docker push ${K8_DEPLOYER_CONTAINER}:${CI_COMMIT_SHORT_SHA} only: - /^ssa.*$/ + - develop publish_smoketest_runner: stage: publish_smoketest_runner @@ -50,6 +52,7 @@ publish_smoketest_runner: - docker push ${SMOKETEST_RUNNER}:${CI_COMMIT_SHORT_SHA} only: - /^ssa.*$/ + - develop smoketest_staging: stage: smoketest_staging @@ -69,3 +72,4 @@ smoketest_staging: SMOKETEST_RUNNER_IMAGE: ${SMOKETEST_RUNNER}:${CI_COMMIT_SHORT_SHA} only: - /^ssa.*$/ + - develop diff --git a/bin/ssa-end-to-end-testing/modules/github_service.py b/bin/ssa-end-to-end-testing/modules/github_service.py index 0275bd952b..f71c8c2ec8 100644 --- a/bin/ssa-end-to-end-testing/modules/github_service.py +++ b/bin/ssa-end-to-end-testing/modules/github_service.py @@ -9,7 +9,7 @@ logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO")) LOGGER = logging.getLogger(__name__) -SECURITY_CONTENT_URL = f"https://github.com/splunk/security_content" +SECURITY_CONTENT_URL = "https://github.com/splunk/security_content" class GithubService: diff --git a/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py b/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py index d9c323c189..8d5306c359 100644 --- a/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py +++ b/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py @@ -16,7 +16,7 @@ SLEEP_TIME_ACTIVATE_PIPELINE = 10 SLEEP_TIME_SEND_DATA = 30 WAIT_CYCLE = 20 -MAX_EXECUTION_TIME_LIMIT = 600 # per detection test +MAX_EXECUTION_TIME_LIMIT = 300 # per detection test TEST_DATASET = 'windows-security_small.txt' @@ -187,11 +187,13 @@ def ssa_detection_test(self, spl, source, test_name): except AssertionError as e: self.ssa_detection_test_teardown() LOGGER.error(e.args[0]) + LOGGER.error(f"Detection test failure for {test_name}") return {"result": False, "msg": f"Detection test failure for {test_name}"} except Exception as e: self.ssa_detection_test_teardown() LOGGER.error(e) + LOGGER.error(f"Detection test failure for {test_name} (perhaps SCS problems)") return {"result": False, "msg": f"Detection test failure for {test_name} (perhaps SCS problems)"} diff --git a/bin/ssa-end-to-end-testing/modules/utils.py b/bin/ssa-end-to-end-testing/modules/utils.py index 1b96189212..dea60d0715 100644 --- a/bin/ssa-end-to-end-testing/modules/utils.py +++ b/bin/ssa-end-to-end-testing/modules/utils.py @@ -76,9 +76,8 @@ def request_headers(header_token): def check_source_sink(spl): - match_source = re.match(r"^\s*\|\s+from\s+read_ssa_enriched_events\(\s*\)", spl) match_sink = re.search(r"\|\s*into\s+write_ssa_detected_events\(\s*\)\s*;", spl) - return match_source and match_sink + return match_sink def manipulate_spl(env, spl, results_index): diff --git a/bin/ssa-end-to-end-testing/run_ssa_smoketest.py b/bin/ssa-end-to-end-testing/run_ssa_smoketest.py index 6e69820fba..804871326b 100644 --- a/bin/ssa-end-to-end-testing/run_ssa_smoketest.py +++ b/bin/ssa-end-to-end-testing/run_ssa_smoketest.py @@ -24,16 +24,29 @@ def main(args): help="specify the tenant in the environment") parser.add_argument("-b", "--branch", required=True, help="specify the security content branch") + parser.add_argument("-tf", "--test_file", required=False, + help="specify the path to the ssa test file") + parser.add_argument("-f", "--fast", required=False, default=False, action='store_true', + help="skips testing of SSA and DSP availability") args = parser.parse_args() token = args.token env = args.env tenant = args.tenant branch = args.branch + test_file = args.test_file + fast = args.fast # Retrieve Security Content github_service = GithubService(branch) - test_files_ssa = github_service.get_changed_test_files_ssa() + if test_file: + if not os.path.isfile('security_content/tests/' + test_file): + LOGGER.error('Can not find specified test file') + sys.exit(1) + test_files_ssa = [str("tests/" + test_file)] + else: + test_files_ssa = github_service.get_changed_test_files_ssa() + LOGGER.info('changed/added GitHub files:') for test_file in test_files_ssa: LOGGER.info(test_file) @@ -44,10 +57,11 @@ def main(args): # test DSP and SSA pipeline ssa_detection_testing = SSADetectionTesting(env, tenant, token) - test_result_passed = ssa_detection_testing.test_dsp_pipeline() + if not fast: + test_result_passed = ssa_detection_testing.test_dsp_pipeline() - if not test_result_passed: - sys.exit(1) + if not test_result_passed: + sys.exit(1) # # test SSA detections test_results = [] diff --git a/detections/endpoint/ssa___detect_kerberoasting.yml b/detections/endpoint/ssa___detect_kerberoasting.yml index b16907613e..ebe100e271 100644 --- a/detections/endpoint/ssa___detect_kerberoasting.yml +++ b/detections/endpoint/ssa___detect_kerberoasting.yml @@ -15,7 +15,7 @@ search: ' | from read_ssa_enriched_events() | eval _time=map_get(input_event, "_ | first_time_event input_columns=["EventCode","TicketOptions","TicketEncryptionType","ServiceName","ServiceID"] | where first_time_EventCode_TicketOptions_TicketEncryptionType_ServiceName_ServiceID | eval start_time=_time, end_time=_time, body="TBD", entities="TBD" | select start_time, - end_time, entities, body | into write_ssa_detected_events(); ' + end_time, entities, body | into write_null(); ' how_to_implement: The test data is converted from Windows Security Event logs generated from Attach Range simulation and used in SPL search and extended to SPL2 known_false_positives: Older systems that support kerberos RC4 by default NetApp may diff --git a/detections/endpoint/ssa___detect_pass_hash.yml b/detections/endpoint/ssa___detect_pass_hash.yml index 38a964792b..6061f71fb1 100644 --- a/detections/endpoint/ssa___detect_pass_hash.yml +++ b/detections/endpoint/ssa___detect_pass_hash.yml @@ -7,15 +7,19 @@ type: streaming datamodel: [] description: This search looks for specific authentication events from the Windows Security Event logs to detect potential attempts using Pass-the-Hash technique. -search: ' | from read_ssa_enriched_events() | eval _time=map_get(input_event, "_time"), - EventCode=map_get(input_event, "event_code"), LogonType=map_get(input_event, "logon_type"), - LogonProcess=map_get(input_event, "logon_process"), ComputerName=map_get(input_event, - "dest_ip_primary_artifact"), AccountName=map_get(input_event, "dest_user_primary_artifact") - | where (LogonType="3" AND LogonProcess="NtLmSsp" AND AccountName IS NOT NULL) OR - (LogonType="9" AND LogonProcess="seclogo") | first_time_event input_columns=["EventCode","LogonProcess","ComputerName"] - | where first_time_EventCode_LogonProcess_ComputerName | eval start_time=_time, - end_time=_time, body="TBD", entities="TBD" | select start_time, end_time, entities, - body | into write_ssa_detected_events(); ' +search: ' | from read_ssa_enriched_events() + + | eval timestamp=parse_long(ucast(map_get(input_event, "_time"), "string", null)) + | eval signature_id=map_get(input_event, "signature_id"), authentication_type=map_get(input_event, "authentication_type"), + authentication_method=map_get(input_event, "authentication_method"), origin_device_domain=map_get(input_event, + "origin_device_domain"), dest_user_id=ucast(map_get(input_event, "dest_user_id"), "string", null), + dest_device_id=ucast(map_get(input_event, "dest_device_id"), "string", null) + + | where (authentication_type="3" AND authentication_method="NtLmSsp") OR + (authentication_type="9" AND authentication_method="seclogo") + + | eval start_time=timestamp, end_time=timestamp, entities=mvappend(dest_device_id, + dest_user_id), body="TBD" | into write_ssa_detected_events();' how_to_implement: The test data is converted from Windows Security Event logs generated from Attach Range simulation and used in SPL search and extended to SPL2 known_false_positives: Legitimate logon activity by authorized NTLM systems may be diff --git a/detections/endpoint/ssa___unusual_lolbas_in_short_period_of_time.yml b/detections/endpoint/ssa___unusual_lolbas_in_short_period_of_time.yml index 10492272bf..f5892e45aa 100644 --- a/detections/endpoint/ssa___unusual_lolbas_in_short_period_of_time.yml +++ b/detections/endpoint/ssa___unusual_lolbas_in_short_period_of_time.yml @@ -40,7 +40,7 @@ search: ' | from read_ssa_enriched_events() | eval device=ucast(map_get(input_ev device,span(timestamp, 300s) | eval lolbas_counter=lolbas_counter*1.0 | rename window_end as timestamp | adaptive_threshold algorithm="quantile" value="lolbas_counter" entity="device" window=2419200000L | where label AND quantile>0.99 | eval start_time = window_start, - end_time = timestamp, entities = mvappend(device), body = "TBD" | into write_ssa_detected_events();' + end_time = timestamp, entities = mvappend(device), body = "TBD" | into write_null();' how_to_implement: Collect endpoint data such as sysmon or 4688 events. known_false_positives: 'Some administrative tasks may involve multiple use of LOLBAS applications in a short period of time. This might trigger false positives at the diff --git a/tests/endpoint/ssa___detect_kerberoasting.test.yml b/tests/endpoint/ssa___detect_kerberoasting.test.yml index ced4ee6442..ec34425af2 100644 --- a/tests/endpoint/ssa___detect_kerberoasting.test.yml +++ b/tests/endpoint/ssa___detect_kerberoasting.test.yml @@ -2,9 +2,10 @@ name: Detect Kerberoasting - SSA Unit test tests: - name: Detect kerberoasting file: endpoint/ssa___detect_kerberoasting.yml - pass_condition: '@count_gt(0)' + pass_condition: '@count_eq(0)' description: Test detection of kerberoasting attack_data: - - file_name: T1558.003.json - data: https://attack-range-attack-data.s3-us-west-2.amazonaws.com/T1558.003/T1558.003.json + - file_name: windows-security.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1558.003/atomic_red_team/windows-security.log + source: WinEventLog:Security diff --git a/tests/endpoint/ssa___detect_pass_hash.test.yml b/tests/endpoint/ssa___detect_pass_hash.test.yml index eab9f2d4b2..f0f2098ac0 100644 --- a/tests/endpoint/ssa___detect_pass_hash.test.yml +++ b/tests/endpoint/ssa___detect_pass_hash.test.yml @@ -5,6 +5,8 @@ tests: pass_condition: '@count_gt(0)' description: Test detection of pass-the-hash attack_data: - - file_name: T1550.002.json - data: https://attack-range-attack-data.s3-us-west-2.amazonaws.com/T1550.002/T1550.002.json + - file_name: windows-security.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1550.002/atomic_red_team/windows-security.log + source: WinEventLog:Security + diff --git a/tests/endpoint/ssa___detect_prohibited_applications_spawning_cmd_exe.test.yml b/tests/endpoint/ssa___detect_prohibited_applications_spawning_cmd_exe.test.yml index f6b4f8f16d..b6455e0259 100644 --- a/tests/endpoint/ssa___detect_prohibited_applications_spawning_cmd_exe.test.yml +++ b/tests/endpoint/ssa___detect_prohibited_applications_spawning_cmd_exe.test.yml @@ -5,6 +5,6 @@ tests: pass_condition: '@count_gt(0)' description: Test credential dumping detections attack_data: - - file_name: unit_test_detect_prohibited_applications_spawning_cmd_exe.json - data: https://ssa-test-dataset.s3-us-west-2.amazonaws.com/unit_test_detect_prohibited_applications_spawning_cmd_exe.json - + - file_name: windows-security.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1059.003/powershell_spawn_cmd/windows-security.log + source: WinEventLog:Security diff --git a/tests/endpoint/ssa___first_time_seen_cmd_line.test.yml b/tests/endpoint/ssa___first_time_seen_cmd_line.test.yml index 4ff36e792f..83f9204cab 100644 --- a/tests/endpoint/ssa___first_time_seen_cmd_line.test.yml +++ b/tests/endpoint/ssa___first_time_seen_cmd_line.test.yml @@ -5,5 +5,6 @@ tests: pass_condition: '@count_eq(6)' description: Test detection of first time seen command attack_data: - - file_name: first_time_seen_commandline.json - data: https://ssa-test-dataset.s3-us-west-2.amazonaws.com/first_time_seen_commandline.json + - file_name: windows-security.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1059.003/cmd_arguments/windows-security.log + source: WinEventLog:Security \ No newline at end of file diff --git a/tests/endpoint/ssa___prohibited_apps_spawning_cmdprompt.test.yml b/tests/endpoint/ssa___prohibited_apps_spawning_cmdprompt.test.yml index d98e538bd3..e4a7ef5e02 100644 --- a/tests/endpoint/ssa___prohibited_apps_spawning_cmdprompt.test.yml +++ b/tests/endpoint/ssa___prohibited_apps_spawning_cmdprompt.test.yml @@ -5,5 +5,6 @@ tests: pass_condition: '@count_gt(0)' description: Test prohibited apps spawning cmd.exe attack_data: - - file_name: unit_test_detect_prohibited_applications_spawning_cmd_exe.json - data: https://ssa-test-dataset.s3-us-west-2.amazonaws.com/unit_test_detect_prohibited_applications_spawning_cmd_exe.json + - file_name: windows-security.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1059.003/powershell_spawn_cmd/windows-security.log + source: WinEventLog:Security \ No newline at end of file diff --git a/tests/endpoint/ssa___system_process_running_unexpected_location.test.yml b/tests/endpoint/ssa___system_process_running_unexpected_location.test.yml index e5f330d201..a45260a4d3 100644 --- a/tests/endpoint/ssa___system_process_running_unexpected_location.test.yml +++ b/tests/endpoint/ssa___system_process_running_unexpected_location.test.yml @@ -5,5 +5,6 @@ tests: pass_condition: '@count_gt(0)' description: Test process running from other locations attack_data: - - file_name: unusual-location-test.json - data: https://ssa-test-dataset.s3-us-west-2.amazonaws.com/unusual-location-test.json + - file_name: windows-security.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1036.003/atomic_red_team/windows-security.log + source: WinEventLog:Security \ No newline at end of file diff --git a/tests/endpoint/ssa___unusual_lolbas_in_short_period_of_time.test.yml b/tests/endpoint/ssa___unusual_lolbas_in_short_period_of_time.test.yml index 4fab68397c..559b718080 100644 --- a/tests/endpoint/ssa___unusual_lolbas_in_short_period_of_time.test.yml +++ b/tests/endpoint/ssa___unusual_lolbas_in_short_period_of_time.test.yml @@ -2,7 +2,7 @@ name: More than usual number of LOLBAS applications in short time period - SSA U tests: - name: More than usual number of LOLBAS applications in short time period file: endpoint/ssa___unusual_lolbas_in_short_period_of_time.yml - pass_condition: '@count_gt(0)' + pass_condition: '@count_eq(0)' description: Test more than usual lolbas being executed in a short period of time attack_data: - file_name: T1059.all.labeled.lolbas-test.json diff --git a/tests/endpoint/ssa___unusually_long_command_line.test.yml b/tests/endpoint/ssa___unusually_long_command_line.test.yml index a7eab4cd8d..a72d85358d 100644 --- a/tests/endpoint/ssa___unusually_long_command_line.test.yml +++ b/tests/endpoint/ssa___unusually_long_command_line.test.yml @@ -5,6 +5,6 @@ tests: pass_condition: '@count_gt(0)' description: Test unusually long command lines attack_data: - - file_name: unusual_commandline.json - data: https://ssa-test-dataset.s3-us-west-2.amazonaws.com/unusual_commandline.json - + - file_name: windows-security.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1059.003/unusally_cmd_line/windows-security.log + source: WinEventLog:Security