Merge pull request #1299 from splunk/ssa_smoketest_nightly

Ssa smoketest nightly

bin/ssa-end-to-end-testing/k8s-deployer/k8s/components/smoketest.jsonnet

@@ -13,7 +13,7 @@ local job = {
         },
     },
     spec: {
-        activeDeadlineSeconds: 1800,
+        activeDeadlineSeconds: 36000,
         ttlSecondsAfterFinished: 100,
         template: {
             metadata: {

bin/ssa-end-to-end-testing/modules/github_service.py

@@ -2,6 +2,7 @@
 import git
 import os
 import logging
+import glob
 
 
 # Logger
@@ -27,28 +28,37 @@ def get_changed_test_files_ssa(self):
         branch1 = self.security_content_branch
         branch2 = 'develop'
         g = git.Git('security_content')
-        differ = g.diff('--name-only', branch1, branch2)
-        changed_files = differ.splitlines()
-
         changed_ssa_test_files = []
 
-        for file_path in changed_files:
-            # added or changed test files
-            if file_path.startswith('tests'):
-                if os.path.basename(file_path).startswith('ssa'):
-                    if file_path not in changed_ssa_test_files:
-                        changed_ssa_test_files.append(file_path)
+        if branch1 != 'develop':
+            differ = g.diff('--name-only', branch1, branch2)
+            changed_files = differ.splitlines()
+
+            for file_path in changed_files:
+                # added or changed test files
+                if file_path.startswith('tests'):
+                    if os.path.basename(file_path).startswith('ssa'):
+                        if file_path not in changed_ssa_test_files:
+                            changed_ssa_test_files.append(file_path)
 
-            # changed detections
-            if file_path.startswith('detections'):
+                # changed detections
+                if file_path.startswith('detections'):
+                    if os.path.basename(file_path).startswith('ssa'):
+                        file_path_base = os.path.splitext(file_path)[0].replace('detections', 'tests') + '.test'
+                        file_path_new = file_path_base + '.yml'
+                        if file_path_new not in changed_ssa_test_files:
+                            changed_ssa_test_files.append(file_path_new)
+
+        # all SSA test files for nightly build
+        else:
+            changed_files = sorted(glob.glob('security_content/tests/*/*.yml'))
+
+            for file_path in changed_files:
+                file_path = file_path.replace('security_content/','')
                 if os.path.basename(file_path).startswith('ssa'):
-                    file_path_base = os.path.splitext(file_path)[0].replace('detections', 'tests') + '.test'
-                    file_path_new = file_path_base + '.yml'
-                    if not os.path.exists(file_path_new):
-                        file_path_new = file_path_base + '.yaml'
-                    if file_path_new not in changed_ssa_test_files:
-                        changed_ssa_test_files.append(file_path_new)
+                    changed_ssa_test_files.append(file_path)
 
         return changed_ssa_test_files
 
 
+

bin/ssa-end-to-end-testing/modules/utils.py

@@ -120,17 +120,19 @@ def read_data(file_name):
     date_rex = r'\d{2}/\d{2}/\d{4} \d{2}:\d{2}:\d{2} [AP]M'
     count = len(open(modified_file).readlines())
     i = 0
+    tmp_counter = 0
     for line in fileinput.input(files=modified_file):
         i = i + 1
         if event != "" and re.match(date_rex, line):
             data.append(event)
+            tmp_counter = 0
             event = line
         else:
+            tmp_counter = tmp_counter + 1
             event = event + line
 
-        if i == count:
-            if len(data) == 0:
-                data.append(event)
+        if i == count and tmp_counter > 10:
+            data.append(event)
 
     return data
 

detections/endpoint/access_lsass_memory_for_dump_creation.yml

@@ -49,4 +49,4 @@ tags:
   - TargetProcessId
   - SourceImage
   - SourceProcessId
-  security_domain: endpoint
+  security_domain: endpoint

detections/endpoint/ssa___illegal_service_and_process_control_via_mimikatz_modules.yml

@@ -33,7 +33,7 @@ tags:
   - CIS 16
   - CIS 20
   dataset:
-  - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1569/logAllMimikatzModules.log
+  - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1003/credential_extraction/logAllMimikatzModules.log
   kill_chain_phases:
   - Actions on Objectives
   mitre_attack_id:

detections/endpoint/ssa___illegal_service_and_process_control_via_powersploit_modules.yml

@@ -42,7 +42,7 @@ tags:
   - CIS 16
   - CIS 20
   dataset:
-  - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1569/logAllPowerSploitModulesWithOldNames.log
+  - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1003/credential_extraction/logAllPowerSploitModulesWithOldNames.log
   kill_chain_phases:
   - Actions on Objectives
   mitre_attack_id:

detections/endpoint/ssa___recon_privilege_escalation_opportunities_via_powersploit_modules.yml

@@ -46,3 +46,4 @@ tags:
   - dest_user_id
   risk_severity: high
   security_domain: endpoint
+

tests/endpoint/ssa___applying_stolen_credentials_via_mimikatz_modules.test.yml

@@ -8,6 +8,4 @@ tests:
       - file_name: logAllMimikatzModules.log
         data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1003/credential_extraction/logAllMimikatzModules.log
         source: WinEventLog:Security
-        sourcetype: xmlwineventlog
-        update_timestamp: True
 

tests/endpoint/ssa___applying_stolen_credentials_via_powersploit_modules.test.yml

@@ -8,5 +8,3 @@ tests:
       - file_name: logAllPowerSploitModulesWithOldNames.log
         data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1555/applying_stolen_credentials/logAllPowerSploitModulesWithOldNames.log
         source: WinEventLog:Security
-        sourcetype: xmlwineventlog
-        update_timestamp: True

tests/endpoint/ssa___assess_credential_strength_via_dsinternals_modules.test.yml

@@ -8,5 +8,3 @@ tests:
       - file_name: logAllDSInternalsModules.log
         data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1003/credential_extraction/logAllDSInternalsModules.log
         source: WinEventLog:Security
-        sourcetype: xmlwineventlog
-        update_timestamp: True

tests/endpoint/ssa___attempted_credential_dump_from_registry_via_reg_exe.test.yml

@@ -8,6 +8,4 @@ tests:
       - file_name: windows-security.log
         data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1003.002/atomic_red_team/windows-security.log
         source: WinEventLog:Security
-        sourcetype: xmlwineventlog
-        update_timestamp: True
 

tests/endpoint/ssa___credential_extraction_dsinternals_conversion_modules.test.yml

@@ -8,6 +8,4 @@ tests:
       - file_name: logAllDSInternalsModules.log
         data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1003/credential_extraction/logAllDSInternalsModules.log
         source: WinEventLog:Security
-        sourcetype: xmlwineventlog
-        update_timestamp: True
 

tests/endpoint/ssa___credential_extraction_dsinternals_modules.test.yml

@@ -8,6 +8,4 @@ tests:
       - file_name: logAllDSInternalsModules.log
         data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1003/credential_extraction/logAllDSInternalsModules.log
         source: WinEventLog:Security
-        sourcetype: xmlwineventlog
-        update_timestamp: True
 

tests/endpoint/ssa___credential_extraction_fgdump_cachedump_s_option.test.yml

@@ -8,6 +8,4 @@ tests:
       - file_name: logFgdump.log
         data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1003/credential_extraction/logFgdump.log
         source: WinEventLog:Security
-        sourcetype: xmlwineventlog
-        update_timestamp: True
 

tests/endpoint/ssa___credential_extraction_fgdump_cachedump_v_option.test.yml

@@ -6,4 +6,5 @@ tests:
     description: Test credential extraction detections
     attack_data:
       - file_name: logFgdump.log
-        data: https://ssa-test-dataset.s3-us-west-2.amazonaws.com/logFgdump.log
+        data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1003/credential_extraction/logFgdump.log
+        source: WinEventLog:Security

tests/endpoint/ssa___credential_extraction_getaddbaccount_from_dump.test.yml

@@ -6,5 +6,5 @@ tests:
     description: Test credential extraction detections
     attack_data:
       - file_name: logPowerShellModule.log
-        data: https://ssa-test-dataset.s3-us-west-2.amazonaws.com/logPowerShellModule.log
-
+        data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1003/credential_extraction/logPowerShellModule.log
+        source: WinEventLog:Security

tests/endpoint/ssa___credential_extraction_lazagne_command_options.test.yml

@@ -6,5 +6,5 @@ tests:
     description: Test credential extraction detections
     attack_data:
       - file_name: logLazagneCredDump.log
-        data: https://ssa-test-dataset.s3-us-west-2.amazonaws.com/logLazagneCredDump.log
-
+        data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1003/credential_extraction/logLazagneCredDump.log
+        source: WinEventLog:Security

tests/endpoint/ssa___credential_extraction_mimikatz_modules.test.yml

@@ -6,5 +6,6 @@ tests:
     description: Test credential extraction detections
     attack_data:
       - file_name: logAllMimikatzModules.log
-        data: https://ssa-test-dataset.s3-us-west-2.amazonaws.com/logAllMimikatzModules.log
+        data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1003/credential_extraction/logAllMimikatzModules.log
+        source: WinEventLog:Security
 

tests/endpoint/ssa___credential_extraction_ms_debuggers_kernel_peek.test.yml

@@ -6,5 +6,5 @@ tests:
     description: Test credential extraction detections
     attack_data:
       - file_name: logLiveKDFullKernelDump.log
-        data: https://ssa-test-dataset.s3-us-west-2.amazonaws.com/logLiveKDFullKernelDump.log
-
+        data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1003/credential_extraction/logLiveKDFullKernelDump.log
+        source: WinEventLog:Security

tests/endpoint/ssa___credential_extraction_ms_debuggers_z_option.test.yml

@@ -6,5 +6,5 @@ tests:
     description: Test credential extraction detections
     attack_data:
       - file_name: logLiveKDFullKernelDump.log
-        data: https://ssa-test-dataset.s3-us-west-2.amazonaws.com/logLiveKDFullKernelDump.log
-
+        data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1003/credential_extraction/logLiveKDFullKernelDump.log
+        source: WinEventLog:Security

tests/endpoint/ssa___credential_extraction_powersploit_modules.test.yml

@@ -6,5 +6,5 @@ tests:
     description: Test credential extraction detections
     attack_data:
       - file_name: logAllPowerSploitModulesWithOldNames.log
-        data: https://ssa-test-dataset.s3-us-west-2.amazonaws.com/logAllPowerSploitModulesWithOldNames.log
-
+        data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1003/credential_extraction/logAllPowerSploitModulesWithOldNames.log
+        source: WinEventLog:Security

tests/endpoint/ssa___detect_dump_lsass_memory_using_comsvcs.test.yml

@@ -5,5 +5,6 @@ tests:
     pass_condition: '@count_gt(0)'
     description: Test credential dumping detections
     attack_data:
-      - file_name: windows-security-events_ssa.log
-        data: https://attack-range-attack-data.s3-us-west-2.amazonaws.com/T1003.001/windows-security-events_ssa.log
+      - file_name: windows-security.log
+        data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1003.001/atomic_red_team/windows-security.log
+        source: WinEventLog:Security

tests/endpoint/ssa___illegal_access_user_content_via_powersploit_modules.test.yml

@@ -6,5 +6,5 @@ tests:
     description: Test illegal access to user content detections
     attack_data:
       - file_name: logAllPowerSploitModulesWithOldNames.log
-        data: https://ssa-test-dataset.s3-us-west-2.amazonaws.com/logAllPowerSploitModulesWithOldNames.log
-
+        data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1021/illegal_access_to_content/logAllPowerSploitModulesWithOldNames.log
+        source: WinEventLog:Security

tests/endpoint/ssa___illegal_account_creation_via_powersploit_modules.test.yml

@@ -6,5 +6,6 @@ tests:
     description: Test illegal account creation detections
     attack_data:
       - file_name: logAllPowerSploitModulesWithOldNames.log
-        data: https://ssa-test-dataset.s3-us-west-2.amazonaws.com/logAllPowerSploitModulesWithOldNames.log
+        data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1021/illegal_access_to_content/logAllPowerSploitModulesWithOldNames.log
+        source: WinEventLog:Security
 

tests/endpoint/ssa___illegal_account_enable_disable_via_dsinternals_modules.test.yml

@@ -6,5 +6,5 @@ tests:
     description: Test enabling or disabling of accounts detections
     attack_data:
       - file_name: logAllDSInternalsModules.log
-        data: https://ssa-test-dataset.s3-us-west-2.amazonaws.com/logAllDSInternalsModules.log
-
+        data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1098/logAllDSInternalsModules.log
+        source: WinEventLog:Security

tests/endpoint/ssa___illegal_log_deletion_via_mimikatz_modules.test.yml

@@ -6,5 +6,5 @@ tests:
     description: Test illegal log deletion detections
     attack_data:
       - file_name: logAllMimikatzModules.log
-        data: https://ssa-test-dataset.s3-us-west-2.amazonaws.com/logAllMimikatzModules.log
-
+        data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1070/illegal_log_deletion/logAllMimikatzModules.log
+        source: WinEventLog:Security

tests/endpoint/ssa___illegal_management_AD_elements_and_policies_via_dsinternals_modules.test.yml

@@ -6,4 +6,5 @@ tests:
     description: Test illegal management of Active Directory elements and policies detections
     attack_data:
       - file_name: logAllDSInternalsModules.log
-        data: https://ssa-test-dataset.s3-us-west-2.amazonaws.com/logAllDSInternalsModules.log
+        data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1484/logAllDSInternalsModules.log
+        source: WinEventLog:Security

tests/endpoint/ssa___illegal_management_computers_and_AD_elements_via_powersploit_modules.test.yml

@@ -6,5 +6,5 @@ tests:
     description: Test illegal management of computers and Active Directory elements detections
     attack_data:
       - file_name: logAllPowerSploitModulesWithOldNames.log
-        data: https://ssa-test-dataset.s3-us-west-2.amazonaws.com/logAllPowerSploitModulesWithOldNames.log
-
+        data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1484/logAllPowerSploitModulesWithOldNames.log
+        source: WinEventLog:Security

tests/endpoint/ssa___illegal_privilege_elevation_and_persistence_via_powersploit_modules.test.yml

@@ -6,5 +6,5 @@ tests:
     description: Test privilege elevation and persistence detections
     attack_data:
       - file_name: logAllPowerSploitModulesWithOldNames.log
-        data: https://ssa-test-dataset.s3-us-west-2.amazonaws.com/logAllPowerSploitModulesWithOldNames.log
-
+        data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1548/logAllPowerSploitModulesWithOldNames.log
+        source: WinEventLog:Security

tests/endpoint/ssa___illegal_privilege_elevation_via_mimikatz_modules.test.yml

@@ -6,5 +6,5 @@ tests:
     description: Test illegal privilege elevation detections
     attack_data:
       - file_name: logAllMimikatzModules.log
-        data: https://ssa-test-dataset.s3-us-west-2.amazonaws.com/logAllMimikatzModules.log
-
+        data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1548/logAllMimikatzModules.log
+        source: WinEventLog:Security

tests/endpoint/ssa___illegal_service_and_process_control_via_mimikatz_modules.test.yml

@@ -6,5 +6,5 @@ tests:
     description: Test illegal service and process control detections
     attack_data:
       - file_name: logAllMimikatzModules.log
-        data: https://ssa-test-dataset.s3-us-west-2.amazonaws.com/logAllMimikatzModules.log
-
+        data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1003/credential_extraction/logAllMimikatzModules.log
+        source: WinEventLog:Security

tests/endpoint/ssa___illegal_service_and_process_control_via_powersploit_modules.test.yml

@@ -6,5 +6,6 @@ tests:
     description: Test illegal service and process control detections
     attack_data:
       - file_name: logAllPowerSploitModulesWithOldNames.log
-        data: https://ssa-test-dataset.s3-us-west-2.amazonaws.com/logAllPowerSploitModulesWithOldNames.log
+        data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1003/credential_extraction/logAllPowerSploitModulesWithOldNames.log
+        source: WinEventLog:Security
 

tests/endpoint/ssa___probing_access_with_stolen_credentials_via_powersploit_modules.test.yml

@@ -6,5 +6,5 @@ tests:
     description: Test access probing with stolen credentials detections
     attack_data:
       - file_name: logAllPowerSploitModulesWithOldNames.log
-        data: https://ssa-test-dataset.s3-us-west-2.amazonaws.com/logAllPowerSploitModulesWithOldNames.log
-
+        data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1003/credential_extraction/logAllPowerSploitModulesWithOldNames.log
+        source: WinEventLog:Security

tests/endpoint/ssa___recon_access_and_persistence_opportunities_via_powersploit_modules.test.yml

@@ -6,5 +6,5 @@ tests:
     description: Test reconnaissance of access and persistence detections
     attack_data:
       - file_name: logAllPowerSploitModulesWithOldNames.log
-        data: https://ssa-test-dataset.s3-us-west-2.amazonaws.com/logAllPowerSploitModulesWithOldNames.log
-
+        data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1003/credential_extraction/logAllPowerSploitModulesWithOldNames.log
+        source: WinEventLog:Security

tests/endpoint/ssa___recon_and_use_accounts_groups_policies_via_powersploit_modules.test.yml

@@ -6,5 +6,6 @@ tests:
     description: Test reconnaissance and access to accounts groups and policies detections
     attack_data:
       - file_name: logAllPowerSploitModulesWithOldNames.log
-        data: https://ssa-test-dataset.s3-us-west-2.amazonaws.com/logAllPowerSploitModulesWithOldNames.log
+        data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1003/credential_extraction/logAllPowerSploitModulesWithOldNames.log
+        source: WinEventLog:Security
 

tests/endpoint/ssa___recon_and_use_accounts_groups_via_mimikatz_modules.test.yml

@@ -6,5 +6,6 @@ tests:
     description: Test reconnaissance and access to accounts and groups detections
     attack_data:
       - file_name: logAllMimikatzModules.log
-        data: https://ssa-test-dataset.s3-us-west-2.amazonaws.com/logAllMimikatzModules.log
+        data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1003/credential_extraction/logAllMimikatzModules.log
+        source: WinEventLog:Security
 

tests/endpoint/ssa___recon_and_use_active_directory_infrastructure_via_powersploit_modules.test.yml

@@ -6,5 +6,6 @@ tests:
     description: Test reconnaissance and access to active directory infrastrucutre detections
     attack_data:
       - file_name: logAllPowerSploitModulesWithOldNames.log
-        data: https://ssa-test-dataset.s3-us-west-2.amazonaws.com/logAllPowerSploitModulesWithOldNames.log
+        data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1003/credential_extraction/logAllPowerSploitModulesWithOldNames.log
+        source: WinEventLog:Security
 

tests/endpoint/ssa___recon_and_use_computers_domains_via_powersploit_modules.test.yml

@@ -6,5 +6,6 @@ tests:
     description: Test reconnaissance and access to computers and domains detections
     attack_data:
       - file_name: logAllPowerSploitModulesWithOldNames.log
-        data: https://ssa-test-dataset.s3-us-west-2.amazonaws.com/logAllPowerSploitModulesWithOldNames.log
+        data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1003/credential_extraction/logAllPowerSploitModulesWithOldNames.log
+        source: WinEventLog:Security
 

tests/endpoint/ssa___recon_and_use_computers_via_mimikatz_modules.test.yml

@@ -6,5 +6,6 @@ tests:
     description: Test reconnaissance and access to computers detections
     attack_data:
       - file_name: logAllMimikatzModules.log
-        data: https://ssa-test-dataset.s3-us-west-2.amazonaws.com/logAllMimikatzModules.log
+        data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1003/credential_extraction/logAllMimikatzModules.log
+        source: WinEventLog:Security
 

tests/endpoint/ssa___recon_and_use_operating_system_elements_via_powersploit_modules.test.yml

@@ -6,5 +6,6 @@ tests:
     description: Test reconnaissance and access to operating system element detections
     attack_data:
       - file_name: logAllPowerSploitModulesWithOldNames.log
-        data: https://ssa-test-dataset.s3-us-west-2.amazonaws.com/logAllPowerSploitModulesWithOldNames.log
+        data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1003/credential_extraction/logAllPowerSploitModulesWithOldNames.log
+        source: WinEventLog:Security
 

tests/endpoint/ssa___recon_and_use_shares_via_mimikatz_modules.test.yml

@@ -6,5 +6,5 @@ tests:
     description: Test reconnaissance and access to network shares detections
     attack_data:
       - file_name: logAllMimikatzModules.log
-        data: https://ssa-test-dataset.s3-us-west-2.amazonaws.com/logAllMimikatzModules.log
-
+        data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1003/credential_extraction/logAllMimikatzModules.log
+        source: WinEventLog:Security