Skip to content

Commit

Permalink
Merge branch 'master' of https://github.com/dhwanishah-crest/Azure-Se…
Browse files Browse the repository at this point in the history 
…ntinel into ArmisAzureDeployToolTipChange
  • Loading branch information
dhwanishah-crest committed Oct 30, 2024
2 parents 83e3582 + bc85dd6 commit ce39e4f
Show file tree
Hide file tree
Showing 127 changed files with 11,581 additions and 4,243 deletions.
8 changes: 6 additions & 2 deletions .script/dataConnectorValidator.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,11 @@ export async function IsValidDataConnectorSchema(filePath: string): Promise<Exit

/* Disabling temporarily till we get confirmation from PM*/
// isValidFileName(filePath
isValidPermissions(jsonFile.permissions, connectorCategory);
/* Skip validation for Solution Microsoft Exchange Security - Exchange On-Premises Solution */
if (!filePath.includes('Microsoft Exchange Security - Exchange On-Premises'))
{
isValidPermissions(jsonFile.permissions, connectorCategory);
}
}
else{
console.warn(`Skipping File as it is of type Events : ${filePath}`)
Expand Down Expand Up @@ -173,4 +177,4 @@ let CheckOptions = {
},
};

runCheckOverChangedFiles(CheckOptions, fileKinds, fileTypeSuffixes, filePathFolderPrefixes);
runCheckOverChangedFiles(CheckOptions, fileKinds, fileTypeSuffixes, filePathFolderPrefixes);
15 changes: 15 additions & 0 deletions .script/tests/KqlvalidationsTests/SkipValidationsTemplates.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2624,6 +2624,16 @@
"templateName": "InfobloxSOCInsightsDataConnector_API.json",
"validationFailReason": "The name 'insightId_g' does not refer to any known column, table, variable or function."
},
{
"id": "ESI-Opt6ExchangeMessageTrackingLogs",
"templateName": "ESI-Opt6ExchangeMessageTrackingLogs.json",
"validationFailReason": "This is a Azure Monitor Connector which doesnt requires more permissions. Skipping this ID as a check is failing for required permissions for Data Connector template. "
},
{
"id": "ESI-Opt7ExchangeHTTPProxyLogs",
"templateName": "ESI-Opt7ExchangeHTTPProxyLogs.json",
"validationFailReason": "This is a Azure Monitor Connector which doesnt requires more permissions. Skipping this ID as a check is failing for required permissions for Data Connector template. "
},
// Temporarily adding Data connector template id's for KQL Validations - End


Expand Down Expand Up @@ -2819,6 +2829,11 @@
"templateName": "ExchangeConfiguration.yaml",
"validationFailReason": "Temporarily Added for Parser KQL Queries validation"
},
{
"id": "0a0f4ea0-6b94-4420-892e-41ca985f2f01",
"templateName": "MESCompareDataOnPMRA.yaml",
"validationFailReason": "Temporarily Added for Parser KQL Queries validation"
},
{
"id": "1acab329-1c11-42a7-b5ea-41264947947a",
"templateName": "ExchangeEnvironmentList.yaml",
Expand Down
5 changes: 5 additions & 0 deletions .script/tests/NonAsciiValidationsTests/SkipValidationsTemplates.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,11 @@
"id": "39f51672-8c63-4600-882a-5db8275f798f",
"templateName": "Microsoft Exchange Security - MESCompareDataMRA parser",
"validationFailReason": "Non-ASCII characters are required to test comparison of strings with non-ASCII characters"
},
{
"id": "0a0f4ea0-6b94-4420-892e-41ca985f2f01",
"templateName": "Microsoft Exchange Security - MESCompareDataOnPMRA parser",
"validationFailReason": "Non-ASCII characters are required to test comparison of strings with non-ASCII characters"
}
]

Expand Down
61 changes: 40 additions & 21 deletions .script/tests/asimParsersTest/ASimFilteringTest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,13 @@
end_time = datetime.now(timezone.utc)
start_time = end_time - timedelta(days = TIME_SPAN_IN_DAYS)

# Define the dictionary mapping schemas to their respective messages
failure_messages = {
'AuditEvent': "This single failure is because only one value exist in 'EventResult' field in 'AuditEvent' schema. Audit Event is a special case where 'EventResult' validations could be partial as only 'Success' events exists. Ignoring this error.",
'Authentication': "This single failure is because only two values exist in 'EventType' field in 'Authentication' schema. 'Authentication' is a special case where 'EventType' validations could be partial as only 'Logon' or 'Logoff' events may exists. Ignoring this error.",
'Dns': "This single failure is because only one value exist in 'EventType' field in 'Dns' schema. 'Dns' is a special case where 'EventType' validations could be 'Query' only. Ignoring this error."
}

def attempt_to_connect():
try:
credential = DefaultAzureCredential()
Expand Down Expand Up @@ -230,6 +237,20 @@ def read_exclusion_list_from_csv():
exclusion_list.append(row[0])
return exclusion_list

# Function to handle printing and flushing
def print_and_flush(message):
print(f"{YELLOW} {message} {RESET}")
sys.stdout.flush()

# Function to handle error printing, flushing, and exiting
def handle_test_failure(parser_file_path, error_message=None):
if error_message:
print(f"::error::{error_message}")
else:
print(f"::error::Tests failed for {parser_file_path}")
sys.stdout.flush()
sys.exit(1) # Uncomment this line to fail workflow when tests are not successful.

def main():
# Get modified ASIM Parser files along with their status
current_directory = os.path.dirname(os.path.abspath(__file__))
Expand Down Expand Up @@ -288,29 +309,27 @@ def main():
if parser_file['EquivalentBuiltInParser'] in read_exclusion_list_from_csv():
print(f"{YELLOW}The parser {parser_file_path} is listed in the exclusions file. Therefore, this workflow run will not fail because of it. To allow this parser to cause the workflow to fail, please remove its name from the exclusions list file located at: {exclusion_file_path}{RESET}")
sys.stdout.flush()
# If Failure count is due to EventResult and EventSchema is AuditEvent, then ignore the failure.
# Audit Event is a special case where 'EventResult' validations could be partial like only 'Success' events.
elif len(result.failures) == 1:
# Check for exception cases where the failure can be ignored
# Check if the failure message and schema match the exception cases
if len(result.failures) == 1:
failure_message = result.failures[0][1]
if "eventresult - validations for this parameter are partial" in failure_message:
if parser_file['Normalization']['Schema'] == 'AuditEvent':
print(f"{YELLOW} This single failure is due to partial result in 'EventResult' field in 'AuditEvent' schema. Audit Event is a special case where 'EventResult' validations could be partial like only 'Success' events. Ignoring this error. {RESET}")
sys.stdout.flush()
elif "eventtype_in - Expected to have less results after filtering." in failure_message:
if parser_file['Normalization']['Schema'] == 'Authentication':
print(f"{YELLOW} This single failure is due to only two values in 'EventType' field in 'Authentication' schema. 'Authentication' is a special case where 'EventType' validations could be 'Logon' or 'Logoff' only. Ignoring this error. {RESET}")
sys.stdout.flush()
else:
print(f"::error::Tests failed for {parser_file_path}")
sys.stdout.flush()
sys.exit(1) # uncomment this line to fail workflow when tests are not successful.
schema = parser_file['Normalization']['Schema']
match schema:
case 'AuditEvent' if "eventresult - validations for this parameter are partial" in failure_message:
print_and_flush(failure_messages['AuditEvent'])
case 'Authentication' if "eventtype_in - Expected to have less results after filtering." in failure_message:
print_and_flush(failure_messages['Authentication'])
case 'Dns' if "eventtype - validations for this parameter are partial" in failure_message:
print_and_flush(failure_messages['Dns'])
case _:
# Default case when single error and if no specific condition matches
handle_test_failure(parser_file_path)
else:
print(f"::error::Tests failed for {parser_file_path}")
sys.stdout.flush() # Explicitly flush stdout
sys.exit(1) # uncomment this line to fail workflow when tests are not successful.
# When more than one failures or no specific exception case
handle_test_failure(parser_file_path)
except subprocess.CalledProcessError as e:
print(f"::error::An error occurred while reading parser file: {e}")
sys.stdout.flush() # Explicitly flush stdout
# Handle exceptions raised during the parser execution e.g. error in KQL query
handle_test_failure(parser_file_path, f"An error occurred while running parser: {e}")


class FilteringTest(unittest.TestCase):
Expand Down Expand Up @@ -404,7 +423,7 @@ def check_required_fields(self, parser_file):
def datetime_test(self, param, query_definition, column_name_in_table):
param_name = param['Name']
# Get count of rows without filtering
no_filter_query = query_definition + f"query()\n"
no_filter_query = query_definition + f"query() | project TimeGenerated \n"
no_filter_response = self.send_query(no_filter_query)
num_of_rows_when_no_filters_in_query = len(no_filter_response.tables[0].rows)
self.assertNotEqual(len(no_filter_response.tables[0].rows) , 0 , f"No data for parameter:{param_name}")
Expand Down
1 change: 1 addition & 0 deletions .script/tests/asimParsersTest/ingestASimSampleData.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -399,6 +399,7 @@ def convert_data_type(schema_result, data_result):
#create dcr for ingestion
guid_columns = []
schema = get_schema_for_builtin(table_name)
data_result = convert_data_type(schema, data_result)
request_body, url_to_call , method_to_use ,stream_name = create_dcr(json.dumps(schema, indent=4),table_name,"Microsoft")
response_body=hit_api(url_to_call,request_body,method_to_use)
print(f"Response of DCR creation: {response_body.text}")
Expand Down
3 changes: 2 additions & 1 deletion .script/tests/asimParsersTest/runAsimTesters.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,8 @@ function testParser([Parser] $parser) {

Write-Host "***************************************************"
Write-Host "${yellow}Running 'Data' tests for '$($parser.Name)' parser${reset}"
$dataTest = "$parserAsletStatement`r`n$letStatementName | invoke ASimDataTester('$($parser.Schema)')"
# Test with only last 30 minutes of data.
$dataTest = "$parserAsletStatement`r`n$letStatementName | where TimeGenerated >= ago(30min) | invoke ASimDataTester('$($parser.Schema)')"
invokeAsimTester $dataTest $parser.Name "data"
Write-Host "***************************************************"
}
Expand Down
2 changes: 1 addition & 1 deletion Parsers/ASimAuthentication/ARM/ASimAuthentication/ASimAuthentication.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
"displayName": "Authentication ASIM parser",
"category": "ASIM",
"FunctionAlias": "ASimAuthentication",
"query": "let DisabledParsers=materialize(_GetWatchlist('ASimDisabledParsers') | where SearchKey in ('Any', 'ExcludeASimAuthentication') | extend SourceSpecificParser=column_ifexists('SourceSpecificParser','') | distinct SourceSpecificParser);\nlet ASimAuthenticationDisabled=toscalar('ExcludeASimAuthentication' in (DisabledParsers) or 'Any' in (DisabledParsers)); \nunion isfuzzy=true\n vimAuthenticationEmpty, \n ASimAuthenticationAADManagedIdentitySignInLogs (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationAADManagedIdentitySignInLogs' in (DisabledParsers) )),\n ASimAuthenticationAADNonInteractiveUserSignInLogs (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationAADNonInteractiveUserSignInLogs' in (DisabledParsers) )),\n ASimAuthenticationAADServicePrincipalSignInLogs (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationAADServicePrincipalSignInLogs' in (DisabledParsers) )),\n ASimAuthenticationAWSCloudTrail (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationAWSCloudTrail' in (DisabledParsers) )),\n ASimAuthenticationBarracudaWAF (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationBarracudaWAF' in (DisabledParsers) )),\n ASimAuthenticationCiscoASA (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationCiscoASA' in (DisabledParsers) )), \n ASimAuthenticationCiscoISE (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationCiscoISE' in (DisabledParsers) )),\n ASimAuthenticationCiscoMeraki (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationCiscoMeraki' in (DisabledParsers) )),\n ASimAuthenticationCiscoMerakiSyslog (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationCiscoMerakiSyslog' in (DisabledParsers) )),\n ASimAuthenticationM365Defender (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationM365Defender' in (DisabledParsers) )),\n ASimAuthenticationMD4IoT (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationMD4IoT' in (DisabledParsers) )),\n ASimAuthenticationMicrosoftWindowsEvent (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationMicrosoftWindowsEvent' in (DisabledParsers) )),\n ASimAuthenticationOktaSSO (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationOktaSSO' in (DisabledParsers) )),\n ASimAuthenticationOktaV2(ASimAuthenticationDisabled or ('ExcludeASimAuthenticationOktaV2' in (DisabledParsers) )),\n ASimAuthenticationPostgreSQL (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationPostgreSQL' in (DisabledParsers) )),\n ASimAuthenticationSigninLogs (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationSigninLogs' in (DisabledParsers) )),\n ASimAuthenticationSshd (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationSshd' in (DisabledParsers) )),\n ASimAuthenticationSu (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationSu' in (DisabledParsers) )),\n ASimAuthenticationSudo (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationSudo' in (DisabledParsers) )),\n ASimAuthenticationSalesforceSC (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationSalesforceSC' in (DisabledParsers) )),\n ASimAuthenticationVectraXDRAudit (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationVectraXDRAudit' in (DisabledParsers) )),\n ASimAuthenticationSentinelOne (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationSentinelOne' in (DisabledParsers) )),\n ASimAuthenticationGoogleWorkspace (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationGoogleWorkspace' in (DisabledParsers) )),\n ASimAuthenticationPaloAltoCortexDataLake (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationPaloAltoCortexDataLake' in (DisabledParsers) )),\n ASimAuthenticationVMwareCarbonBlackCloud (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationVMwareCarbonBlackCloud' in (DisabledParsers) )),\n ASimAuthenticationCrowdStrikeFalconHost (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationCrowdStrikeFalcon' in (DisabledParsers) ))\n",
"query": "let DisabledParsers=materialize(_GetWatchlist('ASimDisabledParsers') | where SearchKey in ('Any', 'ExcludeASimAuthentication') | extend SourceSpecificParser=column_ifexists('SourceSpecificParser','') | distinct SourceSpecificParser);\nlet ASimAuthenticationDisabled=toscalar('ExcludeASimAuthentication' in (DisabledParsers) or 'Any' in (DisabledParsers)); \nunion isfuzzy=true\n vimAuthenticationEmpty, \n ASimAuthenticationAADManagedIdentitySignInLogs (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationAADManagedIdentitySignInLogs' in (DisabledParsers) )),\n ASimAuthenticationAADNonInteractiveUserSignInLogs (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationAADNonInteractiveUserSignInLogs' in (DisabledParsers) )),\n ASimAuthenticationAADServicePrincipalSignInLogs (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationAADServicePrincipalSignInLogs' in (DisabledParsers) )),\n ASimAuthenticationAWSCloudTrail (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationAWSCloudTrail' in (DisabledParsers) )),\n ASimAuthenticationBarracudaWAF (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationBarracudaWAF' in (DisabledParsers) )),\n ASimAuthenticationCiscoASA (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationCiscoASA' in (DisabledParsers) )), \n ASimAuthenticationCiscoISE (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationCiscoISE' in (DisabledParsers) )),\n ASimAuthenticationCiscoMeraki (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationCiscoMeraki' in (DisabledParsers) )),\n ASimAuthenticationCiscoMerakiSyslog (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationCiscoMerakiSyslog' in (DisabledParsers) )),\n ASimAuthenticationM365Defender (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationM365Defender' in (DisabledParsers) )),\n ASimAuthenticationMD4IoT (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationMD4IoT' in (DisabledParsers) )),\n ASimAuthenticationMicrosoftWindowsEvent (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationMicrosoftWindowsEvent' in (DisabledParsers) )),\n ASimAuthenticationOktaSSO (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationOktaSSO' in (DisabledParsers) )),\n ASimAuthenticationOktaV2(ASimAuthenticationDisabled or ('ExcludeASimAuthenticationOktaV2' in (DisabledParsers) )),\n ASimAuthenticationPostgreSQL (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationPostgreSQL' in (DisabledParsers) )),\n ASimAuthenticationSigninLogs (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationSigninLogs' in (DisabledParsers) )),\n ASimAuthenticationSshd (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationSshd' in (DisabledParsers) )),\n ASimAuthenticationSu (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationSu' in (DisabledParsers) )),\n ASimAuthenticationSudo (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationSudo' in (DisabledParsers) )),\n ASimAuthenticationSalesforceSC (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationSalesforceSC' in (DisabledParsers) )),\n ASimAuthenticationVectraXDRAudit (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationVectraXDRAudit' in (DisabledParsers) )),\n ASimAuthenticationSentinelOne (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationSentinelOne' in (DisabledParsers) )),\n ASimAuthenticationGoogleWorkspace (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationGoogleWorkspace' in (DisabledParsers) )),\n ASimAuthenticationPaloAltoCortexDataLake (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationPaloAltoCortexDataLake' in (DisabledParsers) )),\n ASimAuthenticationVMwareCarbonBlackCloud (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationVMwareCarbonBlackCloud' in (DisabledParsers) )),\n ASimAuthenticationCrowdStrikeFalconHost (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationCrowdStrikeFalcon' in (DisabledParsers) )),\n ASimAuthenticationIllumioSaaSCore (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationIllumioSaaS' in (DisabledParsers) ))\n",
"version": 1,
"functionParameters": "disabled:bool=False"
}
Expand Down
Loading

0 comments on commit ce39e4f

Please sign in to comment.