diff --git a/sigma/pipelines/azuremonitor/__init__.py b/sigma/pipelines/azuremonitor/__init__.py new file mode 100644 index 0000000..799bdb5 --- /dev/null +++ b/sigma/pipelines/azuremonitor/__init__.py @@ -0,0 +1,4 @@ +from .azuremonitor import azure_monitor_pipeline +pipelines = { + "azure_monitor": azure_monitor_pipeline, +} diff --git a/sigma/pipelines/azuremonitor/azuremonitor.py b/sigma/pipelines/azuremonitor/azuremonitor.py new file mode 100644 index 0000000..fa1298d --- /dev/null +++ b/sigma/pipelines/azuremonitor/azuremonitor.py @@ -0,0 +1,219 @@ +from typing import Optional + +from sigma.processing.conditions import ( + # DetectionItemProcessingItemAppliedCondition, + ExcludeFieldCondition, + IncludeFieldCondition, + LogsourceCondition, + RuleProcessingItemAppliedCondition, + RuleProcessingStateCondition, +) +from sigma.processing.pipeline import ProcessingItem, ProcessingPipeline +from sigma.processing.transformations import ( + DropDetectionItemTransformation, + ReplaceStringTransformation, + RuleFailureTransformation, +) + +from ..kusto_common.errors import InvalidFieldTransformation +from ..kusto_common.finalization import QueryTableFinalizer +from ..kusto_common.schema import create_schema +from ..kusto_common.transformations import ( + DynamicFieldMappingTransformation, + RegistryActionTypeValueTransformation, + SetQueryTableStateTransformation, +) +from .mappings import ( + AZURE_MONITOR_FIELD_MAPPINGS, + CATEGORY_TO_TABLE_MAPPINGS, +) +from .schema import AzureMonitorSchema +from .tables import AZURE_MONITOR_TABLES +from .transformations import ( + DefaultHashesValuesTransformation, + SecurityEventHashesValuesTransformation, +) + +AZURE_MONITOR_SCHEMA = create_schema(AzureMonitorSchema, AZURE_MONITOR_TABLES) + + +## Fieldmappings +fieldmappings_proc_item = ProcessingItem( + identifier="azure_monitor_table_fieldmappings", + transformation=DynamicFieldMappingTransformation(AZURE_MONITOR_FIELD_MAPPINGS), +) + +## Generic Field Mappings, keep this last +## Exclude any fields already mapped, e.g. if a table mapping has been applied. +# This will fix the case where ProcessId is usually mapped to InitiatingProcessId, EXCEPT for the DeviceProcessEvent table where it stays as ProcessId. +# So we can map ProcessId to ProcessId in the DeviceProcessEvents table mapping, and prevent the generic mapping to InitiatingProcessId from being applied +# by adding a detection item condition that the table field mappings have been applied + +# generic_field_mappings_proc_item = ProcessingItem( +# identifier="azure_monitor_generic_fieldmappings", +# transformation=GenericFieldMappingTransformation(AZURE_MONITOR_FIELD_MAPPINGS), +# detection_item_conditions=[DetectionItemProcessingItemAppliedCondition("azure_monitor_table_fieldmappings")], +# detection_item_condition_linking=any, +# detection_item_condition_negation=True, +# ) + +REGISTRY_FIELDS = [ + "RegistryKey", + "RegistryPreviousKey", + "ObjectName", +] + +## Field Value Replacements ProcessingItems +replacement_proc_items = [ + # Sysmon uses abbreviations in RegistryKey values, replace with full key names as the DeviceRegistryEvents schema + # expects them to be + # Note: Ensure this comes AFTER field mapping renames, as we're specifying DeviceRegistryEvent fields + # + # Do this one first, or else the HKLM only one will replace HKLM and mess up the regex + ProcessingItem( + identifier="azure_monitor_registry_key_replace_currentcontrolset", + transformation=ReplaceStringTransformation( + regex=r"(?i)(^HKLM\\SYSTEM\\CurrentControlSet)", + replacement=r"HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet001", + ), + field_name_conditions=[IncludeFieldCondition(REGISTRY_FIELDS)], + ), + ProcessingItem( + identifier="azure_monitor_registry_key_replace_hklm", + transformation=ReplaceStringTransformation(regex=r"(?i)(^HKLM)", replacement=r"HKEY_LOCAL_MACHINE"), + field_name_conditions=[IncludeFieldCondition(REGISTRY_FIELDS)], + ), + ProcessingItem( + identifier="azure_monitor_registry_key_replace_hku", + transformation=ReplaceStringTransformation(regex=r"(?i)(^HKU)", replacement=r"HKEY_USERS"), + field_name_conditions=[IncludeFieldCondition(REGISTRY_FIELDS)], + ), + ProcessingItem( + identifier="azure_monitor_registry_key_replace_hkcr", + transformation=ReplaceStringTransformation(regex=r"(?i)(^HKCR)", replacement=r"HKEY_LOCAL_MACHINE\\CLASSES"), + field_name_conditions=[IncludeFieldCondition(REGISTRY_FIELDS)], + ), + ProcessingItem( + identifier="azure_monitor_registry_actiontype_value", + transformation=RegistryActionTypeValueTransformation(), + field_name_conditions=[IncludeFieldCondition(["EventType"])], + ), + # Processing item to transform the Hashes field in the SecurityEvent table to get rid of the hash algorithm prefix in each value + ProcessingItem( + identifier="azure_monitor_securityevent_hashes_field_values", + transformation=SecurityEventHashesValuesTransformation(), + field_name_conditions=[IncludeFieldCondition(["FileHash"])], + rule_conditions=[RuleProcessingStateCondition("query_table", "SecurityEvent")], + ), + ProcessingItem( + identifier="azure_monitor_hashes_field_values", + transformation=DefaultHashesValuesTransformation(), + field_name_conditions=[IncludeFieldCondition(["Hashes"])], + rule_conditions=[RuleProcessingStateCondition("query_table", "SecurityEvent")], + rule_condition_negation=True, + ), + # Processing item to essentially ignore initiated field + ProcessingItem( + identifier="azure_monitor_network_initiated_field", + transformation=DropDetectionItemTransformation(), + field_name_conditions=[IncludeFieldCondition(["Initiated"])], + rule_conditions=[LogsourceCondition(category="network_connection")], + ), +] + +# Exceptions/Errors ProcessingItems +# Catch-all for when the query table is not set, meaning the rule could not be mapped to a table or the table name was not set +rule_error_proc_items = [ + # Category Not Supported or Query Table Not Set + ProcessingItem( + identifier="azure_monitor_unsupported_rule_category_or_missing_query_table", + transformation=RuleFailureTransformation( + "Rule category not yet supported by the Azure Monitor pipeline or query_table is not set." + ), + rule_conditions=[ + RuleProcessingItemAppliedCondition("azure_monitor_set_query_table"), + RuleProcessingStateCondition("query_table", None), + ], + rule_condition_linking=all, + ) +] + + +def get_valid_fields(table_name): + return ( + list(AZURE_MONITOR_SCHEMA.tables[table_name].fields.keys()) + + list(AZURE_MONITOR_FIELD_MAPPINGS.table_mappings.get(table_name, {}).keys()) + + list(AZURE_MONITOR_FIELD_MAPPINGS.generic_mappings.keys()) + + ["Hashes"] + ) + + +field_error_proc_items = [] + +for table_name in AZURE_MONITOR_SCHEMA.tables.keys(): + valid_fields = get_valid_fields(table_name) + + field_error_proc_items.append( + ProcessingItem( + identifier=f"azure_monitor_unsupported_fields_{table_name}", + transformation=InvalidFieldTransformation( + f"Please use valid fields for the {table_name} table, or the following fields that have fieldmappings in this " + f"pipeline:\n{', '.join(sorted(set(valid_fields)))}" + ), + field_name_conditions=[ExcludeFieldCondition(fields=valid_fields)], + rule_conditions=[ + RuleProcessingItemAppliedCondition("azure_monitor_set_query_table"), + RuleProcessingStateCondition("query_table", table_name), + ], + rule_condition_linking=all, + ) + ) + +# Add a catch-all error for custom table names +field_error_proc_items.append( + ProcessingItem( + identifier="azure_monitor_unsupported_fields_custom", + transformation=InvalidFieldTransformation( + "Invalid field name for the custom table. Please ensure you're using valid fields for your custom table." + ), + field_name_conditions=[ + ExcludeFieldCondition(fields=list(AZURE_MONITOR_FIELD_MAPPINGS.generic_mappings.keys()) + ["Hashes"]) + ], + rule_conditions=[ + RuleProcessingItemAppliedCondition("azure_monitor_set_query_table"), + RuleProcessingStateCondition("query_table", None), + ], + rule_condition_linking=all, + ) +) + + +def azure_monitor_pipeline(query_table: Optional[str] = None) -> ProcessingPipeline: + """Pipeline for transformations for SigmaRules to use in the Kusto Query Language backend. + + :param query_table: If specified, the table name will be used in the finalizer, otherwise the table name will be selected based on the category of the rule. + :type query_table: Optional[str] + + :return: ProcessingPipeline for Microsoft Azure Monitor + :rtype: ProcessingPipeline + """ + + pipeline_items = [ + ProcessingItem( + identifier="azure_monitor_set_query_table", + transformation=SetQueryTableStateTransformation(query_table, CATEGORY_TO_TABLE_MAPPINGS), + ), + fieldmappings_proc_item, + # generic_field_mappings_proc_item, + *replacement_proc_items, + *rule_error_proc_items, + *field_error_proc_items, + ] + + return ProcessingPipeline( + name="Generic Log Sources to Azure Monitor tables and fields", + priority=10, + items=pipeline_items, + allowed_backends=frozenset(["kusto"]), + finalizers=[QueryTableFinalizer()], + ) diff --git a/sigma/pipelines/azuremonitor/mappings.py b/sigma/pipelines/azuremonitor/mappings.py new file mode 100644 index 0000000..a782502 --- /dev/null +++ b/sigma/pipelines/azuremonitor/mappings.py @@ -0,0 +1,132 @@ +from sigma.pipelines.common import ( + logsource_windows_file_access, + logsource_windows_file_change, + logsource_windows_file_delete, + logsource_windows_file_event, + logsource_windows_file_rename, + logsource_windows_image_load, + logsource_windows_network_connection, + logsource_windows_process_creation, + logsource_windows_registry_add, + logsource_windows_registry_delete, + logsource_windows_registry_event, + logsource_windows_registry_set, +) +from sigma.pipelines.kusto_common.schema import FieldMappings + + +class AzureMonitorFieldMappings(FieldMappings): + pass + + +# Just map to SecurityEvent for now until we have more mappings for other tables +CATEGORY_TO_TABLE_MAPPINGS = { + "process_creation": "SecurityEvent", + "image_load": "SecurityEvent", + "file_access": "SecurityEvent", + "file_change": "SecurityEvent", + "file_delete": "SecurityEvent", + "file_event": "SecurityEvent", + "file_rename": "SecurityEvent", + "registry_add": "SecurityEvent", + "registry_delete": "SecurityEvent", + "registry_event": "SecurityEvent", + "registry_set": "SecurityEvent", + "network_connection": "SecurityEvent", +} + +## Rule Categories -> RuleConditions +CATEGORY_TO_CONDITIONS_MAPPINGS = { + "process_creation": logsource_windows_process_creation(), + "image_load": logsource_windows_image_load(), + "file_access": logsource_windows_file_access(), + "file_change": logsource_windows_file_change(), + "file_delete": logsource_windows_file_delete(), + "file_event": logsource_windows_file_event(), + "file_rename": logsource_windows_file_rename(), + "registry_add": logsource_windows_registry_add(), + "registry_delete": logsource_windows_registry_delete(), + "registry_event": logsource_windows_registry_event(), + "registry_set": logsource_windows_registry_set(), + "network_connection": logsource_windows_network_connection(), +} + + +AZURE_MONITOR_FIELD_MAPPINGS = AzureMonitorFieldMappings( + table_mappings={ + "SecurityEvent": { + "CommandLine": "CommandLine", + "Image": "NewProcessName", + "ParentImage": "ParentProcessName", + "User": "SubjectUserName", + "TargetFilename": "ObjectName", + "SourceIp": "IpAddress", + "DestinationIp": "DestinationIp", + "DestinationPort": "DestinationPort", + "SourcePort": "SourcePort", + "SourceHostname": "WorkstationName", + "DestinationHostname": "DestinationHostname", + "EventID": "EventID", + "ProcessId": "NewProcessId", + "ProcessName": "NewProcessName", + "LogonType": "LogonType", + "TargetUserName": "TargetUserName", + "TargetDomainName": "TargetDomainName", + "TargetLogonId": "TargetLogonId", + "Status": "Status", + "SubStatus": "SubStatus", + "ObjectType": "ObjectType", + "ShareName": "ShareName", + "AccessMask": "AccessMask", + "ServiceName": "ServiceName", + "TicketOptions": "TicketOptions", + "TicketEncryptionType": "TicketEncryptionType", + "TransmittedServices": "TransmittedServices", + "WorkstationName": "WorkstationName", + "LogonProcessName": "LogonProcessName", + "LogonGuid": "LogonGuid", + "Category": "EventSourceName", + "Hashes": "FileHash", + "TargetObject": "ObjectName", + }, + "SigninLogs": { + "User": "UserPrincipalName", + "TargetUserName": "UserPrincipalName", + "src_ip": "IPAddress", + "IpAddress": "IPAddress", + "app": "AppDisplayName", + "Application": "AppDisplayName", + "AuthenticationMethod": "AuthenticationMethodsUsed", + "Status": "Status", + "ResultType": "ResultType", + "ResultDescription": "ResultDescription", + "UserAgent": "UserAgent", + "Location": "Location", + "ClientAppUsed": "ClientAppUsed", + "DeviceDetail": "DeviceDetail", + "CorrelationId": "CorrelationId", + "ConditionalAccessStatus": "ConditionalAccessStatus", + "RiskLevelAggregated": "RiskLevelAggregated", + "RiskLevelDuringSignIn": "RiskLevelDuringSignIn", + "RiskDetail": "RiskDetail", + "RiskState": "RiskState", + "MfaDetail": "MfaDetail", + "NetworkLocationDetails": "NetworkLocationDetails", + "AuthenticationProtocol": "AuthenticationProtocol", + "AuthenticationRequirement": "AuthenticationRequirement", + "SignInIdentifier": "SignInIdentifier", + "SignInIdentifierType": "SignInIdentifierType", + "ResourceDisplayName": "ResourceDisplayName", + "ResourceIdentity": "ResourceIdentity", + "AppId": "AppId", + "AuthenticationProcessingDetails": "AuthenticationProcessingDetails", + "IsInteractive": "IsInteractive", + "TokenIssuerName": "TokenIssuerName", + "TokenIssuerType": "TokenIssuerType", + "UserType": "UserType", + "IPAddress": "IPAddress", + "AutonomousSystemNumber": "AutonomousSystemNumber", + }, + }, + generic_mappings={}, +) diff --git a/sigma/pipelines/azuremonitor/schema.py b/sigma/pipelines/azuremonitor/schema.py new file mode 100644 index 0000000..135044f --- /dev/null +++ b/sigma/pipelines/azuremonitor/schema.py @@ -0,0 +1,13 @@ +from dataclasses import dataclass + +from sigma.pipelines.kusto_common.schema import BaseSchema, FieldMappings + + +@dataclass +class AzureMonitorSchema(BaseSchema): + pass + + +@dataclass +class AzureMonitorFieldMappings(FieldMappings): + pass diff --git a/sigma/pipelines/azuremonitor/transformations.py b/sigma/pipelines/azuremonitor/transformations.py new file mode 100644 index 0000000..e8fe299 --- /dev/null +++ b/sigma/pipelines/azuremonitor/transformations.py @@ -0,0 +1,19 @@ +from ..kusto_common.transformations import BaseHashesValuesTransformation + + +class SecurityEventHashesValuesTransformation(BaseHashesValuesTransformation): + """ + Transforms the FileHash (originally Hashes) field in SecurityEvent table to get rid of the hash algorithm prefix in each value. + """ + + def __init__(self): + super().__init__(valid_hash_algos=["MD5", "SHA1", "SHA256"], field_prefix="FileHash", drop_algo_prefix=True) + + +class DefaultHashesValuesTransformation(BaseHashesValuesTransformation): + """ + Transforms the Hashes field in XDR Tables to create fields for each hash algorithm. + """ + + def __init__(self): + super().__init__(valid_hash_algos=["MD5", "SHA1", "SHA256"], field_prefix="")