From 300d61e08624a0eb154343a878ff2d87d8436934 Mon Sep 17 00:00:00 2001 From: Nasreddine Bencherchali <8741929+nasbench@users.noreply.github.com> Date: Wed, 14 Aug 2024 00:03:07 +0200 Subject: [PATCH 1/5] small updates --- sigma/conversion/base.py | 8 ++--- sigma/conversion/state.py | 2 +- sigma/modifiers.py | 32 ++++++++++---------- sigma/plugins.py | 2 +- sigma/rule.py | 49 ++++++++++++++----------------- sigma/types.py | 2 +- sigma/validators/core/metadata.py | 4 +-- 7 files changed, 47 insertions(+), 52 deletions(-) diff --git a/sigma/conversion/base.py b/sigma/conversion/base.py index 987c1694..28f21ce7 100644 --- a/sigma/conversion/base.py +++ b/sigma/conversion/base.py @@ -1022,8 +1022,8 @@ class variables. If this is not sufficient, the respective methods can be implem # * {count} is the value specified in the condition. # * {field} is the field specified in the condition. # * {referenced_rules} contains the Sigma rules that are referred by the correlation rule. This - # expression is generated by the referenced_rules_expression template in combincation with the - # referennced_rules_expression_joiner defined above. + # expression is generated by the referenced_rules_expression template in combination with the + # referenced_rules_expression_joiner defined above. event_count_condition_expression: ClassVar[Optional[Dict[str, str]]] = None value_count_condition_expression: ClassVar[Optional[Dict[str, str]]] = None temporal_condition_expression: ClassVar[Optional[Dict[str, str]]] = None @@ -1286,7 +1286,7 @@ def convert_condition_field_eq_val_str( ): expr = ( self.startswith_expression - ) # If all conditions are fulfilled, use 'startswith' operartor instead of equal token + ) # If all conditions are fulfilled, use 'startswith' operator instead of equal token value = cond.value[:-1] elif ( # Same as above but for 'endswith' operator: string starts with wildcard and doesn't contains further special characters self.endswith_expression is not None @@ -1336,7 +1336,7 @@ def convert_condition_field_eq_val_str_case_sensitive( ): expr = ( self.case_sensitive_startswith_expression - ) # If all conditions are fulfilled, use 'startswith' operartor instead of equal token + ) # If all conditions are fulfilled, use 'startswith' operator instead of equal token value = cond.value[:-1] elif ( # Same as above but for 'endswith' operator: string starts with wildcard and doesn't contains further special characters self.case_sensitive_endswith_expression is not None diff --git a/sigma/conversion/state.py b/sigma/conversion/state.py index ed232b83..01396034 100644 --- a/sigma/conversion/state.py +++ b/sigma/conversion/state.py @@ -7,7 +7,7 @@ class ConversionState: """ State class which is passed as object to each conversion method in query conversion and - finalization phase. All state information that is required in a later phase of the converison + finalization phase. All state information that is required in a later phase of the conversion should be stored in this class. The base class implements deferred query expressions, which are generated in the finalization diff --git a/sigma/modifiers.py b/sigma/modifiers.py index 96c0b3ec..f2eaf033 100644 --- a/sigma/modifiers.py +++ b/sigma/modifiers.py @@ -395,30 +395,30 @@ def modify(self, val: SigmaString) -> SigmaString: # Mapping from modifier identifier strings to modifier classes modifier_mapping: Dict[str, Type[SigmaModifier]] = { + "all": SigmaAllModifier, + "base64": SigmaBase64Modifier, + "base64offset": SigmaBase64OffsetModifier, + "cased": SigmaCaseSensitiveModifier, + "cidr": SigmaCIDRModifier, "contains": SigmaContainsModifier, - "startswith": SigmaStartswithModifier, + "dotall": SigmaRegularExpressionDotAllFlagModifier, "endswith": SigmaEndswithModifier, "exists": SigmaExistsModifier, - "base64": SigmaBase64Modifier, - "base64offset": SigmaBase64OffsetModifier, - "wide": SigmaWideModifier, - "windash": SigmaWindowsDashModifier, - "re": SigmaRegularExpressionModifier, + "expand": SigmaExpandModifier, + "fieldref": SigmaFieldReferenceModifier, + "gt": SigmaGreaterThanModifier, + "gte": SigmaGreaterThanEqualModifier, "i": SigmaRegularExpressionIgnoreCaseFlagModifier, "ignorecase": SigmaRegularExpressionIgnoreCaseFlagModifier, + "lt": SigmaLessThanModifier, + "lte": SigmaLessThanEqualModifier, "m": SigmaRegularExpressionMultilineFlagModifier, "multiline": SigmaRegularExpressionMultilineFlagModifier, + "re": SigmaRegularExpressionModifier, "s": SigmaRegularExpressionDotAllFlagModifier, - "dotall": SigmaRegularExpressionDotAllFlagModifier, - "cased": SigmaCaseSensitiveModifier, - "cidr": SigmaCIDRModifier, - "all": SigmaAllModifier, - "lt": SigmaLessThanModifier, - "lte": SigmaLessThanEqualModifier, - "gt": SigmaGreaterThanModifier, - "gte": SigmaGreaterThanEqualModifier, - "fieldref": SigmaFieldReferenceModifier, - "expand": SigmaExpandModifier, + "startswith": SigmaStartswithModifier, + "wide": SigmaWideModifier, + "windash": SigmaWindowsDashModifier, } # Mapping from modifier class to identifier diff --git a/sigma/plugins.py b/sigma/plugins.py index f6f7b4c9..df51b8e9 100644 --- a/sigma/plugins.py +++ b/sigma/plugins.py @@ -31,7 +31,7 @@ @dataclass class InstalledSigmaPlugins: - """Discovery and registrstion of installed backends, pipelines and validator checks as plugins. + """Discovery and registration of installed backends, pipelines and validator checks as plugins. This class represents a set of the objects mentioned above that are available. Further it implements autodiscovery of them in the sigma.backends, sigma.pipelines and sigma.validators module namespaces. diff --git a/sigma/rule.py b/sigma/rule.py index 00ababda..f00ade1f 100644 --- a/sigma/rule.py +++ b/sigma/rule.py @@ -131,10 +131,11 @@ def __hash__(self): class SigmaRelatedType(EnumLowercaseStringMixin, Enum): + CORRELATION = auto() + DERIVED = auto() + MERGED = auto() OBSOLETE = auto() RENAMED = auto() - MERGED = auto() - DERIVED = auto() SIMILAR = auto() @@ -491,7 +492,7 @@ def is_keyword(self) -> bool: @dataclass class SigmaDetection(ParentChainMixin): """ - A detection is a set of atomic event defitionions represented by SigmaDetectionItem instances. SigmaDetectionItems + A detection is a set of atomic event definitions represented by SigmaDetectionItem instances. SigmaDetectionItems of a SigmaDetection are OR-linked. A detection can be defined by: @@ -571,10 +572,10 @@ def to_plain(self) -> Union[Dict[str, Union[str, int, None]], List[str]]: } if len(detection_items) == 0: # pragma: no cover - return None # This case is catched by the post init check, so it shouldn't happen. + return None # This case is caught by the post init check, so it shouldn't happen. if len(detection_items) == 1: # Only one detection item? Return it as result. return detection_items[0] - else: # More than one detection iten, it depends now on the types + else: # More than one detection item, it depends now on the types if dict in detection_items_types and len(detection_items_types) > 1: # Merging dicts with other types isn't possibly, at least not in a simple way. # This case can appear in a programmatically instantiated detection, but can't be @@ -598,7 +599,7 @@ def to_plain(self) -> Union[Dict[str, Union[str, int, None]], List[str]]: for k, v in detection_item_converted.items(): if k not in merged: # key doesn't exists in merged dict: just add merged[k] = v - else: # key collision, now the things get complicated... + else: # key collision, now things get complicated... if "|all" in k: # key contains 'all' modifier if not isinstance( merged[k], list @@ -612,7 +613,7 @@ def to_plain(self) -> Union[Dict[str, Union[str, int, None]], List[str]]: else: # key collision without all modifier: trying to merge both keys into one and-linked key ev = merged[k] # already existing value - # Value normalization: extract value from signle-valued lists + # Value normalization: extract value from single-valued lists if isinstance(ev, list) and len(ev) == 1: ev = ev[0] if isinstance(v, list) and len(v) == 1: @@ -892,7 +893,7 @@ class instantiation of an object derived from the SigmaRuleBase class and the er except KeyError: errors.append( sigma_exceptions.SigmaLevelError( - f"'{ level }' is no valid Sigma rule level", source=source + f"'{ level }' is not a valid Sigma rule level", source=source ) ) @@ -911,7 +912,7 @@ class instantiation of an object derived from the SigmaRuleBase class and the er except KeyError: errors.append( sigma_exceptions.SigmaStatusError( - f"'{ status }' is no valid Sigma rule status", source=source + f"'{ status }' is not a valid Sigma rule status", source=source ) ) @@ -920,34 +921,28 @@ class instantiation of an object derived from the SigmaRuleBase class and the er if rule_date is not None: if not isinstance(rule_date, date) and not isinstance(rule_date, datetime): try: - rule_date = date(*(int(i) for i in rule_date.split("/"))) + rule_date = date(*(int(i) for i in rule_date.split("-"))) except ValueError: - try: - rule_date = date(*(int(i) for i in rule_date.split("-"))) - except ValueError: - errors.append( - sigma_exceptions.SigmaDateError( - f"Rule date '{ rule_date }' is invalid, must be yyyy/mm/dd or yyyy-mm-dd", - source=source, - ) + errors.append( + sigma_exceptions.SigmaDateError( + f"Rule date '{ rule_date }' is invalid, must be yyyy-mm-dd", + source=source, ) + ) # parse rule modified if existing rule_modified = rule.get("modified") if rule_modified is not None: if not isinstance(rule_modified, date) and not isinstance(rule_modified, datetime): try: - rule_modified = date(*(int(i) for i in rule_modified.split("/"))) + rule_modified = date(*(int(i) for i in rule_modified.split("-"))) except ValueError: - try: - rule_modified = date(*(int(i) for i in rule_modified.split("-"))) - except ValueError: - errors.append( - sigma_exceptions.SigmaModifiedError( - f"Rule modified '{ rule_modified }' is invalid, must be yyyy/mm/dd or yyyy-mm-dd", - source=source, - ) + errors.append( + sigma_exceptions.SigmaModifiedError( + f"Rule modified '{ rule_modified }' is invalid, must be yyyy-mm-dd", + source=source, ) + ) # Rule fields validation rule_fields = rule.get("fields") diff --git a/sigma/types.py b/sigma/types.py index 75038fb3..7d3ab0c6 100644 --- a/sigma/types.py +++ b/sigma/types.py @@ -817,7 +817,7 @@ class SigmaExpansion(NoPlainConversionMixin, SigmaType): expanded values and is converted as follows: 1. the whole expansion is handled as group which is enclosed in parentheses. - 2. the values contained in the expansion are linked with OR, independend from the linking of the + 2. the values contained in the expansion are linked with OR, independent from the linking of the context that encloses the expansion. """ diff --git a/sigma/validators/core/metadata.py b/sigma/validators/core/metadata.py index a814f3df..604ce2ff 100644 --- a/sigma/validators/core/metadata.py +++ b/sigma/validators/core/metadata.py @@ -163,7 +163,7 @@ def validate(self, rule: SigmaRule) -> List[SigmaValidationIssue]: @dataclass class DuplicateFilenameIssue(SigmaValidationIssue): - description: ClassVar[str] = "Rule filemane used by multiple rules" + description: ClassVar[str] = "Rule filename used by multiple rules" severity: ClassVar[SigmaValidationIssueSeverity] = SigmaValidationIssueSeverity.HIGH filename: str @@ -201,7 +201,7 @@ class FilenameLengthIssue(SigmaValidationIssue): @dataclass(frozen=True) class FilenameLengthValidator(SigmaRuleValidator): - """Check rule filename lengh""" + """Check rule filename length""" min_size: int = 10 max_size: int = 90 From f977078ab641af95ca3c0b4ee2aaf36094fb8253 Mon Sep 17 00:00:00 2001 From: Nasreddine Bencherchali <8741929+nasbench@users.noreply.github.com> Date: Wed, 14 Aug 2024 00:11:07 +0200 Subject: [PATCH 2/5] update dates --- tests/files/rule_filename_errors/Name.yml | 2 +- .../rule_valid/win_codeintegrity_unsigned_driver_loaded.yml | 4 ++-- tests/files/ruleset/subdirectory/test_rule_2.yml | 2 +- tests/files/ruleset/test_rule.yml | 2 +- tests/files/ruleset_duplicate/sub1/test_rule.yml | 2 +- tests/files/ruleset_duplicate/sub2/test_rule.yml | 2 +- tests/files/ruleset_with_errors/test_rule_with_error.yml | 2 +- tests/test_rule.py | 4 ++-- 8 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/files/rule_filename_errors/Name.yml b/tests/files/rule_filename_errors/Name.yml index 6b4d3059..85413ace 100644 --- a/tests/files/rule_filename_errors/Name.yml +++ b/tests/files/rule_filename_errors/Name.yml @@ -9,7 +9,7 @@ - attack.execution - attack.t1059 author: Thomas Patzke - date: 2020/07/13 + date: 2020-07-13 logsource: category: process_creation product: windows diff --git a/tests/files/rule_valid/win_codeintegrity_unsigned_driver_loaded.yml b/tests/files/rule_valid/win_codeintegrity_unsigned_driver_loaded.yml index eb356158..f9f32546 100644 --- a/tests/files/rule_valid/win_codeintegrity_unsigned_driver_loaded.yml +++ b/tests/files/rule_valid/win_codeintegrity_unsigned_driver_loaded.yml @@ -7,8 +7,8 @@ references: - https://learn.microsoft.com/en-us/windows/security/threat-protection/windows-defender-application-control/event-tag-explanations - Internal Research author: test -date: 2023/06/06 -modified: 2023/12/11 +date: 2023-06-06 +modified: 2023-12-11 tags: - attack.privilege_escalation logsource: diff --git a/tests/files/ruleset/subdirectory/test_rule_2.yml b/tests/files/ruleset/subdirectory/test_rule_2.yml index 6b4d3059..85413ace 100644 --- a/tests/files/ruleset/subdirectory/test_rule_2.yml +++ b/tests/files/ruleset/subdirectory/test_rule_2.yml @@ -9,7 +9,7 @@ - attack.execution - attack.t1059 author: Thomas Patzke - date: 2020/07/13 + date: 2020-07-13 logsource: category: process_creation product: windows diff --git a/tests/files/ruleset/test_rule.yml b/tests/files/ruleset/test_rule.yml index a249967d..ef725e5f 100644 --- a/tests/files/ruleset/test_rule.yml +++ b/tests/files/ruleset/test_rule.yml @@ -10,7 +10,7 @@ - attack.execution - attack.t1059 author: Thomas Patzke - date: 2020/07/12 + date: 2020-07-12 logsource: category: process_creation product: windows diff --git a/tests/files/ruleset_duplicate/sub1/test_rule.yml b/tests/files/ruleset_duplicate/sub1/test_rule.yml index e8ef6240..652777f2 100644 --- a/tests/files/ruleset_duplicate/sub1/test_rule.yml +++ b/tests/files/ruleset_duplicate/sub1/test_rule.yml @@ -9,7 +9,7 @@ - attack.execution - attack.t1059 author: Thomas Patzke - date: 2020/07/12 + date: 2020-07-12 logsource: category: process_creation product: windows diff --git a/tests/files/ruleset_duplicate/sub2/test_rule.yml b/tests/files/ruleset_duplicate/sub2/test_rule.yml index e8ef6240..652777f2 100644 --- a/tests/files/ruleset_duplicate/sub2/test_rule.yml +++ b/tests/files/ruleset_duplicate/sub2/test_rule.yml @@ -9,7 +9,7 @@ - attack.execution - attack.t1059 author: Thomas Patzke - date: 2020/07/12 + date: 2020-07-12 logsource: category: process_creation product: windows diff --git a/tests/files/ruleset_with_errors/test_rule_with_error.yml b/tests/files/ruleset_with_errors/test_rule_with_error.yml index 6c798019..d5de68a6 100644 --- a/tests/files/ruleset_with_errors/test_rule_with_error.yml +++ b/tests/files/ruleset_with_errors/test_rule_with_error.yml @@ -9,7 +9,7 @@ - attack.execution - attack.t1059 author: Thomas Patzke - date: 2020/07/12 + date: 2020-07-12 logsource: category: process_creation product: windows diff --git a/tests/test_rule.py b/tests/test_rule.py index 28662f21..6701a7ef 100644 --- a/tests/test_rule.py +++ b/tests/test_rule.py @@ -1120,7 +1120,7 @@ def test_sigmarule_fromyaml(sigma_rule): - attack.execution - attack.t1059 author: Thomas Patzke - date: 2020/07/12 + date: 2020-07-12 logsource: category: process_creation product: windows @@ -1161,7 +1161,7 @@ def test_sigmarule_fromyaml_with_custom_attribute(sigma_rule): - attack.execution - attack.t1059 author: Thomas Patzke - date: 2020/07/12 + date: 2020-07-12 logsource: category: process_creation product: windows From ce633e1c112baf83d6bff41ff84d580e2d2647c5 Mon Sep 17 00:00:00 2001 From: Nasreddine Bencherchali <8741929+nasbench@users.noreply.github.com> Date: Wed, 14 Aug 2024 00:16:32 +0200 Subject: [PATCH 3/5] Update test_rule.py --- tests/test_rule.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_rule.py b/tests/test_rule.py index 6701a7ef..99435c52 100644 --- a/tests/test_rule.py +++ b/tests/test_rule.py @@ -841,14 +841,14 @@ def test_sigmarule_bad_description(): def test_sigmarule_bad_level(): with pytest.raises( - sigma_exceptions.SigmaLevelError, match="no valid Sigma rule level.*test.yml" + sigma_exceptions.SigmaLevelError, match="not a valid Sigma rule level.*test.yml" ): SigmaRule.from_dict({"level": "bad"}, source=sigma_exceptions.SigmaRuleLocation("test.yml")) def test_sigmarule_bad_status(): with pytest.raises( - sigma_exceptions.SigmaStatusError, match="no valid Sigma rule status.*test.yml" + sigma_exceptions.SigmaStatusError, match="not a valid Sigma rule status.*test.yml" ): SigmaRule.from_dict( {"status": "bad"}, source=sigma_exceptions.SigmaRuleLocation("test.yml") From c63960a2aa0a73f26028a18297224317bb9f8250 Mon Sep 17 00:00:00 2001 From: Nasreddine Bencherchali <8741929+nasbench@users.noreply.github.com> Date: Wed, 14 Aug 2024 00:29:24 +0200 Subject: [PATCH 4/5] add error handling for validation config parsing --- sigma/exceptions.py | 6 ++++++ sigma/validation.py | 9 +++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/sigma/exceptions.py b/sigma/exceptions.py index 65e0f28f..a24c741f 100644 --- a/sigma/exceptions.py +++ b/sigma/exceptions.py @@ -241,6 +241,12 @@ class SigmaConfigurationError(SigmaError): pass +class SigmaValidatorConfigurationParsingError(SigmaError): + """Error in parsing of a Sigma validation configuration file.""" + + pass + + class SigmaFeatureNotSupportedByBackendError(SigmaError): """Sigma feature is not supported by the backend.""" diff --git a/sigma/validation.py b/sigma/validation.py index 43be0260..aebb8100 100644 --- a/sigma/validation.py +++ b/sigma/validation.py @@ -1,7 +1,7 @@ from collections import defaultdict from typing import DefaultDict, Dict, Iterable, Iterator, List, Set, Type, Union from uuid import UUID -from sigma.exceptions import SigmaConfigurationError +from sigma.exceptions import SigmaConfigurationError, SigmaValidatorConfigurationParsingError from sigma.rule import SigmaRule from sigma.validators.base import SigmaRuleValidator, SigmaValidationIssue import yaml @@ -109,7 +109,12 @@ def from_dict(cls, d: Dict, validators: Dict[str, SigmaRuleValidator]) -> "Sigma def from_yaml( cls, validator_config: str, validators: Dict[str, SigmaRuleValidator] ) -> "SigmaValidator": - return cls.from_dict(yaml.safe_load(validator_config), validators) + try: + return cls.from_dict(yaml.safe_load(validator_config), validators) + except yaml.parser.ParserError as e: + raise SigmaValidatorConfigurationParsingError( + f"Error in parsing of a Sigma validation configuration file." + ) def validate_rule(self, rule: SigmaRule) -> List[SigmaValidationIssue]: """ From d1c13beb2aea2751b715dd8de90ac5266070973d Mon Sep 17 00:00:00 2001 From: Thomas Patzke Date: Sun, 1 Sep 2024 01:18:23 +0200 Subject: [PATCH 5/5] Verbose YAML errors in validation config --- sigma/validation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sigma/validation.py b/sigma/validation.py index aebb8100..03625910 100644 --- a/sigma/validation.py +++ b/sigma/validation.py @@ -113,8 +113,8 @@ def from_yaml( return cls.from_dict(yaml.safe_load(validator_config), validators) except yaml.parser.ParserError as e: raise SigmaValidatorConfigurationParsingError( - f"Error in parsing of a Sigma validation configuration file." - ) + f"Error in parsing of a Sigma validation configuration file: {str(e)}" + ) from e def validate_rule(self, rule: SigmaRule) -> List[SigmaValidationIssue]: """