Merge pull request #132 from mostafa/men-in-black-v

Linting with Black

.git-blame-ignore-revs

@@ -0,0 +1,5 @@
+# Reformatting by black
+a6b11efb3455b0d13d4d712413052061580d43c4
+# Set line-length to 100 for black
+17185b23e5d18f0977eaf0f96bd941fb6c8d53f1
+

.gitattributes

@@ -0,0 +1,3 @@
+* text=auto
+
+poetry.lock linguist-generated=true

.github/workflows/release.yml

@@ -23,6 +23,8 @@ jobs:
           [ "$(poetry version -s)" == "${GITHUB_REF#refs/tags/v}" ]
       - name: Install dependencies
         run: poetry install
+      - name: Lint with black
+        run: poetry run black --check .
       - name: Run tests
         run: poetry run pytest
       - name: Build packages
@@ -37,4 +39,4 @@ jobs:
         run: poetry publish -r testpypi
       - name: Publish to PyPI
         if: ${{ github.event_name == 'release' }}
-        run: poetry publish
+        run: poetry publish

.github/workflows/test.yml

@@ -1,17 +1,17 @@
 name: Test
 on:
   push:
-    branches: [ "*" ]
+    branches: ["*"]
   pull_request:
-    branches: [ "*" ]
+    branches: ["*"]
   workflow_dispatch:
 
 jobs:
   test:
     strategy:
       matrix:
-        os: [ 'ubuntu-20.04', 'windows-2019', 'macos-12' ]
-        python-version: [ '3.8', '3.9', '3.10', '3.11' ]
+        os: ["ubuntu-20.04", "windows-2019", "macos-12"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v3
@@ -24,6 +24,8 @@ jobs:
           cache: poetry
       - name: Install dependencies
         run: poetry install
+      - name: Lint with black
+        run: poetry run black --check .
       - name: Run tests
         run: poetry run pytest --cov=sigma --cov-report term --cov-report xml:cov.xml -vv
       - name: Store coverage for badge

.pre-commit-config.yaml

@@ -0,0 +1,5 @@
+repos:
+  - repo: https://github.com/psf/black
+    rev: 23.3.0
+    hooks:
+      - id: black

README.md

@@ -210,6 +210,12 @@ pySigma uses pytest as testing framework. Simply run `pytest` to run all tests.
 
 To build your own package run `poetry build`.
 
+## Linting
+
+To lint the code run `poetry run black`. To check for linting errors run `poetry run black --check`.
+
+This project also uses [pre-commit](https://pre-commit.com/), which is installed by poetry as part of dev dependencies. To install the git hooks run `poetry run pre-commit install` after cloning the repository and installing the dependencies.
+
 ## Contributing
 
 Pull requests are welcome. Please feel free to lodge any issues/PRs as discussion points.

docs/conf.py

@@ -12,14 +12,15 @@
 #
 import os
 import sys
-sys.path.insert(0, os.path.abspath('..'))
+
+sys.path.insert(0, os.path.abspath(".."))
 
 
 # -- Project information -----------------------------------------------------
 
-project = 'pySigma'
-copyright = '2021'
-author = 'Thomas Patzke'
+project = "pySigma"
+copyright = "2021"
+author = "Thomas Patzke"
 
 
 # -- General configuration ---------------------------------------------------
@@ -28,27 +29,27 @@
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
-    'sphinx.ext.autodoc',
-    'sphinx.ext.autosummary',
+    "sphinx.ext.autodoc",
+    "sphinx.ext.autosummary",
 ]
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This pattern also affects html_static_path and html_extra_path.
-exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
 
 
 # -- Options for HTML output -------------------------------------------------
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'alabaster'
+html_theme = "alabaster"
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = []
+html_static_path = []

print-coverage.py

@@ -12,4 +12,4 @@
 elif coverage >= 85.0:
     print("COVERAGE_COLOR=orange")
 else:
-    print("COVERAGE_COLOR=red")
+    print("COVERAGE_COLOR=red")

pyproject.toml

@@ -21,18 +21,23 @@ packages = [
 
 [tool.poetry.dependencies]
 python = "^3.8"
-pyyaml = "^6.0"
+packaging = "^22.0"
 pyparsing = "^3.0.7"
+pyyaml = "^6.0"
 requests = "^2.28.1"
-packaging = "^22.0"
 
 [tool.poetry.dev-dependencies]
+black = "^23.3.0"
+mypy = "^0.931"
+pre-commit = "^3.3.3"
+pylint = "^2.15.7"
 pytest = "^6.2.2"
 pytest-cov = "^2.11.1"
 pytest-mypy = "^0.6.2"
-pylint = "^2.15.7"
 Sphinx = "^4.2.0"
-mypy = "^0.931"
+
+[tool.black]
+line-length = 100
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]

sigma/backends/test/__init__.py

@@ -6,4 +6,4 @@
     backends = {
         "test": TextQueryTestBackend,
         "test_mandatory": MandatoryPipelineTestBackend,
-    }
+    }

sigma/backends/test/backend.py

@@ -9,9 +9,10 @@
 from sigma.processing.transformations import FieldMappingTransformation
 from sigma.types import SigmaCompareExpression
 
+
 class TextQueryTestBackend(TextQueryBackend):
-    name : str = "Test backend"
-    formats : Dict[str, str] = {
+    name: str = "Test backend"
+    formats: Dict[str, str] = {
         "default": "Default format",
         "test": "Dummy test format that equals default format",
         "state": "Test format that obtains information from state",
@@ -20,84 +21,96 @@ class TextQueryTestBackend(TextQueryBackend):
         "bytes": "Plain query as bytes",
     }
 
-    group_expression : ClassVar[str] = "({expr})"
+    group_expression: ClassVar[str] = "({expr})"
 
-    or_token : ClassVar[str] = "or"
-    and_token : ClassVar[str] = "and"
-    not_token : ClassVar[str] = "not"
-    eq_token : ClassVar[str] = "="
+    or_token: ClassVar[str] = "or"
+    and_token: ClassVar[str] = "and"
+    not_token: ClassVar[str] = "not"
+    eq_token: ClassVar[str] = "="
 
-    field_quote : ClassVar[str] = "'"
-    field_quote_pattern : ClassVar[Pattern] = re.compile("^\w+$")
+    field_quote: ClassVar[str] = "'"
+    field_quote_pattern: ClassVar[Pattern] = re.compile("^\w+$")
 
-    str_quote : ClassVar[str] = '"'
-    escape_char : ClassVar[str] = "\\"
-    wildcard_multi : ClassVar[str] = "*"
-    wildcard_single : ClassVar[str] = "?"
-    add_escaped : ClassVar[str] = ":"
-    filter_chars : ClassVar[str] = "&"
-    bool_values : ClassVar[Dict[bool, str]] = {
+    str_quote: ClassVar[str] = '"'
+    escape_char: ClassVar[str] = "\\"
+    wildcard_multi: ClassVar[str] = "*"
+    wildcard_single: ClassVar[str] = "?"
+    add_escaped: ClassVar[str] = ":"
+    filter_chars: ClassVar[str] = "&"
+    bool_values: ClassVar[Dict[bool, str]] = {
         True: "1",
         False: "0",
     }
 
-    startswith_expression : ClassVar[str] = "{field} startswith {value}"
-    endswith_expression   : ClassVar[str] = "{field} endswith {value}"
-    contains_expression   : ClassVar[str] = "{field} contains {value}"
-    wildcard_match_expression : ClassVar[str] = "{field} match {value}"
+    startswith_expression: ClassVar[str] = "{field} startswith {value}"
+    endswith_expression: ClassVar[str] = "{field} endswith {value}"
+    contains_expression: ClassVar[str] = "{field} contains {value}"
+    wildcard_match_expression: ClassVar[str] = "{field} match {value}"
 
-    field_exists_expression     : ClassVar[str] = "exists({field})"
-    field_not_exists_expression : ClassVar[str] = "notexists({field})"
+    field_exists_expression: ClassVar[str] = "exists({field})"
+    field_not_exists_expression: ClassVar[str] = "notexists({field})"
 
-    re_expression : ClassVar[str] = "{field}=/{regex}/"
-    re_escape_char : ClassVar[str] = "\\"
-    re_escape : ClassVar[Tuple[str]] = ("/", "bar")
+    re_expression: ClassVar[str] = "{field}=/{regex}/"
+    re_escape_char: ClassVar[str] = "\\"
+    re_escape: ClassVar[Tuple[str]] = ("/", "bar")
 
     case_sensitive_match_expression = "{field} casematch {value}"
-    case_sensitive_startswith_expression : ClassVar[str] = "{field} startswith_cased {value}"
-    case_sensitive_endswith_expression   : ClassVar[str] = "{field} endswith_cased {value}"
-    case_sensitive_contains_expression   : ClassVar[str] = "{field} contains_cased {value}"
-
-    cidr_expression : ClassVar[str] = "cidrmatch('{field}', \"{value}\")"
-
-    compare_op_expression : ClassVar[str] = "{field}{operator}{value}"
-    compare_operators : ClassVar[Dict[SigmaCompareExpression.CompareOperators, str]] = {
-        SigmaCompareExpression.CompareOperators.LT  : "<",
-        SigmaCompareExpression.CompareOperators.LTE : "<=",
-        SigmaCompareExpression.CompareOperators.GT  : ">",
-        SigmaCompareExpression.CompareOperators.GTE : ">=",
+    case_sensitive_startswith_expression: ClassVar[str] = "{field} startswith_cased {value}"
+    case_sensitive_endswith_expression: ClassVar[str] = "{field} endswith_cased {value}"
+    case_sensitive_contains_expression: ClassVar[str] = "{field} contains_cased {value}"
+
+    cidr_expression: ClassVar[str] = "cidrmatch('{field}', \"{value}\")"
+
+    compare_op_expression: ClassVar[str] = "{field}{operator}{value}"
+    compare_operators: ClassVar[Dict[SigmaCompareExpression.CompareOperators, str]] = {
+        SigmaCompareExpression.CompareOperators.LT: "<",
+        SigmaCompareExpression.CompareOperators.LTE: "<=",
+        SigmaCompareExpression.CompareOperators.GT: ">",
+        SigmaCompareExpression.CompareOperators.GTE: ">=",
     }
 
-    field_equals_field_expression : ClassVar[str] = "{field1}=fieldref({field2})"
+    field_equals_field_expression: ClassVar[str] = "{field1}=fieldref({field2})"
 
-    field_null_expression : ClassVar[str] = "{field} is null"
+    field_null_expression: ClassVar[str] = "{field} is null"
 
-    convert_or_as_in : ClassVar[bool] = True
-    convert_and_as_in : ClassVar[bool] = True
-    in_expressions_allow_wildcards : ClassVar[bool] = True
-    field_in_list_expression : ClassVar[str] = "{field} {op} ({list})"
-    or_in_operator : ClassVar[Optional[str]] = "in"
-    and_in_operator : ClassVar[Optional[str]] = "contains-all"
-    list_separator : ClassVar[str] = ", "
+    convert_or_as_in: ClassVar[bool] = True
+    convert_and_as_in: ClassVar[bool] = True
+    in_expressions_allow_wildcards: ClassVar[bool] = True
+    field_in_list_expression: ClassVar[str] = "{field} {op} ({list})"
+    or_in_operator: ClassVar[Optional[str]] = "in"
+    and_in_operator: ClassVar[Optional[str]] = "contains-all"
+    list_separator: ClassVar[str] = ", "
 
-    unbound_value_str_expression : ClassVar[str] = '_={value}'
-    unbound_value_num_expression : ClassVar[str] = '_={value}'
-    unbound_value_re_expression : ClassVar[str] = '_=/{value}/'
+    unbound_value_str_expression: ClassVar[str] = "_={value}"
+    unbound_value_num_expression: ClassVar[str] = "_={value}"
+    unbound_value_re_expression: ClassVar[str] = "_=/{value}/"
 
-    deferred_start : ClassVar[str] = " | "
-    deferred_separator : ClassVar[str] = " | "
-    deferred_only_query : ClassVar[str] = "*"
+    deferred_start: ClassVar[str] = " | "
+    deferred_separator: ClassVar[str] = " | "
+    deferred_only_query: ClassVar[str] = "*"
 
     backend_processing_pipeline = dummy_test_pipeline()
-    output_format_processing_pipeline = defaultdict(ProcessingPipeline,
-        test=ProcessingPipeline([
-            ProcessingItem(FieldMappingTransformation({
-                "fieldC": "mappedC",
-            }))
-        ])
+    output_format_processing_pipeline = defaultdict(
+        ProcessingPipeline,
+        test=ProcessingPipeline(
+            [
+                ProcessingItem(
+                    FieldMappingTransformation(
+                        {
+                            "fieldC": "mappedC",
+                        }
+                    )
+                )
+            ]
+        ),
     )
 
-    def __init__(self, processing_pipeline: Optional[ProcessingPipeline] = None, collect_errors: bool = False, testparam: Optional[str] = None):
+    def __init__(
+        self,
+        processing_pipeline: Optional[ProcessingPipeline] = None,
+        collect_errors: bool = False,
+        testparam: Optional[str] = None,
+    ):
         super().__init__(processing_pipeline, collect_errors)
         self.testparam = testparam
 
@@ -107,8 +120,14 @@ def finalize_query_test(self, rule, query, index, state):
     def finalize_output_test(self, queries):
         return self.finalize_output_default(queries)
 
-    def finalize_query_state(self, rule, query, index, state : ConversionState):
-        return "index=" + state.processing_state.get("index", "default") + " (" + self.finalize_query_default(rule, query, index, state) + ")"
+    def finalize_query_state(self, rule, query, index, state: ConversionState):
+        return (
+            "index="
+            + state.processing_state.get("index", "default")
+            + " ("
+            + self.finalize_query_default(rule, query, index, state)
+            + ")"
+        )
 
     def finalize_output_state(self, queries):
         return self.finalize_output_default(queries)
@@ -118,9 +137,9 @@ def finalize_query_list_of_dict(self, rule, query, index, state):
 
     def finalize_output_list_of_dict(self, queries):
         return [
-            { "query": query, "test": self.testparam }
+            {"query": query, "test": self.testparam}
             if self.testparam is not None
-            else { "query": query }
+            else {"query": query}
             for query in self.finalize_output_default(queries)
         ]
 
@@ -136,5 +155,6 @@ def finalize_query_str(self, rule, query, index, state):
     def finalize_output_str(self, queries):
         return "\n".join(self.finalize_output_default(queries))
 
+
 class MandatoryPipelineTestBackend(TextQueryTestBackend):
-    requires_pipeline : bool = True
+    requires_pipeline: bool = True