First unit test and spark conf

src/dq_suite/df_checker.py

@@ -1,7 +1,7 @@
+import datetime
 from typing import List
 
 import great_expectations
-import datetime
 from great_expectations import Checkpoint, ValidationDefinition
 from great_expectations.checkpoint.actions import CheckpointAction
 from great_expectations.checkpoint.checkpoint import CheckpointResult
@@ -217,7 +217,7 @@ def run(
         validation_settings_obj=validation_settings_obj,
     )
     validation_output = checkpoint_result.describe_dict()
-    run_time = datetime.datetime.now() #TODO: get from RunIdentifier object
+    run_time = datetime.datetime.now()  # TODO: get from RunIdentifier object
 
     # 3) write results to unity catalog
     write_non_validation_tables(
@@ -230,5 +230,5 @@ def run(
         df=df,
         dataset_name=validation_dict["dataset"]["name"],
         unique_identifier=rules_dict["unique_identifier"],
-        run_time = run_time,
+        run_time=run_time,
     )

src/dq_suite/output_transformations.py

@@ -217,7 +217,7 @@ def extract_dq_afwijking_data(
 
     # "validation_results" is typed List[Dict[str, Any]] in GX
     dq_result = dq_result["validation_results"]
-    
+
     extracted_data = []
     if not isinstance(unique_identifier, list):
         unique_identifier = [unique_identifier]

tests/test_data/test_schema.py

@@ -0,0 +1,5 @@
+from pyspark.sql.types import StructType
+
+SCHEMA = (
+    StructType().add("the_string", "string").add("the_timestamp", "timestamp")
+)

tests/test_output_transformations.py

@@ -0,0 +1,22 @@
+
+import pytest
+from pyspark.sql import SparkSession
+
+from src.dq_suite.output_transformations import create_empty_dataframe
+
+from .test_data.test_schema import SCHEMA as AFWIJKING_SCHEMA
+
+
+@pytest.fixture()
+def spark():
+    return SparkSession.builder.master("local").appName("chispa").getOrCreate()
+
+
+@pytest.mark.usefixtures("spark")
+class TestCreateEmptyDataframe:
+    def test_create_empty_dataframe_returns_empty_dataframe(self, spark):
+        empty_dataframe = create_empty_dataframe(
+            spark_session=spark,
+            schema=AFWIJKING_SCHEMA,
+        )
+        assert len(empty_dataframe.head(1)) == 0