GX1.0 fixes

pyproject.toml

@@ -19,6 +19,7 @@ dependencies = [
   "pandas==2.1.4",
   "pyspark==3.5.2",
   "pyhumps==3.8.0",
+  "pyyaml==6.0.2",
 ]
 
 

src/dq_suite/common.py

@@ -1,8 +1,16 @@
 from dataclasses import dataclass
 from typing import Any, Dict, List, Literal
+import yaml
 
 from great_expectations import ExpectationSuite, get_context
-from great_expectations.data_context import AbstractDataContext
+from great_expectations.data_context import (
+    AbstractDataContext,
+    EphemeralDataContext,
+)
+from great_expectations.data_context.types.base import (
+        DataContextConfig,
+        InMemoryStoreBackendDefaults,
+    )
 from great_expectations.exceptions import DataContextError
 from pyspark.sql import DataFrame, SparkSession
 from pyspark.sql.functions import col
@@ -207,10 +215,13 @@ def merge_df_with_unity_table(
         .execute()
 
 
-def get_data_context(
-    data_context_root_dir: str = "/dbfs/great_expectations/",
-) -> AbstractDataContext:  # pragma: no cover - part of GX
-    return get_context(context_root_dir=data_context_root_dir)
+def get_data_context() -> AbstractDataContext:  # pragma: no cover - part of GX
+    return get_context(
+        project_config=DataContextConfig(
+            store_backend_defaults=InMemoryStoreBackendDefaults(),
+            analytics_enabled=False
+        )
+    )
 
 
 @dataclass()
@@ -285,9 +296,7 @@ def initialise_or_update_attributes(self):  # pragma: no cover - complex
             )
 
     def _set_data_context(self):  # pragma: no cover - uses part of GX
-        self.data_context = get_data_context(
-            data_context_root_dir=self.data_context_root_dir
-        )
+        self.data_context = get_data_context()
 
     def _set_expectation_suite_name(self):
         self.expectation_suite_name = f"{self.check_name}_expectation_suite"

src/dq_suite/output_transformations.py

@@ -50,7 +50,7 @@ def construct_regel_id(
 
 
 def create_parameter_list_from_results(result: dict) -> list[dict]:
-    parameters = result["expectation_config"]["kwargs"]
+    parameters = result["kwargs"]
     parameters.pop("batch_id", None)
     return [parameters]
 
@@ -88,7 +88,7 @@ def extract_dq_validatie_data(
             expectation_type = expectation_result["expectation_type"]
             parameter_list = create_parameter_list_from_results(result=expectation_result)
             attribute = expectation_result["kwargs"].get("column")
-            dq_regel_id = f"{df_name}_{expectation_type}_{attribute}"
+
             output = expectation_result["success"]
             output_text = "success" if output else "failure"
             extracted_data.append(