Refactor query execution to avoid toJSON call in SparkExpectationsAct…

…ions
Nike-Inc · Dec 13, 2024 · 01f61f6 · 01f61f6
1 parent 9500ed9
commit 01f61f6
Showing 1 changed file with 2 additions and 5 deletions.
diff --git a/spark_expectations/utils/actions.py b/spark_expectations/utils/actions.py
@@ -328,6 +328,7 @@ def agg_query_dq_detailed_result(
                     )
                 ):
                     for _key, _querydq_query in sub_key_value.items():
+                        _querydq_df = _context.spark.sql(_dq_rule["expectation" + "_" + _key])
                         querydq_output.append(
                             (
                                 _context.get_run_id,
@@ -340,11 +341,7 @@ def agg_query_dq_detailed_result(
                                     [
                                         (
                                             _key,
-                                            _context.spark.sql(
-                                                _dq_rule["expectation" + "_" + _key]
-                                            )
-                                            .toJSON()
-                                            .collect(),
+                                            [row.asDict() for row in _querydq_df.collect()],
                                         )
                                     ]
                                 ),