Skip to content

Commit

Permalink
Update writer.py (#70)
Browse files Browse the repository at this point in the history
* Update writer.py

* Update test_writer.py

* Update CONTRIBUTORS.md
  • Loading branch information
jskrajareddy21 authored Feb 19, 2024
1 parent 426ec27 commit 89da08b
Show file tree
Hide file tree
Showing 3 changed files with 4 additions and 3 deletions.
1 change: 1 addition & 0 deletions CONTRIBUTORS.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ Thanks to the contributors who helped on this project apart from the authors
* [Holden Karau](https://www.linkedin.com/in/holdenkarau/)
* [Araveti Venkata Bharat Kumar](https://www.linkedin.com/in/bharat-kumar-araveti/)
* [Samy Coenen](https://github.com/SamyCoenen)
* [Jagadapi Sivanaga Krishnam Raja Reddy](www.linkedin.com/in/jskrajareddy/)

# Honorary Mentions
Thanks to the team below for invaluable insights and support throughout the initial release of this project
Expand Down
4 changes: 2 additions & 2 deletions spark_expectations/sinks/utils/writer.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ def save_df_as_table(
df = df.withColumn(
self._context.get_run_id_name, lit(f"{self._context.get_run_id}")
).withColumn(
self._context.get_run_date_name,
self._context.get_run_date_time_name,
to_timestamp(
lit(f"{self._context.get_run_date}"), "yyyy-MM-dd HH:mm:ss"
),
Expand Down Expand Up @@ -404,7 +404,7 @@ def write_error_records_final(
self._context.get_run_id_name, lit(self._context.get_run_id)
)
.withColumn(
self._context.get_run_date_name,
self._context.get_run_date_time_name,
lit(self._context.get_run_date),
)
)
Expand Down
2 changes: 1 addition & 1 deletion tests/sinks/utils/test_writer.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ def fixture_writer():
setattr(mock_context, "get_run_date", "2022-12-27 10:39:44")
setattr(mock_context, "get_run_id", "product1_run_test")
setattr(mock_context, "get_run_id_name", "meta_dq_run_id")
setattr(mock_context, "get_run_date_name", "meta_dq_run_date")
setattr(mock_context, "get_run_date_time_name", "meta_dq_run_date")
mock_context.spark = spark
mock_context.product_id = 'product1'

Expand Down

0 comments on commit 89da08b

Please sign in to comment.