From e4b6d3cb5c884210a2e299ece3e43c3f3355060d Mon Sep 17 00:00:00 2001 From: zschira Date: Thu, 19 Dec 2024 17:41:55 -0500 Subject: [PATCH 1/4] Fix row count asset check for vcerare --- src/pudl/transform/vcerare.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/pudl/transform/vcerare.py b/src/pudl/transform/vcerare.py index 6ee9e6ce8..198677546 100644 --- a/src/pudl/transform/vcerare.py +++ b/src/pudl/transform/vcerare.py @@ -9,6 +9,7 @@ import pyarrow as pa import pyarrow.parquet as pq from dagster import ( + AssetCheckExecutionContext, AssetCheckResult, asset, asset_check, @@ -359,16 +360,25 @@ def _load_duckdb_table(): blocking=True, description="Check that row count matches expected.", ) -def check_rows() -> AssetCheckResult: +def check_rows(context: AssetCheckExecutionContext) -> AssetCheckResult: """Check rows.""" logger.info("Check VCE RARE hourly table is the expected length") + + # Define row counts for fast/full etl + row_counts = { + "etl_full": 136437000, + "etl_fast": 27287400, + } + vce = _load_duckdb_table() # noqa: F841 (length,) = duckdb.query("SELECT COUNT(*) FROM vce").fetchone() - if length != 136437000: + if ( + expecteded_length := row_counts[context.op_execution_context.job_name] + ) != length: return AssetCheckResult( passed=False, description="Table unexpected length", - metadata={"table_length": length, "expected_length": 136437000}, + metadata={"table_length": length, "expected_length": expecteded_length}, ) return AssetCheckResult(passed=True) From 9015db720f611ac5f84e313a44013500e346845e Mon Sep 17 00:00:00 2001 From: Zach Schira Date: Thu, 19 Dec 2024 17:49:42 -0500 Subject: [PATCH 2/4] Fix typo Co-authored-by: Dazhong Xia --- src/pudl/transform/vcerare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pudl/transform/vcerare.py b/src/pudl/transform/vcerare.py index 198677546..642422381 100644 --- a/src/pudl/transform/vcerare.py +++ b/src/pudl/transform/vcerare.py @@ -373,7 +373,7 @@ def check_rows(context: AssetCheckExecutionContext) -> AssetCheckResult: vce = _load_duckdb_table() # noqa: F841 (length,) = duckdb.query("SELECT COUNT(*) FROM vce").fetchone() if ( - expecteded_length := row_counts[context.op_execution_context.job_name] + expected_length := row_counts[context.op_execution_context.job_name] ) != length: return AssetCheckResult( passed=False, From c97e5a484529d10c44bc05e01f1b3e21666491ad Mon Sep 17 00:00:00 2001 From: Zach Schira Date: Thu, 19 Dec 2024 17:49:53 -0500 Subject: [PATCH 3/4] Fix typo Co-authored-by: Dazhong Xia --- src/pudl/transform/vcerare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pudl/transform/vcerare.py b/src/pudl/transform/vcerare.py index 642422381..6e28ca889 100644 --- a/src/pudl/transform/vcerare.py +++ b/src/pudl/transform/vcerare.py @@ -378,7 +378,7 @@ def check_rows(context: AssetCheckExecutionContext) -> AssetCheckResult: return AssetCheckResult( passed=False, description="Table unexpected length", - metadata={"table_length": length, "expected_length": expecteded_length}, + metadata={"table_length": length, "expected_length": expected_length}, ) return AssetCheckResult(passed=True) From 8790f65cf2424917d06ea5690847c2992f7c317c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 19 Dec 2024 22:51:27 +0000 Subject: [PATCH 4/4] [pre-commit.ci] auto fixes from pre-commit.com hooks For more information, see https://pre-commit.ci --- src/pudl/transform/vcerare.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/pudl/transform/vcerare.py b/src/pudl/transform/vcerare.py index 6e28ca889..840651f95 100644 --- a/src/pudl/transform/vcerare.py +++ b/src/pudl/transform/vcerare.py @@ -372,9 +372,7 @@ def check_rows(context: AssetCheckExecutionContext) -> AssetCheckResult: vce = _load_duckdb_table() # noqa: F841 (length,) = duckdb.query("SELECT COUNT(*) FROM vce").fetchone() - if ( - expected_length := row_counts[context.op_execution_context.job_name] - ) != length: + if (expected_length := row_counts[context.op_execution_context.job_name]) != length: return AssetCheckResult( passed=False, description="Table unexpected length",