Skip to content

Commit

Permalink
Fix row count asset check for vcerare
Browse files Browse the repository at this point in the history
  • Loading branch information
zschira committed Dec 19, 2024
1 parent 0dd0530 commit e4b6d3c
Showing 1 changed file with 13 additions and 3 deletions.
16 changes: 13 additions & 3 deletions src/pudl/transform/vcerare.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import pyarrow as pa
import pyarrow.parquet as pq
from dagster import (
AssetCheckExecutionContext,
AssetCheckResult,
asset,
asset_check,
Expand Down Expand Up @@ -359,16 +360,25 @@ def _load_duckdb_table():
blocking=True,
description="Check that row count matches expected.",
)
def check_rows() -> AssetCheckResult:
def check_rows(context: AssetCheckExecutionContext) -> AssetCheckResult:
"""Check rows."""
logger.info("Check VCE RARE hourly table is the expected length")

# Define row counts for fast/full etl
row_counts = {
"etl_full": 136437000,
"etl_fast": 27287400,
}

vce = _load_duckdb_table() # noqa: F841
(length,) = duckdb.query("SELECT COUNT(*) FROM vce").fetchone()
if length != 136437000:
if (
expecteded_length := row_counts[context.op_execution_context.job_name]
) != length:
return AssetCheckResult(
passed=False,
description="Table unexpected length",
metadata={"table_length": length, "expected_length": 136437000},
metadata={"table_length": length, "expected_length": expecteded_length},
)
return AssetCheckResult(passed=True)

Expand Down

0 comments on commit e4b6d3c

Please sign in to comment.