Skip to content

Commit

Permalink
only check row nums for years that are in the dataset_settings
Browse files Browse the repository at this point in the history
  • Loading branch information
cmgosnell committed Dec 20, 2024
1 parent ab560e3 commit e03810e
Showing 1 changed file with 8 additions and 5 deletions.
13 changes: 8 additions & 5 deletions src/pudl/transform/ferc714.py
Original file line number Diff line number Diff line change
Expand Up @@ -1279,22 +1279,25 @@ class Ferc714CheckSpec:
def make_check(spec: Ferc714CheckSpec) -> AssetChecksDefinition:
"""Turn the Ferc714CheckSpec into an actual Dagster asset check."""

@asset_check(asset=spec.asset, blocking=True)
def _check(df):
@asset_check(
asset=spec.asset, required_resource_keys={"dataset_settings"}, blocking=True
)
def _row_num_check(context, df):
errors = []
for year, expected_rows in spec.num_rows_by_report_year.items():
for year in context.resources.dataset_settings.ferc714.years:
expected_rows = spec.num_rows_by_report_year[year]
if (num_rows := len(df.loc[df.report_year == year])) != expected_rows:
errors.append(
f"Expected {expected_rows} for report year {year}, found {num_rows}"
)
logger.info(errors)
logger.warning(errors)

if errors:
return AssetCheckResult(passed=False, metadata={"errors": errors})

return AssetCheckResult(passed=True)

return _check
return _row_num_check


_checks = [make_check(spec) for spec in check_specs]

0 comments on commit e03810e

Please sign in to comment.