Skip to content

Commit

Permalink
WIP: Use same dagster Definitions for test and prod.
Browse files Browse the repository at this point in the history
  • Loading branch information
jdangerx committed Dec 20, 2024
1 parent d6e385c commit ab560e3
Show file tree
Hide file tree
Showing 2 changed files with 3 additions and 19 deletions.
2 changes: 0 additions & 2 deletions src/pudl/etl/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,6 @@
from pudl.settings import EtlSettings

from . import (
check_foreign_keys,
cli,
eia_bulk_elec_assets,
epacems_assets,
glue_assets,
Expand Down
20 changes: 3 additions & 17 deletions src/pudl/etl/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,13 @@
import fsspec
from dagster import (
DagsterInstance,
Definitions,
JobDefinition,
build_reconstructable_job,
define_asset_job,
execute_job,
)

import pudl
from pudl.etl import defs
from pudl.helpers import get_dagster_execution_config
from pudl.settings import EpaCemsSettings, EtlSettings
from pudl.workspace.setup import PudlPaths
Expand All @@ -40,21 +39,8 @@ def pudl_etl_job_factory(
def get_pudl_etl_job():
"""Create an pudl_etl_job wrapped by to be wrapped by reconstructable."""
pudl.logging_helpers.configure_root_logger(logfile=logfile, loglevel=loglevel)
jobs = [define_asset_job("etl_job")]
if not process_epacems:
jobs = [
define_asset_job(
"etl_job",
selection=pudl.etl.create_non_cems_selection(
pudl.etl.default_assets
),
)
]
return Definitions(
assets=pudl.etl.default_assets,
resources=pudl.etl.default_resources,
jobs=jobs,
).get_job_def("etl_job")
job_name = "etl_full_no_cems" if not process_epacems else "etl_full"
return defs.get_job_def(job_name)

return get_pudl_etl_job

Expand Down

0 comments on commit ab560e3

Please sign in to comment.